Add integration tests lm generate torch tf

src/transformers/modeling_tf_xlm.py

@@ -408,7 +408,7 @@ def call(
             inputs_embeds = self.embeddings(input_ids)
 
         tensor = inputs_embeds + self.position_embeddings(position_ids)
-        if langs is not None and self.use_lang_emb:
+        if langs is not None and self.use_lang_emb and self.n_langs > 1:
             tensor = tensor + self.lang_embeddings(langs)
         if token_type_ids is not None:
             tensor = tensor + self.embeddings(token_type_ids)

tests/test_modeling_ctrl.py

@@ -219,30 +219,31 @@ class CTRLModelLanguageGenerationTest(unittest.TestCase):
     @slow
     def test_lm_generate_ctrl(self):
         model = CTRLLMHeadModel.from_pretrained("ctrl")
-        input_ids = torch.Tensor([[11859, 586, 20984, 8]]).long()  # Legal My neighbor is
+        input_ids = torch.tensor(
+            [[11859, 0, 1611, 8]], dtype=torch.long, device=torch_device
+        )  # Legal the president is
         expected_output_ids = [
             11859,
-            586,
-            20984,
+            0,
+            1611,
             8,
-            13391,
-            3,
-            980,
-            8258,
-            72,
-            327,
-            148,
+            5,
+            150,
+            26449,
             2,
-            53,
-            29,
-            226,
-            3,
-            780,
-            49,
+            19,
+            348,
+            469,
             3,
-            980,
-        ]  # Legal My neighbor is refusing to pay rent after 2 years and we are having to force him to pay
-        torch.manual_seed(0)
-
-        output_ids = model.generate(input_ids)
+            2595,
+            48,
+            20740,
+            246533,
+            246533,
+            19,
+            30,
+            5,
+        ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a
+
+        output_ids = model.generate(input_ids, do_sample=False)
         self.assertListEqual(output_ids[0].tolist(), expected_output_ids)

tests/test_modeling_gpt2.py

@@ -223,7 +223,7 @@ def create_and_check_gpt2_model_attention_mask_past(
             # append to next input_ids and attn_mask
             next_input_ids = torch.cat([input_ids, next_tokens], dim=-1)
             attn_mask = torch.cat(
-                [attn_mask, torch.ones((attn_mask.shape[0], 1), dtype=torch.long, device=torch_device)], dim=1
+                [attn_mask, torch.ones((attn_mask.shape[0], 1), dtype=torch.long, device=torch_device)], dim=1,
             )
 
             # get two different outputs
@@ -343,39 +343,36 @@ class GPT2ModelLanguageGenerationTest(unittest.TestCase):
     @slow
     def test_lm_generate_gpt2(self):
         model = GPT2LMHeadModel.from_pretrained("gpt2")
-        input_ids = torch.Tensor([[464, 3290, 318, 13779]]).long()  # The dog is cute
+        input_ids = torch.tensor([[464, 3290]], dtype=torch.long, device=torch_device)  # The dog
         expected_output_ids = [
             464,
             3290,
-            318,
-            13779,
-            1165,
-            13,
-            632,
-            7832,
-            284,
-            6437,
-            319,
-            502,
+            373,
+            1043,
+            287,
+            257,
+            2214,
+            1474,
+            262,
+            16246,
+            286,
+            2688,
             290,
-            318,
-            922,
-            329,
-            502,
-            357,
-            1169,
+            2688,
+            27262,
+            13,
+            198,
+            198,
+            464,
             3290,
-        ]  # The dog is cute too. It likes to rub on me and is good for me (the dog
-        torch.manual_seed(0)
-
-        output_ids = model.generate(input_ids)
-
+        ]  # The dog was found in a field near the intersection of West and West Streets.\n\nThe dog
+        output_ids = model.generate(input_ids, do_sample=False)
         self.assertListEqual(output_ids[0].tolist(), expected_output_ids)
 
     @slow
     def test_lm_generate_distilgpt2(self):
         model = GPT2LMHeadModel.from_pretrained("distilgpt2")
-        input_ids = torch.Tensor([[464, 1893]]).long()  # The president
+        input_ids = torch.tensor([[464, 1893]], dtype=torch.long, device=torch_device)  # The president
         expected_output_ids = [
             464,
             1893,

tests/test_modeling_openai.py

@@ -123,7 +123,15 @@ def prepare_config_and_inputs(self):
 
             head_mask = ids_tensor([self.num_hidden_layers, self.num_attention_heads], 2)
 
-            return config, input_ids, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels
+            return (
+                config,
+                input_ids,
+                head_mask,
+                token_type_ids,
+                sequence_labels,
+                token_labels,
+                choice_labels,
+            )
 
         def check_loss_output(self, result):
             self.parent.assertListEqual(list(result["loss"].size()), [])
@@ -139,7 +147,7 @@ def create_and_check_openai_gpt_model(self, config, input_ids, head_mask, token_
 
             result = {"sequence_output": sequence_output}
             self.parent.assertListEqual(
-                list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
+                list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size],
             )
 
         def create_and_check_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args):
@@ -153,7 +161,7 @@ def create_and_check_lm_head_model(self, config, input_ids, head_mask, token_typ
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(
-                list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size]
+                list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size],
             )
 
         def create_and_check_double_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args):
@@ -167,7 +175,7 @@ def create_and_check_double_lm_head_model(self, config, input_ids, head_mask, to
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(
-                list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size]
+                list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size],
             )
 
         def prepare_config_and_inputs_for_common(self):
@@ -181,7 +189,11 @@ def prepare_config_and_inputs_for_common(self):
                 token_labels,
                 choice_labels,
             ) = config_and_inputs
-            inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "head_mask": head_mask}
+            inputs_dict = {
+                "input_ids": input_ids,
+                "token_type_ids": token_type_ids,
+                "head_mask": head_mask,
+            }
 
             return config, inputs_dict
 
@@ -215,30 +227,29 @@ class OPENAIGPTModelLanguageGenerationTest(unittest.TestCase):
     @slow
     def test_lm_generate_openai_gpt(self):
         model = OpenAIGPTLMHeadModel.from_pretrained("openai-gpt")
-        input_ids = torch.Tensor([[481, 2585, 544, 4957]]).long()  # The dog is cute
+        input_ids = torch.tensor([[481, 4735, 544]], dtype=torch.long, device=torch_device)  # the president is
         expected_output_ids = [
             481,
-            2585,
+            4735,
             544,
-            4957,
-            669,
-            512,
-            761,
-            5990,
-            271,
-            645,
+            246,
+            963,
+            870,
+            762,
+            239,
+            244,
+            40477,
+            244,
+            249,
+            719,
+            881,
             487,
-            535,
-            976,
-            2479,
+            544,
             240,
-            487,
-            804,
-            1296,
-            2891,
-            512,
-        ]  # the dog is cute when you're annoyed : if he's really stupid, he 'll stop fighting you
-        torch.manual_seed(0)
-
-        output_ids = model.generate(input_ids)
+            244,
+            603,
+            481,
+        ]  # the president is a very good man. " \n " i\'m sure he is, " said the
+
+        output_ids = model.generate(input_ids, do_sample=False)
         self.assertListEqual(output_ids[0].tolist(), expected_output_ids)

tests/test_modeling_tf_ctrl.py

@@ -24,6 +24,7 @@
 
 
 if is_tf_available():
+    import tensorflow as tf
     from transformers.modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP
 
 
@@ -202,3 +203,35 @@ def test_model_from_pretrained(self):
         for model_name in list(TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
             model = TFCTRLModel.from_pretrained(model_name, cache_dir=CACHE_DIR)
             self.assertIsNotNone(model)
+
+
+class TFCTRLModelLanguageGenerationTest(unittest.TestCase):
+    @slow
+    def test_lm_generate_ctrl(self):
+        model = TFCTRLLMHeadModel.from_pretrained("ctrl")
+        input_ids = tf.convert_to_tensor([[11859, 0, 1611, 8]], dtype=tf.int32)  # Legal the president is
+        expected_output_ids = [
+            11859,
+            0,
+            1611,
+            8,
+            5,
+            150,
+            26449,
+            2,
+            19,
+            348,
+            469,
+            3,
+            2595,
+            48,
+            20740,
+            246533,
+            246533,
+            19,
+            30,
+            5,
+        ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a
+
+        output_ids = model.generate(input_ids, do_sample=False)
+        self.assertListEqual(output_ids[0].numpy().tolist(), expected_output_ids)

tests/test_modeling_tf_gpt2.py

@@ -328,13 +328,35 @@ def test_model_from_pretrained(self):
             self.assertIsNotNone(model)
 
 
-def prepare_generation_special_tokens():
-    return {"bos_token_id": 50256, "eos_token_id": 50256}
-
-
 class TFGPT2ModelLanguageGenerationTest(unittest.TestCase):
-
-    special_tokens = prepare_generation_special_tokens()
+    @slow
+    def test_lm_generate_gpt2(self):
+        model = TFGPT2LMHeadModel.from_pretrained("gpt2")
+        input_ids = tf.convert_to_tensor([[464, 3290]], dtype=tf.int32)  # The dog
+        expected_output_ids = [
+            464,
+            3290,
+            373,
+            1043,
+            287,
+            257,
+            2214,
+            1474,
+            262,
+            16246,
+            286,
+            2688,
+            290,
+            2688,
+            27262,
+            13,
+            198,
+            198,
+            464,
+            3290,
+        ]  # The dog was found in a field near the intersection of West and West Streets.\n\nThe dog
+        output_ids = model.generate(input_ids, do_sample=False)
+        self.assertListEqual(output_ids[0].numpy().tolist(), expected_output_ids)
 
     @slow
     def test_lm_generate_distilgpt2(self):
@@ -363,11 +385,5 @@ def test_lm_generate_distilgpt2(self):
             2635,
         ]  # The president of the United States, and the president of the United Kingdom, have been in the White
 
-        output_ids = model.generate(
-            input_ids,
-            do_sample=False,
-            bos_token_id=self.special_tokens["bos_token_id"],
-            eos_token_ids=self.special_tokens["eos_token_id"],
-        )
-
+        output_ids = model.generate(input_ids, do_sample=False)
         self.assertListEqual(output_ids[0].numpy().tolist(), expected_output_ids)

tests/test_modeling_tf_openai_gpt.py

@@ -238,3 +238,35 @@ def test_model_from_pretrained(self):
         for model_name in list(TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
             model = TFOpenAIGPTModel.from_pretrained(model_name, cache_dir=CACHE_DIR)
             self.assertIsNotNone(model)
+
+
+class TFOPENAIGPTModelLanguageGenerationTest(unittest.TestCase):
+    @slow
+    def test_lm_generate_openai_gpt(self):
+        model = TFOpenAIGPTLMHeadModel.from_pretrained("openai-gpt")
+        input_ids = tf.convert_to_tensor([[481, 4735, 544]], dtype=tf.int32)  # the president is
+        expected_output_ids = [
+            481,
+            4735,
+            544,
+            246,
+            963,
+            870,
+            762,
+            239,
+            244,
+            40477,
+            244,
+            249,
+            719,
+            881,
+            487,
+            544,
+            240,
+            244,
+            603,
+            481,
+        ]  # the president is a very good man. " \n " i\'m sure he is, " said the
+
+        output_ids = model.generate(input_ids, do_sample=False)
+        self.assertListEqual(output_ids[0].numpy().tolist(), expected_output_ids)