Dynamic Shapes (#2442)

Summary:
Pull Request resolved: #2442

Only need to look at tester.py file for the tester changes.

Change is from `.run_method().compare_outputs() ` to `.run_method_and_compare_outputs()`

now if Tester is initialized with dynamic inputs, we will generate random dynamic inputs (according to the specification of the dynamic shapes)  to run on the model. This allows us to test that the inputs fed into the model can be dynamic.

We ad a num_runs to run_method_and_compare_outputs so that we can choose to run a number of different dynamic inputs with dynamic shapes.

Reviewed By: digantdesai

Differential Revision: D54650121

Author: mcr229

backends/xnnpack/test/models/deeplab_v3.py

@@ -36,6 +36,5 @@ def test_fp32_dl3(self):
             .partition()
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )

backends/xnnpack/test/models/edsr.py

@@ -25,8 +25,7 @@ def test_fp32_edsr(self):
             .partition()
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )
 
     def test_qs8_edsr(self):
@@ -38,6 +37,5 @@ def test_qs8_edsr(self):
             .partition()
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )

backends/xnnpack/test/models/emformer_rnnt.py

@@ -43,8 +43,7 @@ def test_fp32_emformer_joiner(self):
             .check(["torch.ops.higher_order.executorch_call_delegate"])
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )
 
     class Predictor(EmformerRnnt):
@@ -69,8 +68,7 @@ def test_fp32_emformer_predictor(self):
             .check(["torch.ops.higher_order.executorch_call_delegate"])
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )
 
     class Transcriber(EmformerRnnt):
@@ -94,6 +92,5 @@ def test_fp32_emformer_transcriber(self):
             .check(["torch.ops.higher_order.executorch_call_delegate"])
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )

backends/xnnpack/test/models/inception_v3.py

@@ -42,8 +42,7 @@ def test_fp32_ic3(self):
             .check_not(list(self.all_operators))
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )
 
     def test_qs8_ic3(self):
@@ -63,6 +62,5 @@ def test_qs8_ic3(self):
             .check_not(list(ops_after_quantization))
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )

backends/xnnpack/test/models/inception_v4.py

@@ -39,8 +39,7 @@ def test_fp32_ic4(self):
             .check_not(list(self.all_operators))
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )
 
     def test_qs8_ic4(self):
@@ -60,6 +59,5 @@ def test_qs8_ic4(self):
             .check_not(list(ops_after_quantization))
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )

backends/xnnpack/test/models/llama2_et_example.py

@@ -45,6 +45,5 @@ def _test(self, dtype: torch.dtype = torch.float):
             .dump_artifact()
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs(atol=5e-2)
+            .run_method_and_compare_outputs(atol=5e-2)
         )

backends/xnnpack/test/models/mobilebert.py

@@ -38,6 +38,5 @@ def test_fp32_mobilebert(self):
             .check_not(list(self.supported_ops))
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )

backends/xnnpack/test/models/mobilenet_v2.py

@@ -40,8 +40,7 @@ def test_fp32_mv2(self):
             .check_not(list(self.all_operators))
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )
 
     def test_qs8_mv2(self):
@@ -61,6 +60,5 @@ def test_qs8_mv2(self):
             .check_not(list(ops_after_quantization))
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )

backends/xnnpack/test/models/mobilenet_v3.py

@@ -42,8 +42,7 @@ def test_fp32_mv3(self):
             .check_not(list(self.all_operators))
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )
 
     def test_qs8_mv3(self):
@@ -63,6 +62,5 @@ def test_qs8_mv3(self):
             .check_not(list(ops_after_lowering))
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )

backends/xnnpack/test/models/resnet.py

@@ -23,8 +23,7 @@ def test_fp32_resnet18(self):
             .partition()
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )
 
     def test_qs8_resnet18(self):
@@ -37,6 +36,5 @@ def test_qs8_resnet18(self):
             .partition()
             .to_executorch()
             .serialize()
-            .run_method()
-            .compare_outputs()
+            .run_method_and_compare_outputs()
         )