gh-62432: unittest runner: Exit code 5 if no tests were run (#102051)

As discussed in https://discuss.python.org/t/unittest-fail-if-zero-tests-were-discovered/21498/7

It is common for test runner misconfiguration to fail to find any tests,
This should be an error.

Fixes: #62432

Author: stefanor

Doc/library/unittest.rst

@@ -2281,7 +2281,8 @@ Loading and running tests
 
    The *testRunner* argument can either be a test runner class or an already
    created instance of it. By default ``main`` calls :func:`sys.exit` with
-   an exit code indicating success or failure of the tests run.
+   an exit code indicating success (0) or failure (1) of the tests run.
+   An exit code of 5 indicates that no tests were run.
 
    The *testLoader* argument has to be a :class:`TestLoader` instance,
    and defaults to :data:`defaultTestLoader`.

Lib/test/test_unittest/test_program.py

@@ -71,15 +71,22 @@ def testExpectedFailure(self):
         def testUnexpectedSuccess(self):
             pass
 
-    class FooBarLoader(unittest.TestLoader):
-        """Test loader that returns a suite containing FooBar."""
+    class Empty(unittest.TestCase):
+        pass
+
+    class TestLoader(unittest.TestLoader):
+        """Test loader that returns a suite containing the supplied testcase."""
+
+        def __init__(self, testcase):
+            self.testcase = testcase
+
         def loadTestsFromModule(self, module):
             return self.suiteClass(
-                [self.loadTestsFromTestCase(Test_TestProgram.FooBar)])
+                [self.loadTestsFromTestCase(self.testcase)])
 
         def loadTestsFromNames(self, names, module):
             return self.suiteClass(
-                [self.loadTestsFromTestCase(Test_TestProgram.FooBar)])
+                [self.loadTestsFromTestCase(self.testcase)])
 
     def test_defaultTest_with_string(self):
         class FakeRunner(object):
@@ -92,7 +99,7 @@ def run(self, test):
         runner = FakeRunner()
         program = unittest.TestProgram(testRunner=runner, exit=False,
                                        defaultTest='test.test_unittest',
-                                       testLoader=self.FooBarLoader())
+                                       testLoader=self.TestLoader(self.FooBar))
         sys.argv = old_argv
         self.assertEqual(('test.test_unittest',), program.testNames)
 
@@ -108,7 +115,7 @@ def run(self, test):
         program = unittest.TestProgram(
             testRunner=runner, exit=False,
             defaultTest=['test.test_unittest', 'test.test_unittest2'],
-            testLoader=self.FooBarLoader())
+            testLoader=self.TestLoader(self.FooBar))
         sys.argv = old_argv
         self.assertEqual(['test.test_unittest', 'test.test_unittest2'],
                           program.testNames)
@@ -118,7 +125,7 @@ def test_NonExit(self):
         program = unittest.main(exit=False,
                                 argv=["foobar"],
                                 testRunner=unittest.TextTestRunner(stream=stream),
-                                testLoader=self.FooBarLoader())
+                                testLoader=self.TestLoader(self.FooBar))
         self.assertTrue(hasattr(program, 'result'))
         out = stream.getvalue()
         self.assertIn('\nFAIL: testFail ', out)
@@ -130,13 +137,13 @@ def test_NonExit(self):
 
     def test_Exit(self):
         stream = BufferedWriter()
-        self.assertRaises(
-            SystemExit,
-            unittest.main,
-            argv=["foobar"],
-            testRunner=unittest.TextTestRunner(stream=stream),
-            exit=True,
-            testLoader=self.FooBarLoader())
+        with self.assertRaises(SystemExit) as cm:
+            unittest.main(
+                argv=["foobar"],
+                testRunner=unittest.TextTestRunner(stream=stream),
+                exit=True,
+                testLoader=self.TestLoader(self.FooBar))
+        self.assertEqual(cm.exception.code, 1)
         out = stream.getvalue()
         self.assertIn('\nFAIL: testFail ', out)
         self.assertIn('\nERROR: testError ', out)
@@ -147,12 +154,11 @@ def test_Exit(self):
 
     def test_ExitAsDefault(self):
         stream = BufferedWriter()
-        self.assertRaises(
-            SystemExit,
-            unittest.main,
-            argv=["foobar"],
-            testRunner=unittest.TextTestRunner(stream=stream),
-            testLoader=self.FooBarLoader())
+        with self.assertRaises(SystemExit):
+            unittest.main(
+                argv=["foobar"],
+                testRunner=unittest.TextTestRunner(stream=stream),
+                testLoader=self.TestLoader(self.FooBar))
         out = stream.getvalue()
         self.assertIn('\nFAIL: testFail ', out)
         self.assertIn('\nERROR: testError ', out)
@@ -161,6 +167,17 @@ def test_ExitAsDefault(self):
                     'expected failures=1, unexpected successes=1)\n')
         self.assertTrue(out.endswith(expected))
 
+    def test_ExitEmptySuite(self):
+        stream = BufferedWriter()
+        with self.assertRaises(SystemExit) as cm:
+            unittest.main(
+                argv=["empty"],
+                testRunner=unittest.TextTestRunner(stream=stream),
+                testLoader=self.TestLoader(self.Empty))
+        self.assertEqual(cm.exception.code, 5)
+        out = stream.getvalue()
+        self.assertIn('\nNO TESTS RAN\n', out)
+
 
 class InitialisableProgram(unittest.TestProgram):
     exit = False

Lib/test/test_unittest/test_result.py

@@ -451,6 +451,7 @@ def testFailFastSetByRunner(self):
         stream = BufferedWriter()
         runner = unittest.TextTestRunner(stream=stream, failfast=True)
         def test(result):
+            result.testsRun += 1
             self.assertTrue(result.failfast)
         result = runner.run(test)
         stream.flush()

Lib/test/test_unittest/test_runner.py

@@ -577,6 +577,16 @@ def test(self):
                 'inner setup', 'inner test', 'inner cleanup',
                 'end outer test', 'outer cleanup'])
 
+    def test_run_empty_suite_error_message(self):
+        class EmptyTest(unittest.TestCase):
+            pass
+
+        suite = unittest.defaultTestLoader.loadTestsFromTestCase(EmptyTest)
+        runner = getRunner()
+        runner.run(suite)
+
+        self.assertIn("\nNO TESTS RAN\n", runner.stream.getvalue())
+
 
 class TestModuleCleanUp(unittest.TestCase):
     def test_add_and_do_ModuleCleanup(self):

Lib/unittest/main.py

@@ -9,6 +9,7 @@
 from .signals import installHandler
 
 __unittest = True
+_NO_TESTS_EXITCODE = 5
 
 MAIN_EXAMPLES = """\
 Examples:
@@ -279,6 +280,12 @@ def runTests(self):
             testRunner = self.testRunner
         self.result = testRunner.run(self.test)
         if self.exit:
-            sys.exit(not self.result.wasSuccessful())
+            if self.result.testsRun == 0:
+                sys.exit(_NO_TESTS_EXITCODE)
+            elif self.result.wasSuccessful():
+                sys.exit(0)
+            else:
+                sys.exit(1)
+
 
 main = TestProgram

Lib/unittest/runner.py

@@ -274,6 +274,8 @@ def run(self, test):
                 infos.append("failures=%d" % failed)
             if errored:
                 infos.append("errors=%d" % errored)
+        elif run == 0:
+            self.stream.write("NO TESTS RAN")
         else:
             self.stream.write("OK")
         if skipped:

Misc/ACKS

@@ -1513,6 +1513,7 @@ Vlad Riscutia
 Wes Rishel
 Daniel Riti
 Juan M. Bello Rivas
+Stefano Rivera
 Llandy Riveron Del Risco
 Mohd Sanad Zaki Rizvi
 Davide Rizzo

Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst

@@ -0,0 +1,3 @@
+The :mod:`unittest` runner will now exit with status code 5 if no tests
+were run. It is common for test runner misconfiguration to fail to find
+any tests, this should be an error.