Exit pytest on collection error (without executing tests)

Add --continue-on-collection-errors option to restore the previous behaviour:
Execute tests (that were successfully collected) even when collection errors
happen.

Some tests had to be modified e.g. because the return code changed to 2
(EXIT_INTERRUPTED) instead of 1 (EXIT_TESTSFAILED) because an Interrupted
exception is raised on collection error.

Implemented via pair programming with:
    Oleg Pidsadnyi <[email protected]>

refs #1421

Author: omarkohl

AUTHORS

@@ -72,6 +72,7 @@ Michael Birtwell
 Michael Droettboom
 Mike Lundy
 Nicolas Delaby
+Oleg Pidsadnyi
 Omar Kohl
 Pieter Mulder
 Piotr Banaszkiewicz

CHANGELOG.rst

@@ -74,6 +74,10 @@
   message to raise when no exception occurred.
   Thanks `@palaviv`_ for the complete PR (`#1616`_).
 
+* Fix `#1421`_: Exit tests if a collection error occurs and add
+  ``--continue-on-collection-errors`` option to restore previous behaviour.
+  Thanks `@olegpidsadnyi`_ and `@omarkohl`_ for the complete PR (`#1628`_).
+
 .. _@milliams: https://github.com/milliams
 .. _@csaftoiu: https://github.com/csaftoiu
 .. _@novas0x2a: https://github.com/novas0x2a
@@ -83,7 +87,9 @@
 .. _@palaviv: https://github.com/palaviv
 .. _@omarkohl: https://github.com/omarkohl
 .. _@mikofski: https://github.com/mikofski
+.. _@olegpidsadnyi: https://github.com/olegpidsadnyi
 
+.. _#1421: https://github.com/pytest-dev/pytest/issues/1421
 .. _#1426: https://github.com/pytest-dev/pytest/issues/1426
 .. _#1428: https://github.com/pytest-dev/pytest/pull/1428
 .. _#1444: https://github.com/pytest-dev/pytest/pull/1444
@@ -98,6 +104,7 @@
 .. _#372: https://github.com/pytest-dev/pytest/issues/372
 .. _#1544: https://github.com/pytest-dev/pytest/issues/1544
 .. _#1616: https://github.com/pytest-dev/pytest/pull/1616
+.. _#1628: https://github.com/pytest-dev/pytest/pull/1628
 
 
 **Bug Fixes**

_pytest/main.py

@@ -48,6 +48,9 @@ def pytest_addoption(parser):
                help="run pytest in strict mode, warnings become errors.")
     group._addoption("-c", metavar="file", type=str, dest="inifilename",
                help="load configuration from `file` instead of trying to locate one of the implicit configuration files.")
+    group._addoption("--continue-on-collection-errors", action="store_true",
+               default=False, dest="continue_on_collection_errors",
+               help="Force test execution even if collection errors occur.")
 
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly', '--collect-only', action="store_true",
@@ -133,6 +136,11 @@ def pytest_collection(session):
     return session.perform_collect()
 
 def pytest_runtestloop(session):
+    if (session.testsfailed and
+            not session.config.option.continue_on_collection_errors):
+        raise session.Interrupted(
+            "%d errors during collection" % session.testsfailed)
+
     if session.config.option.collectonly:
         return True
 

testing/acceptance_test.py

@@ -120,7 +120,7 @@ def test_this():
             "ImportError while importing test module*",
             "'No module named *does_not_work*",
         ])
-        assert result.ret == 1
+        assert result.ret == 2
 
     def test_not_collectable_arguments(self, testdir):
         p1 = testdir.makepyfile("")
@@ -665,11 +665,13 @@ def test_with_failing_collection(self, testdir):
         testdir.makepyfile(self.source)
         testdir.makepyfile(test_collecterror="""xyz""")
         result = testdir.runpytest("--durations=2", "-k test_1")
-        assert result.ret != 0
+        assert result.ret == 2
         result.stdout.fnmatch_lines([
-            "*durations*",
-            "*call*test_1*",
+            "*Interrupted: 1 errors during collection*",
         ])
+        # Collection errors abort test execution, therefore no duration is
+        # output
+        assert "duration" not in result.stdout.str()
 
     def test_with_not(self, testdir):
         testdir.makepyfile(self.source)

testing/test_collection.py

@@ -642,3 +642,114 @@ def test___repr__():
         """)
         reprec = testdir.inline_run("-k repr")
         reprec.assertoutcome(passed=1, failed=0)
+
+
+COLLECTION_ERROR_PY_FILES = dict(
+    test_01_failure="""
+        def test_1():
+            assert False
+        """,
+    test_02_import_error="""
+        import asdfasdfasdf
+        def test_2():
+            assert True
+        """,
+    test_03_import_error="""
+        import asdfasdfasdf
+        def test_3():
+            assert True
+    """,
+    test_04_success="""
+        def test_4():
+            assert True
+    """,
+)
+
+def test_exit_on_collection_error(testdir):
+    """Verify that all collection errors are collected and no tests executed"""
+    testdir.makepyfile(**COLLECTION_ERROR_PY_FILES)
+
+    res = testdir.runpytest()
+    assert res.ret == 2
+
+    res.stdout.fnmatch_lines([
+        "collected 2 items / 2 errors",
+        "*ERROR collecting test_02_import_error.py*",
+        "*No module named 'asdfa*",
+        "*ERROR collecting test_03_import_error.py*",
+        "*No module named 'asdfa*",
+    ])
+
+
+def test_exit_on_collection_with_maxfail_smaller_than_n_errors(testdir):
+    """
+    Verify collection is aborted once maxfail errors are encountered ignoring
+    further modules which would cause more collection errors.
+    """
+    testdir.makepyfile(**COLLECTION_ERROR_PY_FILES)
+
+    res = testdir.runpytest("--maxfail=1")
+    assert res.ret == 2
+
+    res.stdout.fnmatch_lines([
+        "*ERROR collecting test_02_import_error.py*",
+        "*No module named 'asdfa*",
+        "*Interrupted: stopping after 1 failures*",
+    ])
+
+    assert 'test_03' not in res.stdout.str()
+
+
+def test_exit_on_collection_with_maxfail_bigger_than_n_errors(testdir):
+    """
+    Verify the test run aborts due to collection errors even if maxfail count of
+    errors was not reached.
+    """
+    testdir.makepyfile(**COLLECTION_ERROR_PY_FILES)
+
+    res = testdir.runpytest("--maxfail=4")
+    assert res.ret == 2
+
+    res.stdout.fnmatch_lines([
+        "collected 2 items / 2 errors",
+        "*ERROR collecting test_02_import_error.py*",
+        "*No module named 'asdfa*",
+        "*ERROR collecting test_03_import_error.py*",
+        "*No module named 'asdfa*",
+    ])
+
+
+def test_continue_on_collection_errors(testdir):
+    """
+    Verify tests are executed even when collection errors occur when the
+    --continue-on-collection-errors flag is set
+    """
+    testdir.makepyfile(**COLLECTION_ERROR_PY_FILES)
+
+    res = testdir.runpytest("--continue-on-collection-errors")
+    assert res.ret == 1
+
+    res.stdout.fnmatch_lines([
+        "collected 2 items / 2 errors",
+        "*1 failed, 1 passed, 2 error*",
+    ])
+
+
+def test_continue_on_collection_errors_maxfail(testdir):
+    """
+    Verify tests are executed even when collection errors occur and that maxfail
+    is honoured (including the collection error count).
+    4 tests: 2 collection errors + 1 failure + 1 success
+    test_4 is never executed because the test run is with --maxfail=3 which
+    means it is interrupted after the 2 collection errors + 1 failure.
+    """
+    testdir.makepyfile(**COLLECTION_ERROR_PY_FILES)
+
+    res = testdir.runpytest("--continue-on-collection-errors", "--maxfail=3")
+    assert res.ret == 2
+
+    res.stdout.fnmatch_lines([
+        "collected 2 items / 2 errors",
+        "*Interrupted: stopping after 3 failures*",
+        "*1 failed, 2 error*",
+    ])

testing/test_doctest.py

@@ -199,8 +199,20 @@ def test(self):
             "*1 failed*",
         ])
 
+    def test_doctest_unex_importerror_only_txt(self, testdir):
+        testdir.maketxtfile("""
+            >>> import asdalsdkjaslkdjasd
+            >>>
+        """)
+        result = testdir.runpytest()
+        # doctest is never executed because of error during hello.py collection
+        result.stdout.fnmatch_lines([
+            "*>>> import asdals*",
+            "*UNEXPECTED*ImportError*",
+            "ImportError: No module named 'asdal*",
+        ])
 
-    def test_doctest_unex_importerror(self, testdir):
+    def test_doctest_unex_importerror_with_module(self, testdir):
         testdir.tmpdir.join("hello.py").write(_pytest._code.Source("""
             import asdalsdkjaslkdjasd
         """))
@@ -209,10 +221,11 @@ def test_doctest_unex_importerror(self, testdir):
             >>>
         """)
         result = testdir.runpytest("--doctest-modules")
+        # doctest is never executed because of error during hello.py collection
         result.stdout.fnmatch_lines([
-            "*>>> import hello",
-            "*UNEXPECTED*ImportError*",
-            "*import asdals*",
+            "*ERROR collecting hello.py*",
+            "*ImportError: No module named 'asdals*",
+            "*Interrupted: 1 errors during collection*",
         ])
 
     def test_doctestmodule(self, testdir):

testing/test_resultlog.py

@@ -231,6 +231,6 @@ def test_func():
             pass
     """)
     result = testdir.runpytest("--resultlog=log")
-    assert result.ret == 1
+    assert result.ret == 2
 
 

testing/test_terminal.py

@@ -273,7 +273,7 @@ def test_method(self):
     def test_collectonly_error(self, testdir):
         p = testdir.makepyfile("import Errlkjqweqwe")
         result = testdir.runpytest("--collect-only", p)
-        assert result.ret == 1
+        assert result.ret == 2
         result.stdout.fnmatch_lines(_pytest._code.Source("""
             *ERROR*
             *ImportError*