Merge pull request #4164 from Zac-HD/report-spans

Experimental changes for Crosshair support

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: minor
+
+This release adds ``.span_start()`` and ``.span_end()`` methods
+to our internal ``PrimitiveProvider`` interface, for use by
+:ref:`alternative-backends`.

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -1341,6 +1341,27 @@ def draw_bytes(
     ) -> bytes:
         raise NotImplementedError
 
+    def span_start(self, label: int, /) -> None:  # noqa: B027  # non-abstract noop
+        """Marks the beginning of a semantically meaningful span.
+
+        Providers can optionally track this data to learn which sub-sequences
+        of draws correspond to a higher-level object, recovering the parse tree.
+        `label` is an opaque integer, which will be shared by all spans drawn
+        from a particular strategy.
+
+        This method is called from ConjectureData.start_example().
+        """
+
+    def span_end(self, discard: bool, /) -> None:  # noqa: B027  # non-abstract noop
+        """Marks the end of a semantically meaningful span.
+
+        `discard` is True when the draw was filtered out or otherwise marked as
+        unlikely to contribute to the input data as seen by the user's test.
+        Note however that side effects can make this determination unsound.
+
+        This method is called from ConjectureData.stop_example().
+        """
+
 
 class HypothesisProvider(PrimitiveProvider):
     lifetime = "test_case"
@@ -2543,6 +2564,7 @@ def draw(
             self.stop_example()
 
     def start_example(self, label: int) -> None:
+        self.provider.span_start(label)
         self.__assert_not_frozen("start_example")
         self.depth += 1
         # Logically it would make sense for this to just be
@@ -2557,6 +2579,7 @@ def start_example(self, label: int) -> None:
         self.labels_for_structure_stack.append({label})
 
     def stop_example(self, *, discard: bool = False) -> None:
+        self.provider.span_end(discard)
         if self.frozen:
             return
         if discard:

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -284,6 +284,12 @@ def __init__(
         self.__failed_realize_count = 0
         self._verified_by = None  # note unsound verification by alt backends
 
+    @property
+    def using_hypothesis_backend(self):
+        return (
+            self.settings.backend == "hypothesis" or self._switch_to_hypothesis_provider
+        )
+
     def explain_next_call_as(self, explanation: str) -> None:
         self.__pending_call_explanation = explanation
 
@@ -314,7 +320,7 @@ def should_optimise(self) -> bool:
         return Phase.target in self.settings.phases
 
     def __tree_is_exhausted(self) -> bool:
-        return self.tree.is_exhausted and self.settings.backend == "hypothesis"
+        return self.tree.is_exhausted and self.using_hypothesis_backend
 
     def __stoppable_test_function(self, data: ConjectureData) -> None:
         """Run ``self._test_function``, but convert a ``StopTest`` exception
@@ -475,6 +481,9 @@ def test_function(self, data: ConjectureData) -> None:
                     and (self.__failed_realize_count / self.call_count) > 0.2
                 ):
                     self._switch_to_hypothesis_provider = True
+            # skip the post-test-case tracking; we're pretending this never happened
+            interrupted = True
+            return
         except BaseException:
             self.save_buffer(data.buffer)
             raise
@@ -562,7 +571,7 @@ def test_function(self, data: ConjectureData) -> None:
             self.valid_examples += 1
 
         if data.status == Status.INTERESTING:
-            if self.settings.backend != "hypothesis":
+            if not self.using_hypothesis_backend:
                 # drive the ir tree through the test function to convert it
                 # to a buffer
                 initial_origin = data.interesting_origin
@@ -1034,7 +1043,7 @@ def generate_new_examples(self) -> None:
             # a buffer and uses HypothesisProvider as its backing provider,
             # not whatever is specified by the backend. We can improve this
             # once more things are on the ir.
-            if self.settings.backend != "hypothesis":
+            if not self.using_hypothesis_backend:
                 data = self.new_conjecture_data(prefix=b"", max_length=BUFFER_SIZE)
                 with suppress(BackendCannotProceed):
                     self.test_function(data)
@@ -1310,7 +1319,7 @@ def new_conjecture_data_ir(
             HypothesisProvider if self._switch_to_hypothesis_provider else self.provider
         )
         observer = observer or self.tree.new_observer()
-        if self.settings.backend != "hypothesis":
+        if not self.using_hypothesis_backend:
             observer = DataObserver()
 
         return ConjectureData.for_ir_tree(
@@ -1331,7 +1340,7 @@ def new_conjecture_data(
             HypothesisProvider if self._switch_to_hypothesis_provider else self.provider
         )
         observer = observer or self.tree.new_observer()
-        if self.settings.backend != "hypothesis":
+        if not self.using_hypothesis_backend:
             observer = DataObserver()
 
         return ConjectureData(
@@ -1499,7 +1508,7 @@ def check_result(
             prefix=buffer, max_length=max_length, observer=observer
         )
 
-        if self.settings.backend == "hypothesis":
+        if self.using_hypothesis_backend:
             try:
                 self.tree.simulate_test_function(dummy_data)
             except PreviouslyUnseenBehaviour:

hypothesis-python/tests/conjecture/test_alt_backend.py

@@ -8,6 +8,7 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import itertools
 import math
 import sys
 from contextlib import contextmanager
@@ -16,14 +17,15 @@
 
 import pytest
 
-from hypothesis import given, settings, strategies as st
+from hypothesis import HealthCheck, assume, given, settings, strategies as st
 from hypothesis.control import current_build_context
 from hypothesis.database import InMemoryExampleDatabase
 from hypothesis.errors import (
     BackendCannotProceed,
     Flaky,
     HypothesisException,
     InvalidArgument,
+    Unsatisfiable,
 )
 from hypothesis.internal.compat import int_to_bytes
 from hypothesis.internal.conjecture.data import (
@@ -492,15 +494,13 @@ def test_function(_):
 class FallibleProvider(TrivialProvider):
     def __init__(self, conjecturedata: "ConjectureData", /) -> None:
         super().__init__(conjecturedata)
-        self.prng = Random(0)
+        self._it = itertools.cycle([1, 1, 1, "discard_test_case", "other"])
 
     def draw_integer(self, *args, **kwargs):
-        # This is frequent enough that we'll get coverage of the "give up and go
-        # back to Hypothesis' standard backend" code path.
-        if self.prng.getrandbits(1):
-            scope = self.prng.choice(["discard_test_case", "other"])
-            raise BackendCannotProceed(scope)
-        return 1
+        x = next(self._it)
+        if isinstance(x, str):
+            raise BackendCannotProceed(x)
+        return x
 
 
 def test_falls_back_to_default_backend():
@@ -517,6 +517,23 @@ def test_function(x):
         assert seen_other_ints  # must have swapped backends then
 
 
+def test_can_raise_unsatisfiable_after_falling_back():
+    with temp_register_backend("fallible", FallibleProvider):
+
+        @given(st.integers())
+        @settings(
+            backend="fallible",
+            database=None,
+            max_examples=100,
+            suppress_health_check=[HealthCheck.filter_too_much],
+        )
+        def test_function(x):
+            assume(x == "unsatisfiable")
+
+        with pytest.raises(Unsatisfiable):
+            test_function()
+
+
 class ExhaustibleProvider(TrivialProvider):
     scope = "exhausted"