[3.12] GH-112215: Backport C recursion changes (GH-115083)

Author: markshannon

Include/cpython/pystate.h

@@ -251,12 +251,24 @@ struct _ts {
 /* WASI has limited call stack. Python's recursion limit depends on code
    layout, optimization, and WASI runtime. Wasmtime can handle about 700
    recursions, sometimes less. 500 is a more conservative limit. */
-#ifndef C_RECURSION_LIMIT
-#  ifdef __wasi__
+#ifdef Py_DEBUG
+#  if defined(__wasi__)
+#    define C_RECURSION_LIMIT 150
+#  else
+#    define C_RECURSION_LIMIT 500
+#  endif
+#else
+#  if defined(__wasi__)
 #    define C_RECURSION_LIMIT 500
+#  elif defined(__s390x__)
+#    define C_RECURSION_LIMIT 800
+#  elif defined(_WIN32)
+#    define C_RECURSION_LIMIT 3000
+#  elif defined(_Py_ADDRESS_SANITIZER)
+#    define C_RECURSION_LIMIT 4000
 #  else
-    // This value is duplicated in Lib/test/support/__init__.py
-#    define C_RECURSION_LIMIT 1500
+     // This value is duplicated in Lib/test/support/__init__.py
+#    define C_RECURSION_LIMIT 10000
 #  endif
 #endif
 

Lib/test/support/__init__.py

@@ -2112,13 +2112,13 @@ def set_recursion_limit(limit):
     finally:
         sys.setrecursionlimit(original_limit)
 
-def infinite_recursion(max_depth=100):
-    """Set a lower limit for tests that interact with infinite recursions
-    (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some
-    debug windows builds, due to not enough functions being inlined the
-    stack size might not handle the default recursion limit (1000). See
-    bpo-11105 for details."""
-    if max_depth < 3:
+def infinite_recursion(max_depth=None):
+    if max_depth is None:
+        # Pick a number large enough to cause problems
+        # but not take too long for code that can handle
+        # very deep recursion.
+        max_depth = 20_000
+    elif max_depth < 3:
         raise ValueError("max_depth must be at least 3, got {max_depth}")
     depth = get_recursion_depth()
     depth = max(depth - 1, 1)  # Ignore infinite_recursion() frame.
@@ -2362,7 +2362,22 @@ def adjust_int_max_str_digits(max_digits):
 EXCEEDS_RECURSION_LIMIT = 5000
 
 # The default C recursion limit (from Include/cpython/pystate.h).
-C_RECURSION_LIMIT = 1500
+if Py_DEBUG:
+    if is_wasi:
+        C_RECURSION_LIMIT = 150
+    else:
+        C_RECURSION_LIMIT = 500
+else:
+    if is_wasi:
+        C_RECURSION_LIMIT = 500
+    elif hasattr(os, 'uname') and os.uname().machine == 's390x':
+        C_RECURSION_LIMIT = 800
+    elif sys.platform.startswith('win'):
+        C_RECURSION_LIMIT = 3000
+    elif check_sanitizer(address=True):
+        C_RECURSION_LIMIT = 4000
+    else:
+        C_RECURSION_LIMIT = 10000
 
 #Windows doesn't have os.uname() but it doesn't support s390x.
 skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',

Lib/test/test_ast.py

@@ -1087,9 +1087,9 @@ def next(self):
     @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
     @support.cpython_only
     def test_ast_recursion_limit(self):
-        fail_depth = support.EXCEEDS_RECURSION_LIMIT
+        fail_depth = support.C_RECURSION_LIMIT + 1
         crash_depth = 100_000
-        success_depth = 1200
+        success_depth = int(support.C_RECURSION_LIMIT * 0.9)
 
         def check_limit(prefix, repeated):
             expect_ok = prefix + repeated * success_depth

Lib/test/test_call.py

@@ -1,5 +1,5 @@
 import unittest
-from test.support import cpython_only, requires_limited_api, skip_on_s390x
+from test.support import cpython_only, requires_limited_api, skip_on_s390x, is_wasi, Py_DEBUG
 try:
     import _testcapi
 except ImportError:
@@ -932,6 +932,7 @@ def test_multiple_values(self):
 class TestRecursion(unittest.TestCase):
 
     @skip_on_s390x
+    @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack")
     def test_super_deep(self):
 
         def recurse(n):

Lib/test/test_compile.py

@@ -607,9 +607,9 @@ def test_compiler_recursion_limit(self):
         # Expected limit is C_RECURSION_LIMIT * 2
         # Duplicating the limit here is a little ugly.
         # Perhaps it should be exposed somewhere...
-        fail_depth = C_RECURSION_LIMIT * 2 + 1
+        fail_depth = C_RECURSION_LIMIT + 1
         crash_depth = C_RECURSION_LIMIT * 100
-        success_depth = int(C_RECURSION_LIMIT * 1.8)
+        success_depth = int(C_RECURSION_LIMIT * 0.9)
 
         def check_limit(prefix, repeated, mode="single"):
             expect_ok = prefix + repeated * success_depth

Lib/test/test_isinstance.py

@@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to):
     # Make sure that calling isinstance with a deeply nested tuple for its
     # argument will raise RecursionError eventually.
     tuple_arg = (compare_to,)
-    for cnt in range(support.EXCEEDS_RECURSION_LIMIT):
+    for cnt in range(support.C_RECURSION_LIMIT * 2):
         tuple_arg = (tuple_arg,)
         fxn(arg, tuple_arg)
 

Lib/test/test_plistlib.py

@@ -908,7 +908,7 @@ def test_cycles(self):
         self.assertIs(b['x'], b)
 
     def test_deep_nesting(self):
-        tests = [50, 100_000] if support.is_wasi else [50, 300, 100_000]
+        tests = [50, 100_000] if support.is_wasi else [50, 600, 100_000]
         for N in tests:
             chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]
             try:

Lib/test/test_sys_settrace.py

@@ -2965,16 +2965,18 @@ def test_trace_unpack_long_sequence(self):
         self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
 
     def test_trace_lots_of_globals(self):
+        count = min(1000, int(support.C_RECURSION_LIMIT * 0.8))
+
         code = """if 1:
             def f():
                 return (
                     {}
                 )
-        """.format("\n+\n".join(f"var{i}\n" for i in range(1000)))
-        ns = {f"var{i}": i for i in range(1000)}
+        """.format("\n+\n".join(f"var{i}\n" for i in range(count)))
+        ns = {f"var{i}": i for i in range(count)}
         exec(code, ns)
         counts = self.count_traces(ns["f"])
-        self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1})
+        self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1})
 
 
 class TestEdgeCases(unittest.TestCase):

Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst

@@ -0,0 +1,2 @@
+Change the C recursion limits to more closely reflect the underlying
+platform limits.

Parser/asdl_c.py

@@ -1393,15 +1393,14 @@ class PartingShots(StaticVisitor):
 
     int starting_recursion_depth;
     /* Be careful here to prevent overflow. */
-    int COMPILER_STACK_FRAME_SCALE = 2;
     PyThreadState *tstate = _PyThreadState_GET();
     if (!tstate) {
         return NULL;
     }
     struct validator vstate;
-    vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    vstate.recursion_limit = C_RECURSION_LIMIT;
     int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     vstate.recursion_depth = starting_recursion_depth;
 
     PyObject *result = ast2obj_mod(state, &vstate, t);