Merge in the main branch

Author: encukou

.github/workflows/build.yml

@@ -250,7 +250,7 @@ jobs:
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
@@ -316,7 +316,7 @@ jobs:
 
   test_hypothesis:
     name: "Hypothesis tests on Ubuntu"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
@@ -429,7 +429,7 @@ jobs:
 
   build_asan:
     name: 'Address sanitizer'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'

.github/workflows/jit.yml

@@ -75,14 +75,10 @@ jobs:
             architecture: aarch64
             runner: ubuntu-latest
             compiler: gcc
-            # These fail because of emulation, not because of the JIT:
-            exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
           - target: aarch64-unknown-linux-gnu/clang
             architecture: aarch64
             runner: ubuntu-latest
             compiler: clang
-            # These fail because of emulation, not because of the JIT:
-            exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
     env:
       CC: ${{ matrix.compiler }}
     steps:
@@ -97,7 +93,7 @@ jobs:
           choco upgrade llvm -y
           choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
-          ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No PGO or tests (yet):
       - name: Emulated Windows
@@ -115,7 +111,7 @@ jobs:
           SDKROOT="$(xcrun --show-sdk-path)" \
             ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
           make all --jobs 4
-          ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Native Linux
@@ -125,11 +121,12 @@ jobs:
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
           ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
           make all --jobs 4
-          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Emulated Linux
         if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
+        # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux.
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
@@ -146,4 +143,4 @@ jobs:
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
             ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
-          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3

.github/workflows/mypy.yml

@@ -8,6 +8,7 @@ on:
   pull_request:
     paths:
       - ".github/workflows/mypy.yml"
+      - "Lib/_pyrepl/**"
       - "Lib/test/libregrtest/**"
       - "Tools/build/generate_sbom.py"
       - "Tools/cases_generator/**"
@@ -35,8 +36,9 @@ jobs:
     strategy:
       matrix:
         target: [
+          "Lib/_pyrepl",
           "Lib/test/libregrtest",
-          "Tools/build/",
+          "Tools/build",
           "Tools/cases_generator",
           "Tools/clinic",
           "Tools/jit",

.github/workflows/reusable-macos.yml

@@ -22,6 +22,7 @@ jobs:
       HOMEBREW_NO_INSTALL_CLEANUP: 1
       HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
       PYTHONSTRICTEXTENSIONBUILD: 1
+      TERM: linux
     strategy:
       fail-fast: false
       matrix:

.github/workflows/reusable-ubuntu.yml

@@ -12,11 +12,12 @@ jobs:
   build_ubuntu_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       FORCE_COLOR: 1
       OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
+      TERM: linux
     steps:
     - uses: actions/checkout@v4
     - name: Register gcc problem matcher

.github/workflows/reusable-wasi.yml

@@ -9,7 +9,7 @@ jobs:
   build_wasi_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       WASMTIME_VERSION: 18.0.3
       WASI_SDK_VERSION: 21

.pre-commit-config.yaml

@@ -11,6 +11,14 @@ repos:
         args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
         files: ^Tools/clinic/|Lib/test/test_clinic.py
 
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.4.2
+    hooks:
+      - id: black
+        name: Run Black on Tools/jit/
+        files: ^Tools/jit/
+        language_version: python3.12
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:

Doc/c-api/frame.rst

@@ -120,12 +120,19 @@ See also :ref:`Reflection <reflection>`.
 
 .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame)
 
-   Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`).
+   Get the *frame*'s :attr:`~frame.f_locals` attribute.
+   If the frame refers to a function or comprehension, this returns
+   a write-through proxy object that allows modifying the locals.
+   In all other cases (classes, modules) it returns the :class:`dict`
+   representing the frame locals directly.
 
    Return a :term:`strong reference`.
 
    .. versionadded:: 3.11
 
+   .. versionchanged:: 3.13
+      Return a proxy object for functions and comprehensions.
+
 
 .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame)
 

Doc/c-api/index.rst

@@ -25,3 +25,4 @@ document the API functions in detail.
    memory.rst
    objimpl.rst
    apiabiversion.rst
+   monitoring.rst

Doc/c-api/init.rst

@@ -1904,6 +1904,58 @@ Python-level trace functions in previous versions.
 
 .. versionadded:: 3.12
 
+Reference tracing
+=================
+
+.. versionadded:: 3.13
+
+.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data)
+
+   The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`.
+   The first parameter is a Python object that has been just created (when **event**
+   is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event**
+   is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer
+   that was provided when :c:func:`PyRefTracer_SetTracer` was called.
+
+.. versionadded:: 3.13
+
+.. c:var:: int PyRefTracer_CREATE
+
+   The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python
+   object has been created.
+
+.. c:var:: int PyRefTracer_DESTROY
+
+   The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python
+   object has been destroyed.
+
+.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data)
+
+   Register a reference tracer function. The function will be called when a new
+   Python has been created or when an object is going to be destroyed. If
+   **data** is provided it must be an opaque pointer that will be provided when
+   the tracer function is called. Return ``0`` on success. Set an exception and
+   return ``-1`` on error.
+
+   Not that tracer functions **must not** create Python objects inside or
+   otherwise the call will be re-entrant. The tracer also **must not** clear
+   any existing exception or set an exception.  The GIL will be held every time
+   the tracer function is called.
+
+   The GIL must be held when calling this function.
+
+.. versionadded:: 3.13
+
+.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data)
+
+   Get the registered reference tracer function and the value of the opaque data
+   pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called.
+   If no tracer was registered this function will return NULL and will set the
+   **data** pointer to NULL.
+
+   The GIL must be held when calling this function.
+
+.. versionadded:: 3.13
 
 .. _advanced-debugging: