gh-103743: Add PyUnstable_Object_GC_NewWithExtraData (#103743)

Author: jbradaric

Doc/c-api/gcsupport.rst

@@ -65,6 +65,20 @@ rules:
    Analogous to :c:func:`PyObject_NewVar` but for container objects with the
    :const:`Py_TPFLAGS_HAVE_GC` flag set.
 
+.. c:function:: TYPE* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size)
+
+   Analogous to :c:func:`PyObject_GC_New` but for container objects that
+   have additional data at the end of the object not managed by Python.
+
+   .. warning::
+       The function is marked as unstable because the final mechanism
+       for reserving extra data after an instance is not yet decided.
+       Once `PEP-697<https://peps.python.org/pep-0697/>`_ is
+       implemented, the mechanism described there can be used to reserve
+       the extra data.
+
+   .. versionadded:: 3.12
+
 
 .. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize)
 

Include/objimpl.h

@@ -131,6 +131,10 @@ PyAPI_FUNC(PyVarObject *) PyObject_InitVar(PyVarObject *,
 PyAPI_FUNC(PyObject *) _PyObject_New(PyTypeObject *);
 PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t);
 
+#if !defined(Py_LIMITED_API)
+PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *, size_t);
+#endif
+
 #define PyObject_New(type, typeobj) ((type *)_PyObject_New(typeobj))
 
 // Alias to PyObject_New(). In Python 3.8, PyObject_NEW() called directly

Lib/test/test_capi/test_misc.py

@@ -1043,6 +1043,20 @@ class dictsub(dict): ...  # dict subclasses must work
         self.assertEqual(_testcapi.function_get_kw_defaults(some), None)
         self.assertEqual(some.__kwdefaults__, None)
 
+    def test_unstable_gc_new_with_extra_data(self):
+        class Data(_testcapi.ObjExtraData):
+            __slots__ = ('x', 'y')
+
+        d = Data()
+        d.x = 10
+        d.y = 20
+        d.extra = 30
+        self.assertEqual(d.x, 10)
+        self.assertEqual(d.y, 20)
+        self.assertEqual(d.extra, 30)
+        del d.extra
+        self.assertIsNone(d.extra)
+
 
 class TestPendingCalls(unittest.TestCase):
 

Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst

@@ -0,0 +1,2 @@
+Add `PyUnstable_Object_GC_NewWithExtraData` function that can be used to
+allocate additional memory after an object for data not managed by Python.

Modules/_testcapimodule.c

@@ -3343,7 +3343,7 @@ test_gc_visit_objects_basic(PyObject *Py_UNUSED(self),
     }
     state.target = obj;
     state.found = 0;
-    
+
     PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state);
     Py_DECREF(obj);
     if (!state.found) {
@@ -3380,6 +3380,94 @@ test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self),
     Py_RETURN_NONE;
 }
 
+typedef struct {
+    PyObject_HEAD
+} ObjExtraData;
+
+static PyObject *
+obj_extra_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    size_t extra_size = sizeof(PyObject *);
+    PyObject *obj = PyUnstable_Object_GC_NewWithExtraData(type, extra_size);
+    if (obj == NULL)
+        return PyErr_NoMemory();
+    memset(obj, '\0', type->tp_basicsize + extra_size);
+    PyObject_Init(obj, type);
+    PyObject_GC_Track(obj);
+    return obj;
+}
+
+static PyObject **
+obj_extra_data_get_extra_storage(PyObject *self)
+{
+    return (PyObject **)((char *)self + Py_TYPE(self)->tp_basicsize);
+}
+
+static PyObject *
+obj_extra_data_get(PyObject *self, void *Py_UNUSED(ignored))
+{
+    PyObject **extra_storage = obj_extra_data_get_extra_storage(self);
+    PyObject *value = *extra_storage;
+    if (!value)
+        Py_RETURN_NONE;
+    Py_INCREF(value);
+    return value;
+}
+
+static int
+obj_extra_data_set(PyObject *self, PyObject *newval, void *Py_UNUSED(ignored))
+{
+    PyObject **extra_storage = obj_extra_data_get_extra_storage(self);
+    Py_CLEAR(*extra_storage);
+    if (newval) {
+        Py_INCREF(newval);
+        *extra_storage = newval;
+    }
+    return 0;
+}
+
+static PyGetSetDef obj_extra_data_getset[] = {
+    {"extra", (getter)obj_extra_data_get, (setter)obj_extra_data_set, NULL},
+};
+
+static int
+obj_extra_data_traverse(PyObject *self, visitproc visit, void *arg)
+{
+    PyObject **extra_storage = obj_extra_data_get_extra_storage(self);
+    PyObject *value = *extra_storage;
+    Py_VISIT(value);
+    return 0;
+}
+
+static int
+obj_extra_data_clear(PyObject *self)
+{
+    PyObject **extra_storage = obj_extra_data_get_extra_storage(self);
+    Py_CLEAR(*extra_storage);
+    return 0;
+}
+
+static void
+obj_extra_data_dealloc(PyObject *self)
+{
+    PyObject_GC_UnTrack(self);
+    obj_extra_data_clear(self);
+    Py_TYPE(self)->tp_free(self);
+}
+
+static PyTypeObject ObjExtraData_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "obj_with_extra_data",
+    sizeof(ObjExtraData),
+    0,
+    .tp_getset = obj_extra_data_getset,
+    .tp_dealloc = obj_extra_data_dealloc,
+    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+    .tp_traverse = (traverseproc)obj_extra_data_traverse,
+    .tp_clear = (inquiry)obj_extra_data_clear,
+    .tp_new = obj_extra_data_new,
+    .tp_free = PyObject_GC_Del,
+};
 
 struct atexit_data {
     int called;
@@ -4103,6 +4191,11 @@ PyInit__testcapi(void)
     Py_INCREF(&MethStatic_Type);
     PyModule_AddObject(m, "MethStatic", (PyObject *)&MethStatic_Type);
 
+    if (PyType_Ready(&ObjExtraData_Type) < 0)
+        return NULL;
+    Py_INCREF(&ObjExtraData_Type);
+    PyModule_AddObject(m, "ObjExtraData", (PyObject *)&ObjExtraData_Type);
+
     PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX));
     PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN));
     PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX));

Modules/gcmodule.c

@@ -2357,6 +2357,18 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems)
     return op;
 }
 
+PyObject *
+PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size)
+{
+    size_t presize = _PyType_PreHeaderSize(tp);
+    PyObject *op = gc_alloc(_PyObject_SIZE(tp) + extra_size, presize);
+    if (op == NULL) {
+        return NULL;
+    }
+    _PyObject_Init(op, tp);
+    return op;
+}
+
 PyVarObject *
 _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems)
 {