Reland: Allow re-initialization and caching of foreign types (#47407)

Author: vchuravy

src/datatype.c

@@ -72,7 +72,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu
     jl_atomic_store_relaxed(&tn->cache, jl_emptysvec);
     jl_atomic_store_relaxed(&tn->linearcache, jl_emptysvec);
     tn->names = NULL;
-    tn->hash = bitmix(bitmix(module ? module->build_id : 0, name->hash), 0xa1ada1da);
+    tn->hash = bitmix(bitmix(module ? module->build_id.lo : 0, name->hash), 0xa1ada1da);
     tn->_reserved = 0;
     tn->abstract = abstract;
     tn->mutabl = mutabl;
@@ -826,6 +826,22 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name,
     return bt;
 }
 
+JL_DLLEXPORT int jl_reinit_foreign_type(jl_datatype_t *dt,
+                                        jl_markfunc_t markfunc,
+                                        jl_sweepfunc_t sweepfunc)
+{
+    if (!jl_is_foreign_type(dt))
+        return 0;
+    const jl_datatype_layout_t *layout = dt->layout;
+    jl_fielddescdyn_t * desc =
+      (jl_fielddescdyn_t *) ((char *)layout + sizeof(*layout));
+    assert(!desc->markfunc);
+    assert(!desc->sweepfunc);
+    desc->markfunc = markfunc;
+    desc->sweepfunc = sweepfunc;
+    return 1;
+}
+
 JL_DLLEXPORT int jl_is_foreign_type(jl_datatype_t *dt)
 {
     return jl_is_datatype(dt) && dt->layout && dt->layout->fielddesc_type == 3;

src/jl_exported_funcs.inc

@@ -343,6 +343,7 @@
     XX(jl_new_code_info_uninit) \
     XX(jl_new_datatype) \
     XX(jl_new_foreign_type) \
+    XX(jl_reinit_foreign_type) \
     XX(jl_new_method_instance_uninit) \
     XX(jl_new_method_table) \
     XX(jl_new_method_uninit) \

src/julia_gcext.h

@@ -49,6 +49,13 @@ JL_DLLEXPORT jl_datatype_t *jl_new_foreign_type(
         int haspointers,
         int large);
 
+
+#define HAVE_JL_REINIT_FOREIGN_TYPE 1
+JL_DLLEXPORT int jl_reinit_foreign_type(
+        jl_datatype_t *dt,
+        jl_markfunc_t markfunc,
+        jl_sweepfunc_t sweepfunc);
+
 JL_DLLEXPORT int jl_is_foreign_type(jl_datatype_t *dt);
 
 JL_DLLEXPORT size_t jl_gc_max_internal_obj_size(void);

test/gcext/.gitignore

@@ -1,2 +1,3 @@
 /gcext
 /gcext-debug
+/Foreign/deps

test/gcext/DependsOnForeign/Manifest.toml

@@ -0,0 +1,14 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.8.3"
+manifest_format = "2.0"
+project_hash = "e7199d961a5f4ebad68a3deaf5beaa7406a0afcb"
+
+[[deps.Foreign]]
+deps = ["Libdl"]
+path = "../Foreign"
+uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
+version = "0.1.0"
+
+[[deps.Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

test/gcext/DependsOnForeign/Project.toml

@@ -0,0 +1,6 @@
+name = "DependsOnForeign"
+uuid = "4b0716e0-dfb5-4e00-8b44-e2685a41517f"
+version = "0.1.0"
+
+[deps]
+Foreign = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"

test/gcext/DependsOnForeign/src/DependsOnForeign.jl

@@ -0,0 +1,14 @@
+module DependsOnForeign
+
+using Foreign
+
+f(obj::FObj) = Base.pointer_from_objref(obj)
+precompile(f, (FObj,))
+
+const FObjRef = Ref{FObj}()
+
+function __init__()
+    FObjRef[] = FObj()
+end
+
+end # module DependsOnForeign

test/gcext/Foreign/Manifest.toml

@@ -0,0 +1,8 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.9.0-DEV"
+manifest_format = "2.0"
+project_hash = "7b70172a2edbdc772ed789e79d4411d7528eae86"
+
+[[deps.Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

test/gcext/Foreign/Project.toml

@@ -0,0 +1,6 @@
+name = "Foreign"
+uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
+version = "0.1.0"
+
+[deps]
+Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

test/gcext/Foreign/deps/foreignlib.c

@@ -0,0 +1,56 @@
+// This file is a part of Julia. License is MIT: https://julialang.org/license
+
+#include "julia.h"
+#include "julia_gcext.h"
+
+// TODO make these atomics
+int nmarks = 0;
+int nsweeps = 0;
+
+uintptr_t mark(jl_ptls_t ptls, jl_value_t *p)
+{
+    nmarks += 1;
+    return 0;
+}
+
+void sweep(jl_value_t *p)
+{
+    nsweeps++;
+}
+
+JL_DLLEXPORT jl_datatype_t *declare_foreign(jl_sym_t* name, jl_module_t *module, jl_datatype_t *parent)
+{
+     return jl_new_foreign_type(name, module, parent, mark, sweep, 1, 0);
+}
+
+// #define GC_MAX_SZCLASS (2032 - sizeof(void *))
+
+JL_DLLEXPORT int reinit_foreign(jl_datatype_t *dt)
+{
+    int ret = jl_reinit_foreign_type(dt, mark, sweep);
+    nmarks = nsweeps = 0;
+    if (ret == 0)
+        return 0;
+    if (dt->layout->npointers != 1)
+        return -1;
+    if (dt->layout->size != 0)
+        return -2;
+    return ret;
+}
+
+JL_DLLEXPORT jl_value_t *allocate_foreign(jl_ptls_t ptls, size_t sz, jl_datatype_t *dt)
+{
+    jl_value_t* obj = jl_gc_alloc_typed(ptls, sz, dt);
+    jl_gc_schedule_foreign_sweepfunc(ptls, obj);
+    return obj;
+}
+
+JL_DLLEXPORT int nmark_counter()
+{
+    return nmarks;
+}
+
+JL_DLLEXPORT int nsweep_counter()
+{
+    return nsweeps;
+}