Skip to content

Commit be9df3f

Browse files
authored
Merge pull request #48881 from JuliaLang/vc/backports-1.9
[release-1.9] Backport #48751 and #48832
2 parents fcde569 + c1cb058 commit be9df3f

File tree

3 files changed

+98
-127
lines changed

3 files changed

+98
-127
lines changed

src/method.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -603,6 +603,8 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo)
603603
for (int i = 0; i < jl_array_len(func->code); ++i) {
604604
jl_value_t *stmt = jl_array_ptr_ref(func->code, i);
605605
if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == jl_new_opaque_closure_sym) {
606+
if (jl_options.incremental && jl_generating_output())
607+
jl_error("Impossible to correctly handle OpaqueClosure inside @generated returned during precompile process.");
606608
linfo->uninferred = jl_copy_ast((jl_value_t*)func);
607609
jl_gc_wb(linfo, linfo->uninferred);
608610
break;

src/staticdata.c

Lines changed: 52 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -608,10 +608,10 @@ static uintptr_t jl_fptr_id(void *fptr)
608608

609609
// `jl_queue_for_serialization` adds items to `serialization_order`
610610
#define jl_queue_for_serialization(s, v) jl_queue_for_serialization_((s), (jl_value_t*)(v), 1, 0)
611-
static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate);
611+
static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED;
612612

613613

614-
static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_t *m)
614+
static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_t *m) JL_GC_DISABLED
615615
{
616616
jl_queue_for_serialization(s, m->name);
617617
jl_queue_for_serialization(s, m->parent);
@@ -648,7 +648,7 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_
648648
// you want to handle uniquing of `Dict{String,Float64}` before you tackle `Vector{Dict{String,Float64}}`.
649649
// Uniquing is done in `serialization_order`, so the very first mention of such an object must
650650
// be the "source" rather than merely a cross-reference.
651-
static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate)
651+
static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED
652652
{
653653
jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v);
654654
jl_queue_for_serialization_(s, (jl_value_t*)t, 1, immediate);
@@ -672,23 +672,34 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_
672672
immediate = 0; // do not handle remaining fields immediately (just field types remains)
673673
}
674674
if (s->incremental && jl_is_method_instance(v)) {
675+
jl_method_instance_t *mi = (jl_method_instance_t*)v;
676+
jl_value_t *def = mi->def.value;
675677
if (needs_uniquing(v)) {
676678
// we only need 3 specific fields of this (the rest are not used)
677-
jl_method_instance_t *mi = (jl_method_instance_t*)v;
678679
jl_queue_for_serialization(s, mi->def.value);
679680
jl_queue_for_serialization(s, mi->specTypes);
680681
jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals);
681682
recursive = 0;
682683
goto done_fields;
683684
}
684-
else if (needs_recaching(v)) {
685+
else if (jl_is_method(def) && jl_object_in_image(def)) {
685686
// we only need 3 specific fields of this (the rest are restored afterward, if valid)
686-
jl_method_instance_t *mi = (jl_method_instance_t*)v;
687+
// in particular, cache is repopulated by jl_mi_cache_insert for all foreign function,
688+
// so must not be present here
687689
record_field_change((jl_value_t**)&mi->uninferred, NULL);
688690
record_field_change((jl_value_t**)&mi->backedges, NULL);
689691
record_field_change((jl_value_t**)&mi->callbacks, NULL);
690692
record_field_change((jl_value_t**)&mi->cache, NULL);
691693
}
694+
else {
695+
assert(!needs_recaching(v));
696+
}
697+
// n.b. opaque closures cannot be inspected and relied upon like a
698+
// normal method since they can get improperly introduced by generated
699+
// functions, so if they appeared at all, we will probably serialize
700+
// them wrong and segfault. The jl_code_for_staged function should
701+
// prevent this from happening, so we do not need to detect that user
702+
// error now.
692703
}
693704
if (jl_is_typename(v)) {
694705
jl_typename_t *tn = (jl_typename_t*)v;
@@ -700,6 +711,15 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_
700711
assert(!jl_object_in_image((jl_value_t*)tn->wrapper));
701712
}
702713
}
714+
if (s->incremental && jl_is_code_instance(v)) {
715+
jl_code_instance_t *ci = (jl_code_instance_t*)v;
716+
// make sure we don't serialize other reachable cache entries of foreign methods
717+
if (jl_object_in_image((jl_value_t*)ci->def->def.value)) {
718+
// TODO: if (ci in ci->defs->cache)
719+
record_field_change((jl_value_t**)&ci->next, NULL);
720+
}
721+
}
722+
703723

704724
if (immediate) // must be things that can be recursively handled, and valid as type parameters
705725
assert(jl_is_immutable(t) || jl_is_typevar(v) || jl_is_symbol(v) || jl_is_svec(v));
@@ -775,7 +795,7 @@ done_fields: ;
775795
*bp = (void*)((char*)HT_NOTFOUND + 1 + idx);
776796
}
777797

778-
static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate)
798+
static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED
779799
{
780800
if (!jl_needs_serialization(s, v))
781801
return;
@@ -818,7 +838,7 @@ static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, i
818838
// Do a pre-order traversal of the to-serialize worklist, in the identical order
819839
// to the calls to jl_queue_for_serialization would occur in a purely recursive
820840
// implementation, but without potentially running out of stack.
821-
static void jl_serialize_reachable(jl_serializer_state *s)
841+
static void jl_serialize_reachable(jl_serializer_state *s) JL_GC_DISABLED
822842
{
823843
size_t i, prevlen = 0;
824844
while (object_worklist.len) {
@@ -2877,10 +2897,11 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl
28772897
*method_roots_list = (jl_array_t*)jl_delayed_reloc(&s, offset_method_roots_list);
28782898
*ext_targets = (jl_array_t*)jl_delayed_reloc(&s, offset_ext_targets);
28792899
*edges = (jl_array_t*)jl_delayed_reloc(&s, offset_edges);
2900+
if (!*new_specializations)
2901+
*new_specializations = jl_alloc_vec_any(0);
28802902
}
28812903
s.s = NULL;
28822904

2883-
28842905
// step 3: apply relocations
28852906
assert(!ios_eof(f));
28862907
jl_read_symbols(&s);
@@ -3142,19 +3163,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl
31423163
jl_code_instance_t *ci = (jl_code_instance_t*)obj;
31433164
assert(s.incremental);
31443165
ci->min_world = world;
3145-
if (ci->max_world == 1) { // sentinel value: has edges to external callables
3146-
ptrhash_put(&new_code_instance_validate, ci, (void*)(~(uintptr_t)HT_NOTFOUND)); // "HT_FOUND"
3147-
}
3148-
else if (ci->max_world) {
3149-
// It's valid, but it may not be connected
3150-
if (!ci->def->cache)
3151-
ci->def->cache = ci;
3152-
}
3153-
else {
3154-
// Ensure this code instance is not connected
3155-
if (ci->def->cache == ci)
3156-
ci->def->cache = NULL;
3157-
}
3166+
if (ci->max_world != 0)
3167+
jl_array_ptr_1d_push(*new_specializations, (jl_value_t*)ci);
31583168
}
31593169
else if (jl_is_globalref(obj)) {
31603170
continue; // wait until all the module binding tables have been initialized
@@ -3298,7 +3308,7 @@ static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_
32983308
}
32993309

33003310
// TODO?: refactor to make it easier to create the "package inspector"
3301-
static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *image, jl_array_t *depmods, int complete)
3311+
static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *image, jl_array_t *depmods, int completeinfo)
33023312
{
33033313
uint64_t checksum = 0;
33043314
int64_t dataendpos = 0;
@@ -3330,7 +3340,6 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im
33303340
else {
33313341
ios_close(f);
33323342
ios_static_buffer(f, sysimg, len);
3333-
htable_new(&new_code_instance_validate, 0);
33343343
pkgcachesizes cachesizes;
33353344
jl_restore_system_image_from_stream_(f, image, depmods, checksum, (jl_array_t**)&restored, &init_order, &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges, &base, &ccallable_list, &cachesizes);
33363345
JL_SIGATOMIC_END();
@@ -3342,25 +3351,25 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im
33423351
jl_copy_roots(method_roots_list, jl_worklist_key((jl_array_t*)restored));
33433352
// Handle edges
33443353
jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)ext_targets, (jl_array_t*)new_specializations); // restore external backedges (needs to be last)
3345-
// check new CodeInstances and validate any that lack external backedges
3346-
validate_new_code_instances();
33473354
// reinit ccallables
33483355
jl_reinit_ccallable(&ccallable_list, base, NULL);
33493356
arraylist_free(&ccallable_list);
3350-
htable_free(&new_code_instance_validate);
3351-
if (complete) {
3352-
cachesizes_sv = jl_alloc_svec_uninit(7);
3353-
jl_svec_data(cachesizes_sv)[0] = jl_box_long(cachesizes.sysdata);
3354-
jl_svec_data(cachesizes_sv)[1] = jl_box_long(cachesizes.isbitsdata);
3355-
jl_svec_data(cachesizes_sv)[2] = jl_box_long(cachesizes.symboldata);
3356-
jl_svec_data(cachesizes_sv)[3] = jl_box_long(cachesizes.tagslist);
3357-
jl_svec_data(cachesizes_sv)[4] = jl_box_long(cachesizes.reloclist);
3358-
jl_svec_data(cachesizes_sv)[5] = jl_box_long(cachesizes.gvarlist);
3359-
jl_svec_data(cachesizes_sv)[6] = jl_box_long(cachesizes.fptrlist);
3357+
3358+
if (completeinfo) {
3359+
cachesizes_sv = jl_alloc_svec(7);
3360+
jl_svecset(cachesizes_sv, 0, jl_box_long(cachesizes.sysdata));
3361+
jl_svecset(cachesizes_sv, 1, jl_box_long(cachesizes.isbitsdata));
3362+
jl_svecset(cachesizes_sv, 2, jl_box_long(cachesizes.symboldata));
3363+
jl_svecset(cachesizes_sv, 3, jl_box_long(cachesizes.tagslist));
3364+
jl_svecset(cachesizes_sv, 4, jl_box_long(cachesizes.reloclist));
3365+
jl_svecset(cachesizes_sv, 5, jl_box_long(cachesizes.gvarlist));
3366+
jl_svecset(cachesizes_sv, 6, jl_box_long(cachesizes.fptrlist));
33603367
restored = (jl_value_t*)jl_svec(8, restored, init_order, extext_methods, new_specializations, method_roots_list,
33613368
ext_targets, edges, cachesizes_sv);
3362-
} else
3369+
}
3370+
else {
33633371
restored = (jl_value_t*)jl_svec(2, restored, init_order);
3372+
}
33643373
}
33653374
}
33663375

@@ -3373,24 +3382,24 @@ static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, uin
33733382
jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
33743383
}
33753384

3376-
JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int complete)
3385+
JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo)
33773386
{
33783387
ios_t f;
33793388
ios_static_buffer(&f, (char*)buf, sz);
3380-
jl_value_t *ret = jl_restore_package_image_from_stream(&f, image, depmods, complete);
3389+
jl_value_t *ret = jl_restore_package_image_from_stream(&f, image, depmods, completeinfo);
33813390
ios_close(&f);
33823391
return ret;
33833392
}
33843393

3385-
JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int complete)
3394+
JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int completeinfo)
33863395
{
33873396
ios_t f;
33883397
if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) {
33893398
return jl_get_exceptionf(jl_errorexception_type,
33903399
"Cache file \"%s\" not found.\n", fname);
33913400
}
33923401
jl_image_t pkgimage = {};
3393-
jl_value_t *ret = jl_restore_package_image_from_stream(&f, &pkgimage, depmods, complete);
3402+
jl_value_t *ret = jl_restore_package_image_from_stream(&f, &pkgimage, depmods, completeinfo);
33943403
ios_close(&f);
33953404
return ret;
33963405
}
@@ -3440,7 +3449,7 @@ JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len)
34403449
JL_SIGATOMIC_END();
34413450
}
34423451

3443-
JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods, int complete)
3452+
JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods, int completeinfo)
34443453
{
34453454
void *pkgimg_handle = jl_dlopen(fname, JL_RTLD_LAZY);
34463455
if (!pkgimg_handle) {
@@ -3492,7 +3501,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j
34923501
}
34933502
#endif
34943503

3495-
jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, complete);
3504+
jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, completeinfo);
34963505

34973506
return mod;
34983507
}

0 commit comments

Comments
 (0)