From 61d9a4c9a86c331d899c750d5a99dc8c757eeda0 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 22 Apr 2025 02:24:24 -0500 Subject: [PATCH 1/3] Add binding invalidations to log Currently we log the invalidated backedges of a binding invalidation, but the actual trigger of the invalidation is not logged. This is needed to allow SnoopCompile to attribute a cause to those invalidations. --- base/invalidation.jl | 6 +++++- src/gf.c | 13 +++++++++++++ test/precompile.jl | 26 ++++++++++++++++++++++++++ test/worlds.jl | 24 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/base/invalidation.jl b/base/invalidation.jl index 34f260f7379fd..47b4c6a83ba8d 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -67,19 +67,21 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid binding = convert(Core.Binding, gr) if isdefined(method, :source) src = _uncompressed_ir(method) - old_stmts = src.code invalidate_all = should_invalidate_code_for_globalref(gr, src) end for mi in specializations(method) isdefined(mi, :cache) || continue ci = mi.cache + invalidated = false while true if ci.max_world > new_max_world && (invalidate_all || scan_edge_list(ci, binding)) ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), ci, new_max_world) + invalidated = true end isdefined(ci, :next) || break ci = ci.next end + invalidated && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), mi) end end @@ -130,6 +132,8 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core end end end + # This assumes that we haven't gotten here unless something needed to be invalidated + ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), invalidated_bpart) end if need_to_invalidate_code || need_to_invalidate_export diff --git a/src/gf.c b/src/gf.c index 4bfe910ac6cae..19f6452eafc5c 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1890,6 +1890,19 @@ JL_DLLEXPORT void jl_invalidate_code_instance(jl_code_instance_t *replaced, size invalidate_code_instance(replaced, max_world, 1); } +JL_DLLEXPORT void jl_maybe_log_binding_invalidation(jl_value_t *replaced) +{ + if (_jl_debug_method_invalidation) { + if (replaced) { + jl_array_ptr_1d_push(_jl_debug_method_invalidation, replaced); + } + jl_value_t *loctag = jl_cstr_to_string("jl_maybe_log_binding_invalidation"); + JL_GC_PUSH1(&loctag); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + JL_GC_POP(); + } +} + static void _invalidate_backedges(jl_method_instance_t *replaced_mi, jl_code_instance_t *replaced_ci, size_t max_world, int depth) { uint8_t recursion_flags = 0; jl_array_t *backedges = jl_mi_get_backedges_mutate(replaced_mi, &recursion_flags); diff --git a/test/precompile.jl b/test/precompile.jl index 2555086214c77..e95df0d50ae60 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -967,6 +967,12 @@ precompile_test_harness("code caching") do dir const gib = makewib(1) fib() = gib.ib.x + struct LogBindingInvalidation + x::Int + end + const glbi = LogBindingInvalidation(1) + flbi() = @__MODULE__().glbi.x + # force precompilation build_stale(37) stale('c') @@ -991,10 +997,13 @@ precompile_test_harness("code caching") do dir useA() = $StaleA.stale("hello") useA2() = useA() + useflbi() = $StaleA.flbi() + # force precompilation begin Base.Experimental.@force_compile useA2() + useflbi() end precompile($StaleA.fib, ()) @@ -1035,6 +1044,13 @@ precompile_test_harness("code caching") do dir end const gib = makewib(2.0) end) + # TODO: test a "method_globalref" invalidation also + Base.eval(MA, quote + struct LogBindingInvalidation # binding invalidations can't be done during precompilation + x::Float64 + end + const glbi = LogBindingInvalidation(2.0) + end) @eval using $StaleC invalidations = Base.StaticData.debug_method_invalidation(true) @eval using $StaleB @@ -1096,6 +1112,16 @@ precompile_test_harness("code caching") do dir @test !hasvalid(mi, world) @test any(x -> x isa Core.CodeInstance && x.def === mi, invalidations) + idxb = findfirst(x -> x isa Core.Binding, invalidations) + @test invalidations[idxb+1] == "insert_backedges_callee" + idxv = findnext(==("verify_methods"), invalidations, idxb) + if invalidations[idxv-1].def.def.name === :getproperty + idxv = findnext(==("verify_methods"), invalidations, idxv+1) + end + @test invalidations[idxv-1].def.def.name === :flbi + idxv = findnext(==("verify_methods"), invalidations, idxv+1) + @test invalidations[idxv-1].def.def.name === :useflbi + m = only(methods(MB.map_nbits)) @test !hasvalid(m.specializations::Core.MethodInstance, world+1) # insert_backedges invalidations also trigger their backedges end diff --git a/test/worlds.jl b/test/worlds.jl index 4b0ef208f9c7e..9aac6121b6117 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -436,6 +436,30 @@ idxi = findfirst(==(m58080i), logmeths) @test logmeths[end-1] == m58080s @test logmeths[end] == "jl_method_table_insert" +# logging binding invalidations +struct LogBindingInvalidation + x::Int +end +const glbi = LogBindingInvalidation(1) +oLBI, oglbi = LogBindingInvalidation, glbi +flbi() = @__MODULE__().glbi.x +flbi() +milbi = only(Base.specializations(only(methods(flbi)))) +logmeths = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) +struct LogBindingInvalidation + x::Float64 +end +const glbi = LogBindingInvalidation(2.0) +@test flbi() === 2.0 +ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) +@test milbi.cache.next.def ∈ logmeths +T = logmeths[1].restriction +@test T === oLBI +@test logmeths[2] == "jl_maybe_log_binding_invalidation" +T = logmeths[end-1].restriction +@test T === oglbi +@test logmeths[end] == "jl_maybe_log_binding_invalidation" + # issue #50091 -- missing invoke edge affecting nospecialized dispatch module ExceptionUnwrapping @nospecialize From 29e57db99beb53a7d6ff28882349ffdf41e08acb Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 25 Apr 2025 08:42:54 -0500 Subject: [PATCH 2/3] Add checks for code invalidation --- base/invalidation.jl | 19 +++++++++++++------ test/worlds.jl | 16 ++++++++++------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/base/invalidation.jl b/base/invalidation.jl index 47b4c6a83ba8d..e0df34ec544a9 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -69,6 +69,7 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid src = _uncompressed_ir(method) invalidate_all = should_invalidate_code_for_globalref(gr, src) end + invalidated_any = false for mi in specializations(method) isdefined(mi, :cache) || continue ci = mi.cache @@ -82,7 +83,9 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid ci = ci.next end invalidated && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), mi) + invalidated_any |= invalidated end + return invalidated_any end export_affecting_partition_flags(bpart::Core.BindingPartition) = @@ -106,18 +109,20 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core need_to_invalidate_export = export_affecting_partition_flags(invalidated_bpart) !== export_affecting_partition_flags(new_bpart) + invalidated_any = false if need_to_invalidate_code if (b.flags & BINDING_FLAG_ANY_IMPLICIT_EDGES) != 0 nmethods = ccall(:jl_module_scanned_methods_length, Csize_t, (Any,), gr.mod) for i = 1:nmethods method = ccall(:jl_module_scanned_methods_getindex, Any, (Any, Csize_t), gr.mod, i)::Method - invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) + invalidated_any |= invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) end end if isdefined(b, :backedges) for edge in b.backedges if isa(edge, CodeInstance) ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world) + invalidated_any = true elseif isa(edge, Core.Binding) isdefined(edge, :partitions) || continue latest_bpart = edge.partitions @@ -126,14 +131,13 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core if is_some_binding_imported(binding_kind(latest_bpart)) partition_restriction(latest_bpart) === b || continue end - invalidate_code_for_globalref!(edge, latest_bpart, latest_bpart, new_max_world) + invalidated_any |= invalidate_code_for_globalref!(edge, latest_bpart, latest_bpart, new_max_world) else - invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) + invalidated_any |= invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) end end end - # This assumes that we haven't gotten here unless something needed to be invalidated - ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), invalidated_bpart) + invalidated_any && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), invalidated_bpart) end if need_to_invalidate_code || need_to_invalidate_export @@ -152,11 +156,14 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world) : latest_bpart if need_to_invalidate_code || new_bpart !== latest_bpart - invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, new_bpart, new_max_world) + invalidated = invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, new_bpart, new_max_world) + invalidated && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), latest_bpart) + invalidated_any |= invalidated end end end end + return invalidated_any end invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) = invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world) diff --git a/test/worlds.jl b/test/worlds.jl index 9aac6121b6117..5c9510e67d2a1 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -440,22 +440,26 @@ idxi = findfirst(==(m58080i), logmeths) struct LogBindingInvalidation x::Int end -const glbi = LogBindingInvalidation(1) +makelbi(x) = LogBindingInvalidation(x) +const glbi = makelbi(1) oLBI, oglbi = LogBindingInvalidation, glbi flbi() = @__MODULE__().glbi.x flbi() -milbi = only(Base.specializations(only(methods(flbi)))) +milbi1 = only(Base.specializations(only(methods(makelbi)))) +milbi2 = only(Base.specializations(only(methods(flbi)))) logmeths = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) struct LogBindingInvalidation x::Float64 end -const glbi = LogBindingInvalidation(2.0) +const glbi = makelbi(2.0) @test flbi() === 2.0 ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) -@test milbi.cache.next.def ∈ logmeths -T = logmeths[1].restriction +@test milbi1.cache.def ∈ logmeths +@test milbi2.cache.next.def ∈ logmeths +i = findfirst(x -> isa(x, Core.BindingPartition), logmeths) +T = logmeths[i].restriction @test T === oLBI -@test logmeths[2] == "jl_maybe_log_binding_invalidation" +@test logmeths[i+1] == "jl_maybe_log_binding_invalidation" T = logmeths[end-1].restriction @test T === oglbi @test logmeths[end] == "jl_maybe_log_binding_invalidation" From 48b1f7f15621d79970b76b17bb458a8cdd5f4aa9 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 6 May 2025 16:58:28 -0500 Subject: [PATCH 3/3] disentangle sequence in log --- base/invalidation.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/base/invalidation.jl b/base/invalidation.jl index e0df34ec544a9..8057c16bd9fa9 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -110,6 +110,7 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core export_affecting_partition_flags(new_bpart) invalidated_any = false + queued_bindings = Tuple{Core.Binding, Core.BindingPartition, Core.BindingPartition}[] # defer handling these to keep the logging coherent if need_to_invalidate_code if (b.flags & BINDING_FLAG_ANY_IMPLICIT_EDGES) != 0 nmethods = ccall(:jl_module_scanned_methods_length, Csize_t, (Any,), gr.mod) @@ -131,13 +132,12 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core if is_some_binding_imported(binding_kind(latest_bpart)) partition_restriction(latest_bpart) === b || continue end - invalidated_any |= invalidate_code_for_globalref!(edge, latest_bpart, latest_bpart, new_max_world) + push!(queued_bindings, (edge, latest_bpart, latest_bpart)) else invalidated_any |= invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) end end end - invalidated_any && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), invalidated_bpart) end if need_to_invalidate_code || need_to_invalidate_export @@ -156,13 +156,15 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world) : latest_bpart if need_to_invalidate_code || new_bpart !== latest_bpart - invalidated = invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, new_bpart, new_max_world) - invalidated && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), latest_bpart) - invalidated_any |= invalidated + push!(queued_bindings, (convert(Core.Binding, user_binding), latest_bpart, new_bpart)) end end end end + invalidated_any && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), invalidated_bpart) + for (edge, invalidated_bpart, new_bpart) in queued_bindings + invalidated_any |= invalidate_code_for_globalref!(edge, invalidated_bpart, new_bpart, new_max_world) + end return invalidated_any end invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) =