allow loading extensions when a trigger is loaded from below the parent's load path (#49701)

also allow loading extensions of the active project

Author: KristofferC

base/loading.jl

@@ -413,7 +413,9 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)
                     @goto done
                 end
             end
-            stopenv == env && @goto done
+            if !(loading_extension || precompiling_extension)
+                stopenv == env && @goto done
+            end
         end
     else
         for env in load_path()
@@ -428,7 +430,9 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)
                 path = entry_path(path, pkg.name)
                 @goto done
             end
-            stopenv == env && break
+            if !(loading_extension || precompiling_extension)
+                stopenv == env && break
+            end
         end
         # Allow loading of stdlibs if the name/uuid are given
         # e.g. if they have been explicitly added to the project/manifest
@@ -619,6 +623,24 @@ function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothi
             pkg_uuid = explicit_project_deps_get(project_file, name)
             return PkgId(pkg_uuid, name)
         end
+        d = parsed_toml(project_file)
+        exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing}
+        if exts !== nothing
+            # Check if `where` is an extension of the project
+            if where.name in keys(exts) && where.uuid == uuid5(proj.uuid, where.name)
+                # Extensions can load weak deps...
+                weakdeps = get(d, "weakdeps", nothing)::Union{Dict{String, Any}, Nothing}
+                if weakdeps !== nothing
+                    wuuid = get(weakdeps, name, nothing)::Union{String, Nothing}
+                    if wuuid !== nothing
+                        return PkgId(UUID(wuuid), name)
+                    end
+                end
+                # ... and they can load same deps as the project itself
+                mby_uuid = explicit_project_deps_get(project_file, name)
+                mby_uuid === nothing || return PkgId(mby_uuid, name)
+            end
+        end
         # look for manifest file and `where` stanza
         return explicit_manifest_deps_get(project_file, where, name)
     elseif project_file
@@ -636,6 +658,8 @@ function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missi
             # if `pkg` matches the project, return the project itself
             return project_file_path(project_file)
         end
+        mby_ext = project_file_ext_path(project_file, pkg.name)
+        mby_ext === nothing || return mby_ext
         # look for manifest file and `where` stanza
         return explicit_manifest_uuid_path(project_file, pkg)
     elseif project_file
@@ -645,6 +669,25 @@ function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missi
     return nothing
 end
 
+
+function find_ext_path(project_path::String, extname::String)
+    extfiledir = joinpath(project_path, "ext", extname, extname * ".jl")
+    isfile(extfiledir) && return extfiledir
+    return joinpath(project_path, "ext", extname * ".jl")
+end
+
+function project_file_ext_path(project_file::String, name::String)
+    d = parsed_toml(project_file)
+    p = project_file_path(project_file)
+    exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing}
+    if exts !== nothing
+        if name in keys(exts)
+            return find_ext_path(p, name)
+        end
+    end
+    return nothing
+end
+
 # find project file's top-level UUID entry (or nothing)
 function project_file_name_uuid(project_file::String, name::String)::PkgId
     d = parsed_toml(project_file)
@@ -876,9 +919,7 @@ function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{No
                     error("failed to find source of parent package: \"$name\"")
                 end
                 p = normpath(dirname(parent_path), "..")
-                extfiledir = joinpath(p, "ext", pkg.name, pkg.name * ".jl")
-                isfile(extfiledir) && return extfiledir
-                return joinpath(p, "ext", pkg.name * ".jl")
+                return find_ext_path(p, pkg.name)
             end
         end
     end
@@ -1160,6 +1201,18 @@ end
 function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missing}
     project_file = env_project_file(env)
     if project_file isa String
+        # Look in project for extensions to insert
+        proj_pkg = project_file_name_uuid(project_file, pkg.name)
+        if pkg == proj_pkg
+            d_proj = parsed_toml(project_file)
+            weakdeps = get(d_proj, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}}
+            extensions = get(d_proj, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
+            extensions === nothing && return
+            weakdeps === nothing && return
+            return _insert_extension_triggers(pkg, extensions, weakdeps)
+        end
+
+        # Now look in manifest
         manifest_file = project_file_manifest_path(project_file)
         manifest_file === nothing && return
         d = get_deps(parsed_toml(manifest_file))
@@ -1224,6 +1277,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, <:An
 end
 
 loading_extension::Bool = false
+precompiling_extension::Bool = false
 function run_extension_callbacks(extid::ExtensionId)
     assert_havelock(require_lock)
     succeeded = try
@@ -1251,30 +1305,8 @@ function run_extension_callbacks(pkgid::PkgId)
     extids === nothing && return
     for extid in extids
         if extid.ntriggers > 0
-            # It is possible that pkgid was loaded in an environment
-            # below the one of the parent. This will cause a load failure when the
-            # pkg ext tries to load the triggers. Therefore, check this first
-            # before loading the pkg ext.
-            pkgenv = identify_package_env(extid.id, pkgid.name)
-            ext_not_allowed_load = false
-            if pkgenv === nothing
-                ext_not_allowed_load = true
-            else
-                pkg, env = pkgenv
-                path = locate_package(pkg, env)
-                if path === nothing
-                    ext_not_allowed_load = true
-                end
-            end
-            if ext_not_allowed_load
-                @debug "Extension $(extid.id.name) of $(extid.parentid.name) will not be loaded \
-                        since $(pkgid.name) loaded in environment lower in load path"
-                # indicate extid is expected to fail
-                extid.ntriggers *= -1
-            else
-                # indicate pkgid is loaded
-                extid.ntriggers -= 1
-            end
+            # indicate pkgid is loaded
+            extid.ntriggers -= 1
         end
         if extid.ntriggers < 0
             # indicate pkgid is loaded
@@ -2148,6 +2180,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::
     # write data over stdin to avoid the (unlikely) case of exceeding max command line size
     write(io.in, """
         empty!(Base.EXT_DORMITORY) # If we have a custom sysimage with `EXT_DORMITORY` prepopulated
+        Base.precompiling_extension = $(loading_extension)
         Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)),
             $(repr(load_path)), $deps, $(repr(source_path(nothing))))
         """)

test/loading.jl

@@ -1056,8 +1056,6 @@ end
         envs = [joinpath(@__DIR__, "project", "Extensions", "EnvWithHasExtensionsv2"), joinpath(@__DIR__, "project", "Extensions", "EnvWithHasExtensions")]
         cmd = addenv(```$(Base.julia_cmd()) --startup-file=no -e '
         begin
-
-
             push!(empty!(DEPOT_PATH), '$(repr(depot_path))')
             using HasExtensions
             using ExtDep
@@ -1067,6 +1065,22 @@ end
         '
         ```, "JULIA_LOAD_PATH" => join(envs, sep))
         @test success(cmd)
+
+        test_ext_proj = """
+        begin
+            using HasExtensions
+            using ExtDep
+            Base.get_extension(HasExtensions, :Extension) isa Module || error("expected extension to load")
+            using ExtDep2
+            Base.get_extension(HasExtensions, :ExtensionFolder) isa Module || error("expected extension to load")
+        end
+        """
+        for compile in (`--compiled-modules=no`, ``)
+            cmd_proj_ext = `$(Base.julia_cmd()) $compile --startup-file=no -e $test_ext_proj`
+            proj = joinpath(@__DIR__, "project", "Extensions")
+            cmd_proj_ext = addenv(cmd_proj_ext, "JULIA_LOAD_PATH" => join([joinpath(proj, "HasExtensions.jl"), joinpath(proj, "EnvWithDeps")], sep))
+            run(cmd_proj_ext)
+        end
     finally
         try
             rm(depot_path, force=true, recursive=true)

test/project/Extensions/EnvWithDeps/Manifest.toml

@@ -0,0 +1,21 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.9.0-rc3"
+manifest_format = "2.0"
+project_hash = "ec25ff8df3a5e2212a173c3de2c7d716cc47cd36"
+
+[[deps.ExtDep]]
+deps = ["SomePackage"]
+path = "../ExtDep.jl"
+uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c"
+version = "0.1.0"
+
+[[deps.ExtDep2]]
+path = "../ExtDep2"
+uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
+version = "0.1.0"
+
+[[deps.SomePackage]]
+path = "../SomePackage"
+uuid = "678608ae-7bb3-42c7-98b1-82102067a3d8"
+version = "0.1.0"

test/project/Extensions/EnvWithDeps/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
+ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
+SomePackage = "678608ae-7bb3-42c7-98b1-82102067a3d8"