Skip to content

loading: provide code for reifying a load-path lookup result #37632

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

loading: provide code for reifying a load-path lookup result #37632

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
251 changes: 226 additions & 25 deletions base/loading.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,12 +234,14 @@ end
## package identification: determine unique identity of package to be loaded ##

# Used by Pkg but not used in loading itself
function find_package(arg)
pkg = identify_package(arg)
find_package(name::String) = find_package(PkgId(""), name) # `where` without a uuid will be ignored
function find_package(where::Union{Module,PkgId}, name::String)
pkg = identify_package(where, name)
pkg === nothing && return nothing
return locate_package(pkg)
end


## package identity: given a package name and a context, try to return its identity ##
identify_package(where::Module, name::String) = identify_package(PkgId(where), name)

Expand Down Expand Up @@ -267,6 +269,29 @@ function identify_package(name::String)::Union{Nothing,PkgId}
return nothing
end

# identify_package computes the list of packages that can be loaded by where
function identify_package_deps(where::PkgId)::Vector{PkgId}
where.uuid === nothing && return identify_package_deps()
deps = PkgId[]
for env in load_path()
deps = manifest_deps_list(env, where)
deps === nothing || return deps # found--return it
end
return PkgId[where]
end

# identify_package lists all packages that can be loaded from any toplevel context
# by looking through the Project.toml files and directories
function identify_package_deps()::Vector{PkgId}
deps = PkgId[]
for env in load_path()
add = project_deps_list(env)
add === nothing || append!(deps, add)
end
return deps
end


## package location: given a package identity, find file to load ##
function locate_package(pkg::PkgId)::Union{Nothing,String}
if pkg.uuid === nothing
Expand All @@ -281,7 +306,7 @@ function locate_package(pkg::PkgId)::Union{Nothing,String}
return implicit_manifest_uuid_path(env, pkg)
end
@assert found.uuid !== nothing
return locate_package(found) # restart search now that we know the uuid for pkg
return locate_package(found) # restart search now that we know the uuid for pkg (TODO: the existance of this line of code is probably a bug)
end
else
for env in load_path()
Expand Down Expand Up @@ -362,9 +387,18 @@ function project_deps_get(env::String, name::String)::Union{Nothing,PkgId}
return nothing
end

function project_deps_list(env::String)::Union{Nothing,Vector{PkgId}}
project_file = env_project_file(env)
if project_file isa String
return explicit_project_deps_list(project_file)
elseif project_file
return implicit_project_deps_list(env)
end
return nothing
end

function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId}
uuid = where.uuid
@assert uuid !== nothing
@assert where.uuid !== nothing
project_file = env_project_file(env)
if project_file isa String
# first check if `where` names the Project itself
Expand All @@ -375,14 +409,34 @@ function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothi
return PkgId(pkg_uuid, name)
end
# look for manifest file and `where` stanza
return explicit_manifest_deps_get(project_file, uuid, name)
return explicit_manifest_deps_get(project_file, where, name)
elseif project_file
# if env names a directory, search it
return implicit_manifest_deps_get(env, where, name)
end
return nothing
end

function manifest_deps_list(env::String, where::PkgId)::Union{Nothing,Vector{PkgId}}
@assert where.uuid !== nothing
project_file = env_project_file(env)
if project_file isa String
# first check if `where` names the Project itself
proj = project_file_name_uuid(project_file, where.name)
if proj == where
# if `where` matches the project, use [deps] section as manifest, and stop searching
return explicit_project_deps_list(project_file)
end
# look for manifest file and `where` stanza
return explicit_manifest_deps_list(project_file, where)
elseif project_file
# if env names a directory, search it
return implicit_manifest_deps_list(env, where)
end
return nothing
end


function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String}
project_file = env_project_file(env)
if project_file isa String
Expand All @@ -402,10 +456,9 @@ end

# find project file's top-level UUID entry (or nothing)
function project_file_name_uuid(project_file::String, name::String)::PkgId
uuid = dummy_uuid(project_file)
d = parsed_toml(project_file)
uuid′ = get(d, "uuid", nothing)::Union{String, Nothing}
uuid′ === nothing || (uuid = UUID(uuid′))
uuid = uuid′ === nothing ? dummy_uuid(project_file) : UUID(uuid′)
name = get(d, "name", name)::String
return PkgId(uuid, name)
end
Expand Down Expand Up @@ -433,6 +486,7 @@ end

# given a directory (implicit env from LOAD_PATH) and a name,
# check if it is an implicit package
# TODO: aren't we supposed to first check for the Project file first and see if it declares a path?
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the path entry in Project the project file seems to me like a bit of an experiment that turned out to be unused. Pkg doesn't know about it for example.

function entry_point_and_project_file_inside(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}}
path = normpath(joinpath(dir, "src", "$name.jl"))
isfile_casesensitive(path) || return nothing, nothing
Expand Down Expand Up @@ -472,10 +526,9 @@ end
# return `nothing` if `name` is not found
function explicit_project_deps_get(project_file::String, name::String)::Union{Nothing,UUID}
d = parsed_toml(project_file)
root_uuid = dummy_uuid(project_file)
if get(d, "name", nothing)::Union{String, Nothing} === name
uuid = get(d, "uuid", nothing)::Union{String, Nothing}
return uuid === nothing ? root_uuid : UUID(uuid)
return uuid === nothing ? dummy_uuid(project_file) : UUID(uuid)
end
deps = get(d, "deps", nothing)::Union{Dict{String, Any}, Nothing}
if deps !== nothing
Expand All @@ -485,29 +538,57 @@ function explicit_project_deps_get(project_file::String, name::String)::Union{No
return nothing
end

function explicit_project_deps_list(project_file::String)::Union{Nothing,Vector{PkgId}}
d = parsed_toml(project_file)
list = PkgId[]
name = get(d, "name", nothing)::Union{String, Nothing}
if name !== nothing
uuid = get(d, "uuid", nothing)::Union{String, Nothing}
uuid = uuid === nothing ? dummy_uuid(project_file) : UUID(uuid)
push!(list, PkgId(uuid, name))
end
deps = get(d, "deps", nothing)::Union{Dict{String, Any}, Nothing}
if deps !== nothing
for (name, uuid) in deps
push!(list, PkgId(UUID(uuid), name))
end
end
return list
end

# find `where` stanza and return the PkgId for `name`
# return `nothing` if it did not find `where` (indicating caller should continue searching)
function explicit_manifest_deps_get(project_file::String, where::UUID, name::String)::Union{Nothing,PkgId}
function explicit_manifest_deps_get(project_file::String, where::PkgId, name::String)::Union{Nothing,PkgId}
manifest_file = project_file_manifest_path(project_file)
manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH
d = parsed_toml(manifest_file)
found_where = false
found_name = false
for (dep_name, entries) in d
entries::Vector{Any}
entries = get(d, where.name, nothing)::Union{Vector{Any}, Nothing}
if entries !== nothing
for entry in entries
entry = entry::Dict{String, Any}
uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
uuid === nothing && continue
if UUID(uuid) === where
if UUID(uuid) === where.uuid
found_where = true
# deps is either a list of names (deps = ["DepA", "DepB"]) or
# a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."}
deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
deps === nothing && continue
if deps isa Vector{String}
found_name = name in deps
break
if found_name
# we have a unique name for each dep
name_deps = get(d, name, nothing)::Union{Nothing, Vector{Any}}
if name_deps === nothing || length(name_deps) != 1
error("expected a single entry for $(repr(name)) in $(repr(project_file))")
end
entry = first(name_deps::Vector{Any})::Dict{String, Any}
uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
uuid === nothing && return nothing
return PkgId(UUID(uuid), name)
end
break # TODO: it seems wrong that we use all of break, continue, and return nothing to handle the failure cases here
else
deps = deps::Dict{String, Any}
for (dep, uuid) in deps
Expand All @@ -521,18 +602,54 @@ function explicit_manifest_deps_get(project_file::String, where::UUID, name::Str
end
end
found_where || return nothing
found_name || return PkgId(name)
# Only reach here if deps was not a dict which mean we have a unique name for the dep
name_deps = get(d, name, nothing)::Union{Nothing, Vector{Any}}
if name_deps === nothing || length(name_deps) != 1
error("expected a single entry for $(repr(name)) in $(repr(project_file))")
return PkgId(name)
end

# find `where` stanzas
function explicit_manifest_deps_list(project_file::String, where::PkgId)::Union{Nothing,Vector{PkgId}}
manifest_file = project_file_manifest_path(project_file)
manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH
d = parsed_toml(manifest_file)
found_where = false
entries = get(d, where.name, nothing)::Union{Vector{Any}, Nothing}
if entries !== nothing
for entry in entries
entry::Dict{String, Any}
uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
uuid === nothing && continue
if UUID(uuid) === where.uuid
found_where = true
# deps is either a list of names (deps = ["DepA", "DepB"]) or
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could some of this code be shared with explicit_manifest_deps_get?

# a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."}
deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
deps === nothing && continue
list = [where]
if deps isa Vector{String}
for name in deps
# we have a unique name for each dep
name_deps = get(d, name, nothing)::Union{Nothing, Vector{Any}}
if name_deps === nothing || length(name_deps) != 1
error("expected a single entry for $(repr(name)) in $(repr(project_file))")
end
entry = first(name_deps::Vector{Any})::Dict{String, Any}
uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
uuid === nothing && continue
push!(list, PkgId(UUID(uuid), name))
end
else
for (dep, uuid) in deps::Dict{String, Any}
push!(list, PkgId(UUID(uuid::String), dep))
end
end
return list
end
end
end
entry = first(name_deps::Vector{Any})::Dict{String, Any}
uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
uuid === nothing && return nothing
return PkgId(UUID(uuid), name)
found_where || return nothing
return [where]
end


# find `uuid` stanza, return the corresponding path
function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{Nothing,String}
manifest_file = project_file_manifest_path(project_file)
Expand Down Expand Up @@ -587,6 +704,40 @@ function implicit_project_deps_get(dir::String, name::String)::Union{Nothing,Pkg
return proj
end

# look for entry points accessible to names in a top-level package (no environment)
# from an implicit environment (e.g. stdlib)
function implicit_project_deps_list(dir::String)::Vector{PkgId}
list = PkgId[]
function add_path(name, project_file::Nothing)
push!(list, PkgId(name))
nothing
end
function add_path(name, project_file)
id = project_file_name_uuid(project_file, name)
if id.name == name
push!(list, id)
end
nothing
end
for fname in readdir(dir)
fpath = joinpath(dir, fname)
s = stat(fpath)
isjl = endswith(fname, ".jl")
name = isjl ? fname[1:prevind(fname, end - 2)] : fname
if isjl && isfile(s)
add_path(name, nothing)
elseif isdir(s)
if isjl
path, project_file = entry_point_and_project_file_inside(fpath, name)
path === nothing || add_path(name, project_file)
end
path, project_file = entry_point_and_project_file_inside(fpath, fname)
path === nothing || add_path(fname, project_file)
end
end
return list
end

# look for an entry-point for `name`, check that UUID matches
# if there's a project file, look up `name` in its deps and return that
# otherwise return `nothing` to indicate the caller should keep searching
Expand All @@ -601,6 +752,18 @@ function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Un
return PkgId(pkg_uuid, name)
end

# look for entry-point for pkg names, if if there's a project file,
# otherwise return `nothing` to indicate the caller should keep searching
function implicit_manifest_deps_list(dir::String, where::PkgId)::Union{Nothing,Vector{PkgId}}
@assert where.uuid !== nothing
project_file = entry_point_and_project_file(dir, where.name)[2]
project_file === nothing && return nothing # a project file is mandatory for a package with a uuid
proj = project_file_name_uuid(project_file, where.name)
proj == where || return nothing # verify that this is the correct project file
# this is the correct project, so stop searching here
return explicit_project_deps_list(project_file)
end

# look for an entry-point for `pkg` and return its path if UUID matches
function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String}
path, project_file = entry_point_and_project_file(dir, pkg.name)
Expand Down Expand Up @@ -1792,6 +1955,44 @@ function stale_cachefile(modpath::String, cachefile::String)
end
end


# starting with a particular `pkg` as the root, list all packages that can possibly be loaded
function identify_all_deps(pkg::PkgId)
lists = Dict{PkgId,Vector{PkgId}}()
q = [pkg]
while !isempty(q)
pkg = pop!(q)
get!(lists, pkg) do
deps = identify_package_deps(pkg)
append!(q, deps)
deps
end
end
return lists
end

# map locate_package across a list of pkgs
function locate_all_packages(pkgs)
pths = Dict{PkgId,String}()
for pkg in pkgs
pth = locate_package(pkg)
pth === nothing || (pths[pkg] = pth)
end
return pths
end

# get a tree of the information for all loadable packages starting from a
# particular `pkg` as the root
function find_all_packages(pkg::PkgId)
deps = identify_all_deps(pkg)
return locate_all_packages(keys(deps)) => deps
end
find_all_packages(name::String) =
find_all_packages(PkgId(""), name) # `where` without a uuid will be ignored
find_all_packages(where::Union{Module,PkgId}, name::String) =
find_all_packages(something(identify_package(where, name)))


"""
@__FILE__ -> AbstractString

Expand Down