diff --git a/.github/workflows/MyLib.yml b/.github/workflows/MyLib.yml new file mode 100644 index 00000000..a4ac3a9c --- /dev/null +++ b/.github/workflows/MyLib.yml @@ -0,0 +1,31 @@ +name: Build MyLib +on: + pull_request: + branches: + - 'master' + - 'release-*' + push: + branches: + - 'master' + - 'release-*' + tags: '*' +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: '1' + - uses: julia-actions/julia-buildpkg@v1 + with: + project: 'examples/MyLib' + - uses: julia-actions/julia-buildpkg@v1 + with: + project: 'examples/MyLib/build' + - run: | + cd examples/MyLib + make + - run: ./examples/MyLib/my_application.out + env: + LD_LIBRARY_PATH: 'examples/MyLib/MyLibCompiled/lib' diff --git a/.github/workflows/test.yml b/.github/workflows/ci.yml similarity index 70% rename from .github/workflows/test.yml rename to .github/workflows/ci.yml index e8db25a5..d755a43c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Run tests +name: CI on: pull_request: branches: @@ -13,6 +13,11 @@ concurrency: group: ${{ github.head_ref || github.ref_name || github.run_id }} cancel-in-progress: true jobs: + ci_started: + timeout-minutes: 3 + runs-on: ubuntu-latest + steps: + - run: exit 0 test: timeout-minutes: 90 runs-on: ${{ matrix.os }} @@ -57,3 +62,17 @@ jobs: - uses: codecov/codecov-action@v3 with: file: lcov.info + docs: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@latest + with: + version: '1' + - name: Build and deploy docs + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key + run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate(); include("docs/make.jl")' + diff --git a/.github/workflows/documenter-workflow.yml b/.github/workflows/documenter-workflow.yml deleted file mode 100644 index 7115c7be..00000000 --- a/.github/workflows/documenter-workflow.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Documentation -on: - push: - branches: - - master - tags: '*' - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@latest - with: - version: '1' - - name: Build and deploy - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token - DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key - run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate(); include("docs/make.jl")' diff --git a/.gitignore b/.gitignore index c943eee8..5f6fd0e2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,8 @@ .DS_Store /Manifest.toml /examples/MyApp/MyApp -/examples/*Compiled +/examples/MyLib/my_application.out +/examples/MyLib/MyLibCompiled /dev/ *.so *.dll diff --git a/Project.toml b/Project.toml index 51b383a6..96e1bc13 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "PackageCompiler" uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" -version = "2.1.4" +version = "2.1.7" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -19,6 +19,7 @@ julia = "1.6" [extras] Example = "7876af07-990d-54b4-ab0e-23690620f79a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" [targets] -test = ["Test", "Example"] +test = ["Test", "Example", "TOML"] diff --git a/README.md b/README.md index 518b0881..2caf96be 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # PackageCompiler -[![Build Status](https://github.com/JuliaLang/PackageCompiler.jl/actions/workflows/test.yml/badge.svg)](https://github.com/JuliaLang/PackageCompiler.jl/actions/workflows/test.yml) +[![Continuous integration](https://github.com/JuliaLang/PackageCompiler.jl/actions/workflows/ci.yml/badge.svg)](https://github.com/JuliaLang/PackageCompiler.jl/actions/workflows/ci.yml) [![Codecov](https://codecov.io/gh/JuliaLang/PackageCompiler.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/JuliaLang/PackageCompiler.jl) [![][docs-stable-img]][docs-stable-url] diff --git a/docs/src/devdocs/relocatable_part_3.md b/docs/src/devdocs/relocatable_part_3.md index 54b1cccb..25068705 100644 --- a/docs/src/devdocs/relocatable_part_3.md +++ b/docs/src/devdocs/relocatable_part_3.md @@ -2,7 +2,7 @@ !!! info This section is for people who want to understand PackageCompiler.jl under - the hood. It is not required reading to use the pacakge. + the hood. It is not required reading to use the package. In the previous tutorials, we created a custom sysimage and a binary (app) that did some simple CSV parsing with an (depending on the exact demands) acceptable diff --git a/docs/src/examples/plots.md b/docs/src/examples/plots.md index 8d24cd8b..c88b4101 100644 --- a/docs/src/examples/plots.md +++ b/docs/src/examples/plots.md @@ -49,7 +49,11 @@ julia> @time display(p); 0.331869 seconds (278.38 k allocations: 7.900 MiB) ``` -which is a sizeable speedup. +which is a sizeable speedup. + +When using the sysimage, you may need to call `Plots.__init__()` after loading +Plots for the plots to display; otherwise, you might need to use `gui` or +pass argument `show = true` to `plot`. Note that since we have more stuff in our sysimage, Julia is slightly slower to start (0.35 seconds on this machine): diff --git a/examples/MyApp/Project.toml b/examples/MyApp/Project.toml index cfe103b8..00bd9a7a 100644 --- a/examples/MyApp/Project.toml +++ b/examples/MyApp/Project.toml @@ -10,5 +10,5 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Example = "7876af07-990d-54b4-ab0e-23690620f79a" HelloWorldC_jll = "dca1746e-5efc-54fc-8249-22745bc95a49" LLVMExtra_jll = "dad2f222-ce93-54a1-a47d-0025e8a3acab" -MKL_jll = "856f044c-d86e-5d09-b602-aeab76dc8ba7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +micromamba_jll = "f8abcde7-e9b7-5caa-b8af-a437887ae8e4" diff --git a/examples/MyApp/src/MyApp.jl b/examples/MyApp/src/MyApp.jl index 04c317f6..61e9162c 100644 --- a/examples/MyApp/src/MyApp.jl +++ b/examples/MyApp/src/MyApp.jl @@ -9,7 +9,7 @@ if VERSION >= v"1.7.0" using LLVMExtra_jll end -using MKL_jll +using micromamba_jll const myrand = rand() @@ -93,10 +93,10 @@ function real_main() end end - if isfile(MKL_jll.libmkl_core_path) - println("MKL_jll path: ok!") + if isfile(micromamba_jll.micromamba_path) + println("micromamba_jll path: ok!") else - println("MKL_jll path: fail!") + println("micromamba_jll path: fail!") end return end diff --git a/examples/MyLib/Makefile b/examples/MyLib/Makefile index 6d9f79b5..4606e9a1 100644 --- a/examples/MyLib/Makefile +++ b/examples/MyLib/Makefile @@ -1,18 +1,28 @@ # Makefile +.DEFAULT_GOAL := all + JULIA ?= julia DLEXT := $(shell $(JULIA) --startup-file=no -e 'using Libdl; print(Libdl.dlext)') -TARGET="../MyLibCompiled" +TARGET="MyLibCompiled" MYLIB_INCLUDES = $(TARGET)/include/julia_init.h $(TARGET)/include/mylib.h MYLIB_PATH := $(TARGET)/lib/libmylib.$(DLEXT) -$(MYLIB_PATH) $(MYLIB_INCLUDES): build/build.jl src/MyLib.jl +build-library: build/build.jl src/MyLib.jl $(JULIA) --startup-file=no --project=. -e 'using Pkg; Pkg.instantiate()' $(JULIA) --startup-file=no --project=build -e 'using Pkg; Pkg.instantiate(); include("build/build.jl")' -.PHONY: clean +INCLUDE_DIR = $(TARGET)/include + +build-executable: + gcc my_application.c -o my_application.out -I$(INCLUDE_DIR) -L$(TARGET)/lib -ljulia -lmylib + +all: build-library build-executable + clean: $(RM) *~ *.o *.$(DLEXT) $(RM) -Rf $(TARGET) + +.PHONY: build-library build-executable clean all diff --git a/examples/MyLib/build/Manifest.toml b/examples/MyLib/build/Manifest.toml index f3ff0560..9752ea48 100644 --- a/examples/MyLib/build/Manifest.toml +++ b/examples/MyLib/build/Manifest.toml @@ -2,6 +2,7 @@ [[ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" [[Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -14,20 +15,30 @@ deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +[[LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + [[LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" [[LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" [[LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] @@ -36,6 +47,7 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" [[Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -50,22 +62,26 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.0+0" [[MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.2.1" [[NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" [[PackageCompiler]] -deps = ["Libdl", "Pkg", "UUIDs"] +deps = ["Artifacts", "LazyArtifacts", "Libdl", "Pkg", "Printf", "RelocatableFolders", "TOML", "UUIDs"] path = "../../.." uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" -version = "1.4.1" +version = "2.1.0" [[Pkg]] deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.8.0" [[Printf]] deps = ["Unicode"] @@ -76,11 +92,24 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[Random]] -deps = ["Serialization"] +deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +[[RelocatableFolders]] +deps = ["SHA", "Scratch"] +git-tree-sha1 = "90bc7a7c96410424509e4263e277e43250c05691" +uuid = "05181044-ff0b-4ac5-8273-598c1e38db00" +version = "1.0.0" + [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[Scratch]] +deps = ["Dates"] +git-tree-sha1 = "f94f779c94e58bf9ea243e77a37e16d9de9126bd" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.1.1" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -91,10 +120,12 @@ uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.0" [[Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.1" [[UUIDs]] deps = ["Random", "SHA"] @@ -106,11 +137,14 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.12+3" [[nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" [[p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/examples/MyLib/build/build.jl b/examples/MyLib/build/build.jl index acadb58b..5614f216 100644 --- a/examples/MyLib/build/build.jl +++ b/examples/MyLib/build/build.jl @@ -1,6 +1,6 @@ using PackageCompiler -target_dir = get(ENV, "OUTDIR", "$(@__DIR__)/../../MyLibCompiled") +target_dir = get(ENV, "OUTDIR", "$(@__DIR__)/../MyLibCompiled") target_dir = replace(target_dir, "\\"=>"/") # Change Windows paths to use "/" println("Creating library in $target_dir") @@ -10,5 +10,6 @@ PackageCompiler.create_library(".", target_dir; precompile_statements_file=["$(@__DIR__)/additional_precompile.jl"], incremental=false, filter_stdlibs=true, + force=true, # Overwrite target_dir. header_files = ["$(@__DIR__)/mylib.h"], ) diff --git a/examples/MyLib/my_application.c b/examples/MyLib/my_application.c new file mode 100644 index 00000000..39c06d00 --- /dev/null +++ b/examples/MyLib/my_application.c @@ -0,0 +1,16 @@ +#include + +#include "julia_init.h" +#include "mylib.h" + +int main(int argc, char *argv[]) +{ + init_julia(argc, argv); + + int incremented = increment32(3); + printf("Incremented value: %i", incremented); + + shutdown_julia(0); + return 0; +} + diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index d2d50e1b..d9fd4c80 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -58,6 +58,9 @@ end ############# function create_pkg_context(project) + if isfile(project) + error("`project` should be a path to a directory containing a Project/Manifest, not a file") + end project_toml_path = Pkg.Types.projectfile_path(project; strict=true) if project_toml_path === nothing error("could not find project at $(repr(project))") @@ -211,7 +214,7 @@ function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* stdlibs, ",\n"), "]")) end -function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String) +function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String, sysimage_build_args::Cmd) tmp = mktempdir() sysimg_source_path = Base.find_source_file("sysimg.jl") base_dir = dirname(sysimg_source_path) @@ -219,20 +222,21 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String) tmp_sys_ji = joinpath(tmp, "sys.ji") compiler_source_path = joinpath(base_dir, "compiler", "compiler.jl") + # we can't strip the IR from the base sysimg, so we filter out this flag + # also presumably `--compile=all` and maybe a few others we missed here... + sysimage_build_args_strs = map(p -> "$(p...)", values(sysimage_build_args)) + filter!(p -> !contains(p, "--compile") && p ∉ ("--strip-ir",), sysimage_build_args_strs) + sysimage_build_args = Cmd(sysimage_build_args_strs) + spinner = TerminalSpinners.Spinner(msg = "PackageCompiler: compiling base system image (incremental=false)") TerminalSpinners.@spin spinner begin cd(base_dir) do # Create corecompiler.ji - cmd = if isdefined(Base, :Linking) # pkgimages feature flag - cmd = `$(get_julia_cmd()) --output-ji $tmp_corecompiler_ji -g0 -O0 $compiler_source_path` - @debug "running $cmd" JULIA_CPU_TARGET = cpu_target - addenv(cmd, "JULIA_CPU_TARGET" => cpu_target) - else - cmd = `$(get_julia_cmd()) --cpu-target $cpu_target --output-ji $tmp_corecompiler_ji - -g0 -O0 $compiler_source_path` - @debug "running $cmd" - cmd - end + cmd = `$(get_julia_cmd()) --cpu-target $cpu_target + --output-ji $tmp_corecompiler_ji $sysimage_build_args + $compiler_source_path` + @debug "running $cmd" + read(cmd) # Use that to create sys.ji @@ -241,18 +245,12 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String) new_sysimage_source_path = joinpath(tmp, "sysimage_packagecompiler_$(uuid1()).jl") write(new_sysimage_source_path, new_sysimage_content) try - cmd = if isdefined(Base, :Linking) # pkgimages feature flag - cmd = `$(get_julia_cmd()) --sysimage=$tmp_corecompiler_ji - -g1 -O0 --output-ji=$tmp_sys_ji $new_sysimage_source_path` - @debug "running $cmd" JULIA_CPU_TARGET = cpu_target - addenv(cmd, "JULIA_CPU_TARGET" => cpu_target) - else - cmd = `$(get_julia_cmd()) --cpu-target $cpu_target - --sysimage=$tmp_corecompiler_ji - -g1 -O0 --output-ji=$tmp_sys_ji $new_sysimage_source_path` - @debug "running $cmd" - cmd - end + cmd = `$(get_julia_cmd()) --cpu-target $cpu_target + --sysimage=$tmp_corecompiler_ji + $sysimage_build_args --output-ji=$tmp_sys_ji + $new_sysimage_source_path` + @debug "running $cmd" + read(cmd) finally rm(new_sysimage_source_path; force=true) @@ -301,6 +299,22 @@ function create_sysimg_object_file(object_file::String, sysimage_build_args::Cmd, extra_precompiles::String, incremental::Bool) + julia_code_buffer = IOBuffer() + # include all packages into the sysimg + print(julia_code_buffer, """ + Base.reinit_stdio() + @eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String + @eval Sys STDLIB = $(repr(abspath(Sys.BINDIR, "../share/julia/stdlib", string('v', VERSION.major, '.', VERSION.minor)))) + copy!(LOAD_PATH, [$(repr(project))]) # Only allow loading packages from current project + Base.init_depot_path() + """) + + for pkg in packages_sysimg + print(julia_code_buffer, """ + Base.require(Base.PkgId(Base.UUID("$(string(pkg.uuid))"), $(repr(pkg.name)))) + """) + end + # Handle precompilation precompile_files = String[] @debug "running precompilation execution script..." @@ -315,22 +329,23 @@ function create_sysimg_object_file(object_file::String, @eval Module() begin using Base.Meta PrecompileStagingArea = Module() - for (_pkgid, _mod) in Base.loaded_modules - if !(_pkgid.name in ("Main", "Core", "Base")) - eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod)) - end - end + precompile_files = String[ $(join(map(repr, precompile_files), "\n" * " " ^ 8)) ] for file in precompile_files, statement in eachline(file) - try - # println(statement) - # This is taken from https://github.com/JuliaLang/julia/blob/2c9e051c460dd9700e6814c8e49cc1f119ed8b41/contrib/generate_precompile.jl#L375-L393 - ps = Meta.parse(statement) - isexpr(ps, :call) || continue - popfirst!(ps.args) # precompile(...) - ps.head = :tuple + # println(statement) + # This is taken from https://github.com/JuliaLang/julia/blob/2c9e051c460dd9700e6814c8e49cc1f119ed8b41/contrib/generate_precompile.jl#L375-L393 + ps = try + Meta.parse(statement) + catch + # guard against precompile statements that are not valid Julia syntax + continue + end + isexpr(ps, :call) || continue + popfirst!(ps.args) # precompile(...) + ps.head = :tuple + @static if VERSION <= v"1.9.0" l = ps.args[end] if (isexpr(l, :tuple) || isexpr(l, :curly)) && length(l.args) > 0 # Tuple{...} or (...) # XXX: precompile doesn't currently handle overloaded Vararg arguments very well. @@ -340,40 +355,48 @@ function create_sysimg_object_file(object_file::String, push!(l.args, 100) # form Vararg{T, 100} instead end end - # println(ps) - ps = Core.eval(PrecompileStagingArea, ps) - # XXX: precompile doesn't currently handle overloaded nospecialize arguments very well. - # Skipping them avoids the warning. - ms = length(ps) == 1 ? Base._methods_by_ftype(ps[1], 1, Base.get_world_counter()) : Base.methods(ps...) - ms isa Vector || continue - precompile(ps...) - catch e - # See julia issue #28808 - @debug "failed to execute \$statement" end + # println(ps) + local ps + while true + try + ps = Core.eval(PrecompileStagingArea, ps) + @static if VERSION <= v"1.9.0-beta1" + # XXX: precompile doesn't currently handle overloaded nospecialize arguments very well. + # Skipping them avoids the warning. + ms = length(ps) == 1 ? Base._methods_by_ftype(ps[1], 1, Base.get_world_counter()) : Base.methods(ps...) + ms isa Vector || @goto skip_precompile + end + break + catch e + if e isa UndefVarError + dep = string(e.var) + mods = filter(p -> p.first.name == dep, Base.loaded_modules) + if length(mods) != 1 + @debug "zero or multiple modules loaded with name \$dep" + @goto skip_precompile + else + _, mod = only(mods) + @debug "importing \$dep into PrecompileStagingArea" + Base.eval(PrecompileStagingArea, :(\$(Symbol(dep)) = \$(mod))) + end + else + # See julia issue #28808 + @debug "failed to execute \$statement: \$e" + @goto skip_precompile + end + end + end + precompile(ps...) + @label skip_precompile end + @eval PrecompileStagingArea begin $extra_precompiles end end # module """ - julia_code_buffer = IOBuffer() - # include all packages into the sysimg - print(julia_code_buffer, """ - Base.reinit_stdio() - @eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String - @eval Sys STDLIB = $(repr(abspath(Sys.BINDIR, "../share/julia/stdlib", string('v', VERSION.major, '.', VERSION.minor)))) - copy!(LOAD_PATH, [$(repr(project))]) # Only allow loading packages from current project - Base.init_depot_path() - """) - - for pkg in packages_sysimg - print(julia_code_buffer, """ - Base.require(Base.PkgId(Base.UUID("$(string(pkg.uuid))"), $(repr(pkg.name)))) - """) - end - # Make packages available in Main. It is unclear if this is the right thing to do. for pkg in packages print(julia_code_buffer, """ @@ -399,17 +422,11 @@ function create_sysimg_object_file(object_file::String, write(outputo_file, julia_code) # Read the input via stdin to avoid hitting the maximum command line limit - cmd = if isdefined(Base, :Linking) # pkgimages feature flag - cmd = `$(get_julia_cmd()) -O3 $sysimage_build_args - --sysimage=$base_sysimage --project=$project --output-o=$(object_file) $outputo_file` - @debug "running $cmd" JULIA_CPU_TARGET = cpu_target - addenv(cmd, "JULIA_CPU_TARGET" => cpu_target) - else - cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target -O3 $sysimage_build_args - --sysimage=$base_sysimage --project=$project --output-o=$(object_file) $outputo_file` + cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target $sysimage_build_args + --sysimage=$base_sysimage --project=$project --output-o=$(object_file) + $outputo_file` @debug "running $cmd" - cmd - end + non = incremental ? "" : "non" spinner = TerminalSpinners.Spinner(msg = "PackageCompiler: compiling $(non)incremental system image") @monitor_oom TerminalSpinners.@spin spinner run(cmd) @@ -431,7 +448,7 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_path::String`: The path to where the resulting sysimage should be saved. -- `project::String`: The project that should be active when the sysimage is created, +- `project::String`: The project directory that should be active when the sysimage is created, defaults to the currently active project. - `precompile_execution_file::Union{String, Vector{String}}`: A file or list of @@ -483,6 +500,13 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector compat_level::String="major", extra_precompiles::String = "", ) + # We call this at the very beginning to make sure that the user has a compiler available. Therefore, if no compiler + # is found, we throw an error immediately, instead of making the user wait a while before the error is thrown. + get_compiler_cmd() + + if isdir(sysimage_path) + error("The provided sysimage_path is a directory: $(sysimage_path). Please specify a full path including the sysimage filename.") + end if filter_stdlibs && incremental error("must use `incremental=false` to use `filter_stdlibs=true`") @@ -513,7 +537,7 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector error("cannot specify `base_sysimage` when `incremental=false`") end sysimage_stdlibs = filter_stdlibs ? gather_stdlibs_project(ctx) : stdlibs_in_sysimage() - base_sysimage = create_fresh_base_sysimage(sysimage_stdlibs; cpu_target) + base_sysimage = create_fresh_base_sysimage(sysimage_stdlibs; cpu_target, sysimage_build_args) else base_sysimage = something(base_sysimage, unsafe_string(Base.JLOptions().image_file)) end @@ -733,6 +757,8 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage, for example `-O1 --check-bounds=yes`. + +- `script::String`: Path to a file that gets executed in the `--output-o` process. """ function create_app(package_dir::String, app_dir::String; @@ -746,11 +772,18 @@ function create_app(package_dir::String, cpu_target::String=default_app_cpu_target(), include_lazy_artifacts::Bool=false, sysimage_build_args::Cmd=``, - include_transitive_dependencies::Bool=true) + include_transitive_dependencies::Bool=true, + script::Union{Nothing, String}=nothing) warn_official() + if filter_stdlibs && incremental + error("must use `incremental=false` to use `filter_stdlibs=true`") + end + # We call this at the very beginning to make sure that the user has a compiler available. Therefore, if no compiler + # is found, we throw an error immediately, instead of making the user wait a while before the error is thrown. + get_compiler_cmd() ctx = create_pkg_context(package_dir) - ctx.env.pkg === nothing && error("expected package to have a `name`-entry") + ctx.env.pkg === nothing && error("expected package to have a `name` and `uuid`") Pkg.instantiate(ctx, verbose=true, allow_autoprecomp = false) if executables === nothing @@ -771,6 +804,7 @@ function create_app(package_dir::String, # add precompile statements for functions that will be called from the C main() wrapper precompiles = String[] for (_, julia_main) in executables + push!(precompiles, "import $package_name") push!(precompiles, "isdefined($package_name, :$julia_main) && precompile(Tuple{typeof($package_name.$julia_main)})") end push!(precompiles, "precompile(Tuple{typeof(append!), Vector{String}, Vector{Any}})") @@ -785,7 +819,8 @@ function create_app(package_dir::String, cpu_target, sysimage_build_args, include_transitive_dependencies, - extra_precompiles = join(precompiles, "\n")) + extra_precompiles = join(precompiles, "\n"), + script) for (app_name, julia_main) in executables create_executable_from_sysimg(joinpath(app_dir, "bin", app_name), c_driver_program, string(package_name, ".", julia_main)) @@ -940,7 +975,7 @@ function create_library(package_dir::String, end ctx = create_pkg_context(package_dir) - ctx.env.pkg === nothing && error("expected package to have a `name`-entry") + ctx.env.pkg === nothing && error("expected package to have a `name` and `uuid`") Pkg.instantiate(ctx, verbose=true, allow_autoprecomp = false) lib_name = something(lib_name, ctx.env.pkg.name) diff --git a/test/runtests.jl b/test/runtests.jl index 5a1f8f41..c7dc88a3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,20 +3,36 @@ using Test using Libdl using Pkg +import TOML + ENV["JULIA_DEBUG"] = "PackageCompiler" # Make a new depot -new_depot = mktempdir() +const new_depot = mktempdir() mkpath(joinpath(new_depot, "registries")) ENV["JULIA_DEPOT_PATH"] = new_depot Base.init_depot_path() -is_slow_ci = haskey(ENV, "CI") && Sys.ARCH == :aarch64 +const is_ci = tryparse(Bool, get(ENV, "CI", "")) === true +const is_slow_ci = is_ci && Sys.ARCH == :aarch64 +const is_julia_1_6 = VERSION.major == 1 && VERSION.minor == 6 + +if is_julia_1_6 + @warn "This is Julia 1.6. Some tests will be skipped or modified." VERSION +end -if haskey(ENV, "CI") +if is_ci @show Sys.ARCH end +function remove_llvmextras(project_file) + proj = TOML.parsefile(project_file) + delete!(proj["deps"], "LLVMExtra_jll") + open(project_file, "w") do io + TOML.print(io, proj) + end +end + @testset "PackageCompiler.jl" begin @testset "create_sysimage" begin new_project = mktempdir() @@ -65,6 +81,10 @@ end @info "starting: create_app testset" incremental filter tmp_app_source_dir = joinpath(tmp, "MyApp") cp(app_source_dir, tmp_app_source_dir) + if is_julia_1_6 + # Issue #706 "Cannot locate artifact 'LLVMExtra'" on 1.6 so remove + remove_llvmextras(joinpath(tmp_app_source_dir, "Project.toml")) + end try create_app(tmp_app_source_dir, app_compiled_dir; incremental=incremental, force=true, filter_stdlibs=filter, include_lazy_artifacts=true, precompile_execution_file=joinpath(app_source_dir, "precompile_app.jl"), @@ -118,10 +138,10 @@ end @test occursin("From worker 4:\t8", app_output) @test occursin("From worker 5:\t8", app_output) - if VERSION >= v"1.7.0" + if VERSION >= v"1.7-" @test occursin("LLVMExtra path: ok!", app_output) end - @test occursin("MKL_jll path: ok!", app_output) + @test occursin("micromamba_jll path: ok!", app_output) # Test second app app_output = read(`$(app_path("SecondApp"))`, String)