Trying to Use PackageCompiler.jl

General Usage
1

I’m trying to use packagecompiler.jl and I don’t know what I’m doing wrong:

(NNFuelGage.jl) pkg> 

julia> using Interpolations

julia> using JuMP

julia> using LazySets
usin
julia> using MathOptInterface

julia> using Gurobi

julia> PackageCompiler.create_sysimage(["SymEngine", "MacroTools", "Interpolations", "JuMP", "LazySets", "MathOptInterface", "Calculus", "NLsolve", "Gurobi", "Plots"]; sysimage_path="fuel_gage_sysimage", precompile_statements_file="fuel_gage_2.jl", replace_default=false)
Precompiling project...
  2 dependencies successfully precompiled in 10 seconds (282 already precompiled)
ERROR: package(s) Interpolations, JuMP, LazySets, MathOptInterface, Gurobi not in project

The packages are clearly in my current project and I can “using” them but package compiler can’t find them?

Also why do I need to provide a list of packages when those packages are already called / used in the precompile trace file?

Update ~~ I’m unsure as to why I was able to “using” packages like JuMP w/o them being in my Project.toml file for the current active project, but I ]add’ed them and then ran PackageCompiler again and got the following error:


julia> PackageCompiler.create_sysimage(["SymEngine", "MacroTools", "Interpolations", "JuMP", "LazySets", "MathOptInterface", "Calculus", "NLsolve", "Gurobi", "Plots"]; sysimage_path="fuel_gage_sysimage.so", precompile_statements_file="fuel_gage_2.jl", replace_default=false)
[ Info: PackageCompiler: creating system image object file, this might take a while...
ERROR: failed process: Process(`/home/chelsea/Software/julia-1.6.1/bin/julia --color=yes --startup-file=no --cpu-target=native --sysimage=/home/chelsea/Dropbox/nn_fuel_gage/src/NNFuelGage.jl/fuel_gage_sysimage.so --project=/home/chelsea/Dropbox/nn_fuel_gage/src/NNFuelGage.jl --output-o=/tmp/jl_N3Wg41.o -e "Base.reinit_stdio()
@eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String
@eval Sys STDLIB = \"/home/chelsea/Software/julia-1.6.1/share/julia/stdlib/v1.6\"
Base.init_load_path()
if isdefined(Base, :init_active_project)
    Base.init_active_project()
end
Base.init_depot_path()
import SymEngine
import MacroTools
import Interpolations
import JuMP
import LazySets
import MathOptInterface
import Calculus
import NLsolve
import Gurobi
import Plots
# This @eval prevents symbols from being put into Main
@eval Module() begin
    using Base.Meta
    PrecompileStagingArea = Module()
    for (_pkgid, _mod) in Base.loaded_modules
        if !(_pkgid.name in (\"Main\", \"Core\", \"Base\"))
            eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod))
        end
    end
    precompile_files = String[
        \"/tmp/jl_P2NBX3\"
        \"/home/chelsea/Dropbox/nn_fuel_gage/src/NNFuelGage.jl/fuel_gage_2.jl\"
    ]
    for file in precompile_files, statement in eachline(file)
        try
            # println(statement)
            # This is taken from https://github.com/JuliaLang/julia/blob/2c9e051c460dd9700e6814c8e49cc1f119ed8b41/contrib/generate_precompile.jl#L375-L393
            ps = Meta.parse(statement)
            isexpr(ps, :call) || continue
            popfirst!(ps.args) # precompile(...)
            ps.head = :tuple
            l = ps.args[end]
            if (isexpr(l, :tuple) || isexpr(l, :curly)) && length(l.args) > 0 # Tuple{...} or (...)
                # XXX: precompile doesn't currently handle overloaded Vararg arguments very well.
                # Replacing N with a large number works around it.
                l = l.args[end]
                if isexpr(l, :curly) && length(l.args) == 2 && l.args[1] === :Vararg # Vararg{T}
                    push!(l.args, 100) # form Vararg{T, 100} instead
                end
            end
            # println(ps)
            ps = Core.eval(PrecompileStagingArea, ps)
            # XXX: precompile doesn't currently handle overloaded nospecialize arguments very well.
            # Skipping them avoids the warning.
            ms = length(ps) == 1 ? Base._methods_by_ftype(ps[1], 1, Base.get_world_counter()) : Base.methods(ps...)
            ms isa Vector || continue
            precompile(ps...)
        catch e
            # See julia issue #28808
            @debug \"failed to execute \$statement\"
        end
    end
end # module
empty!(LOAD_PATH)
empty!(DEPOT_PATH)
"`, ProcessSignaled(9)) [0]

Stacktrace:
 [1] pipeline_error
   @ ./process.jl:525 [inlined]
 [2] run(::Cmd; wait::Bool)
   @ Base ./process.jl:440
 [3] run
   @ ./process.jl:438 [inlined]
 [4] create_sysimg_object_file(object_file::String, packages::Vector{String}; project::String, base_sysimage::String, precompile_execution_file::Vector{String}, precompile_statements_file::Vector{String}, cpu_target::String, script::Nothing, isapp::Bool, sysimage_build_args::Cmd)
   @ PackageCompiler ~/.julia/packages/PackageCompiler/r8Xcy/src/PackageCompiler.jl:371
 [5] create_sysimage(packages::Vector{String}; sysimage_path::String, project::String, precompile_execution_file::Vector{String}, precompile_statements_file::String, incremental::Bool, filter_stdlibs::Bool, replace_default::Bool, base_sysimage::Nothing, isapp::Bool, julia_init_c_file::Nothing, version::Nothing, compat_level::String, soname::Nothing, cpu_target::String, script::Nothing, sysimage_build_args::Cmd)
   @ PackageCompiler ~/.julia/packages/PackageCompiler/r8Xcy/src/PackageCompiler.jl:538
 [6] top-level scope
   @ REPL[27]:1
2

Try running that command on the command line to see if the error is more informative:

/home/chelsea/Software/julia-1.6.1/bin/julia --color=yes --startup-file=no --cpu-target=native --sysimage=/home/chelsea/Dropbox/nn_fuel_gage/src/NNFuelGage.jl/fuel_gage_sysimage.so --project=/home/chelsea/Dropbox/nn_fuel_gage/src/NNFuelGage.jl --output-o=/tmp/jl_N3Wg41.o -e "Base.reinit_stdio()
@eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String
@eval Sys STDLIB = \"/home/chelsea/Software/julia-1.6.1/share/julia/stdlib/v1.6\"
Base.init_load_path()
if isdefined(Base, :init_active_project)
    Base.init_active_project()
end
Base.init_depot_path()
import SymEngine
import MacroTools
import Interpolations
import JuMP
import LazySets
import MathOptInterface
import Calculus
import NLsolve
import Gurobi
import Plots
# This @eval prevents symbols from being put into Main
@eval Module() begin
    using Base.Meta
    PrecompileStagingArea = Module()
    for (_pkgid, _mod) in Base.loaded_modules
        if !(_pkgid.name in (\"Main\", \"Core\", \"Base\"))
            eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod))
        end
    end
    precompile_files = String[
        \"/tmp/jl_P2NBX3\"
        \"/home/chelsea/Dropbox/nn_fuel_gage/src/NNFuelGage.jl/fuel_gage_2.jl\"
    ]
    for file in precompile_files, statement in eachline(file)
        try
            # println(statement)
            # This is taken from https://github.com/JuliaLang/julia/blob/2c9e051c460dd9700e6814c8e49cc1f119ed8b41/contrib/generate_precompile.jl#L375-L393
            ps = Meta.parse(statement)
            isexpr(ps, :call) || continue
            popfirst!(ps.args) # precompile(...)
            ps.head = :tuple
            l = ps.args[end]
            if (isexpr(l, :tuple) || isexpr(l, :curly)) && length(l.args) > 0 # Tuple{...} or (...)
                # XXX: precompile doesn't currently handle overloaded Vararg arguments very well.
                # Replacing N with a large number works around it.
                l = l.args[end]
                if isexpr(l, :curly) && length(l.args) == 2 && l.args[1] === :Vararg # Vararg{T}
                    push!(l.args, 100) # form Vararg{T, 100} instead
                end
            end
            # println(ps)
            ps = Core.eval(PrecompileStagingArea, ps)
            # XXX: precompile doesn't currently handle overloaded nospecialize arguments very well.
            # Skipping them avoids the warning.
            ms = length(ps) == 1 ? Base._methods_by_ftype(ps[1], 1, Base.get_world_counter()) : Base.methods(ps...)
            ms isa Vector || continue
            precompile(ps...)
        catch e
            # See julia issue #28808
            @debug \"failed to execute \$statement\"
        end
    end
end # module
empty!(LOAD_PATH)
empty!(DEPOT_PATH)
"
3

I tried but it just failed and said “killed” after a while.

4

Does this happen even without Gurobi? I am trying to reproduce it but Gurobi.jl requires non-free softwar and it errors if you don’t have it installed.

5

I am not sure it will help here, but I think the key thing you want to do is ensure that your global environment is as clean as possible. Basically Revise, IJulia, PackageCompiler and BenchmarkTools is about all I have in mine. At that point, there is no chance of anything funky going on with it using a package from the “stacked” environments and whatever is in your Project.toml should be gospel.

… but as I said, that might not solve this particular problem.

6

Are you saying project environments stack on top of the e.g. julia1.6 environment?

7

Exactly. Because of this, I find it extremely important to put as little as possible in the global environment. Things like PackageCompiler or IJulia that intuitively layer on top of an existing project make sense. Otherwise, the cleaner the better.

1 Like
8

Tried it again, got this new error:

julia1.6 --project="." package_compiler_build.jl 
ERROR: LoadError: type Dict has no field deps
Stacktrace:
 [1] getproperty(x::Dict{Base.UUID, Pkg.Types.PackageEntry}, f::Symbol)
   @ Base ./Base.jl:33
 [2] create_sysimage(packages::Vector{String}; sysimage_path::String, project::String, precompile_execution_file::Vector{String}, precompile_statements_file::String, incremental::Bool, filter_stdlibs::Bool, replace_default::Bool, cpu_target::String, script::Nothing, sysimage_build_args::Cmd, include_transitive_dependencies::Bool, base_sysimage::Nothing, isapp::Bool, julia_init_c_file::Nothing, version::Nothing, compat_level::String, soname::Nothing)
   @ PackageCompiler ~/.julia/packages/PackageCompiler/r1Ahd/src/PackageCompiler.jl:583
 [3] top-level scope
   @ ~/Dropbox/.../package_compiler_build.jl:3
in expression starting at /home/chelsea/Dropbox/.../package_compiler_build.jl:3

Going to try to make a new fake environment and see if it works in the simplest case. Not sure it will be helpful for my development needs immediately but maybe it will get to the bottom of this.

9

Update to 1.6.2 or wait a day until the next version that includes a fix for earlier Julia 1.6 versions. Ref https://github.com/JuliaLang/PackageCompiler.jl/pull/593#issuecomment-939846579