MWE of a segmentation fault when trying to use a dev local package that contains a function that calls a Python function/structure

Hello,
For coherence I leave the original formulation in this cell but I have made a MWE in the second cell below. Thanks

I am building a package that has some dependencies including PythonCall. The module is like this:

module IntrospectiveStreams

    using DataFrames, DataFramesMeta
    using PolygonOps
    using StaticArrays
    using PythonCall
    using CairoMakie, AlgebraOfGraphics
    using ElectronDisplay

    @py begin
        import galstreams
        import astropy.coordinates as coord
        import astropy.units as u
        import astropy.table as at
        import ezmist
        import gala.coordinates as galacoord
        import pyia
    end

    export  correct_extinction_Gaia,
            curation!,
            load_stream_track,
            get_isochrone,
            compute_in_self_coords!,
            mask_gc!,
            filter_cmd!,
            filter_on_sky!,
            filter_on_Ī¼_plane!,
            filter_along_Ļ•ā‚,
            filter_along_Ļ•ā‚!,
            filter_with_Ļ•ā‚‚!,
            filter_PWB18!,
            filter_box_on_Ī¼_plane!,
            reflex_correct!,
            clean_xmatch!

    export  plot_histog_on_sky,
            plot_histog_on_sky_with_gc,
            plot_histog_on_sky_self_frame,
            plot_scatter_on_sky_self_frame,
            plot_scatter_on_sky_Ī¼_arrows_self_frame,
            plot_scatter_on_sky_Ī¼_corr_arrows_self_frame,
            plot_histog_on_Ī¼_plane,
            plot_histog_on_Ī¼_plane_self_frame,
            plot_scatter_on_Ī¼_plane_self_frame,
            plot_scatter_on_Ī¼_corr_plane_self_frame,
            plot_track_on_Ī¼_corr_plane_self_frame,
            plot_histog_cmd,
            plot_isochrone_cmd

    export correct_extinction_Gaia_loop,
           basic_pipeline

    include("data_methods.jl")
    include("plot_methods.jl")
    include("pipelines.jl")

    include("run_extinction.jl")


end

Three other important parts of code are the ā€œrun_extinction.jlā€:

"""Run extinction correction for all the streams."""

name_s = ["GD-1", "Pal5", "Jhelum", "PS1-A", "Fjorm-M68"]

correct_extinction_Gaia_loop(name_s)

and two functions:

"""Extinction correction for Gaia magnitudes from Gaia dataset."""
function correct_extinction_Gaia(file_orig::String, file_corr::String)::Nothing
    println(file_orig)
    data = at.Table.read(file_orig)
    println("done")
    g = pyia.GaiaData(data)
    bp0 = g.get_BP0()
    rp0 = g.get_RP0()
    g0 = g.get_G0()
    data["bp"] = bp0
    data["rp"] = rp0
    data["g"] = g0
    data.write(file_corr, format="fits", overwrite=true)
    return nothing
end
"""Performing extinction correction for a given list of streams."""
function correct_extinction_Gaia_loop(name_s::Vector{String})
    for i in 1:length(name_s)
        println("Correcting extinction of stream $(name_s[i])")
        host_dir = "/home/mmestre/casa/work/data/cats"
        file_orig = "$(host_dir)/GaiaDR3-$(name_s[i])-all.fits"
        file_corr = "$(host_dir)/GaiaDR3-$(name_s[i])-all_extincorr.fits"
        correct_extinction_Gaia(file_orig, file_corr)
    end
end

As presented, everything works as expected, the files are correctly processed. But I would like that the ā€œrun_extinction.jlā€ file not to be included in the package. I want that action to be done from the REPL after ā€œusing IntrospectiveStreamsā€ (not while using as it is now working). But it fails with this log related to PythonCall:

julia> name_s = ["GD-1", "Pal5", "Jhelum", "PS1-A", "Fjorm-M68"]
5-element Vector{String}:
 "GD-1"
 "Pal5"
 "Jhelum"
 "PS1-A"
 "Fjorm-M68"

julia> correct_extinction_Gaia_loop(name_s)
Correcting extinction of stream GD-1
/home/mmestre/casa/work/data/cats/GaiaDR3-GD-1-all.fits

[922770] signal (11.1): Segmentation fault
in expression starting at REPL[7]:1
PyObject_GetAttr at /usr/local/src/conda/python-3.10.8/Objects/object.c:922
PyObject_GetAttr at /home/mmestre/.julia/packages/PythonCall/dsECZ/src/cpython/pointers.jl:299 [inlined]
macro expansion at /home/mmestre/.julia/packages/PythonCall/dsECZ/src/Py.jl:131 [inlined]
pygetattr at /home/mmestre/.julia/packages/PythonCall/dsECZ/src/abstract/object.jl:60
getproperty at /home/mmestre/.julia/packages/PythonCall/dsECZ/src/Py.jl:272
unknown function (ip: 0x7f61aeee3fd6)
_jl_invoke at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940
correct_extinction_Gaia at /home/mmestre/.julia/dev/IntrospectiveStreams/src/data_methods.jl:4
correct_extinction_Gaia_loop at /home/mmestre/.julia/dev/IntrospectiveStreams/src/pipelines.jl:8
jfptr_correct_extinction_Gaia_loop_624 at /home/mmestre/.julia/compiled/v1.9/IntrospectiveStreams/ZtSfS_tWCnV.so (unknown line)
_jl_invoke at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940
jl_apply at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/julia.h:1879 [inlined]
do_call at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/interpreter.c:126
eval_value at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/interpreter.c:226
eval_stmt_value at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/interpreter.c:177 [inlined]
eval_body at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/interpreter.c:624
jl_interpret_toplevel_thunk at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/interpreter.c:762
jl_toplevel_eval_flex at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/toplevel.c:912
jl_toplevel_eval_flex at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/toplevel.c:856
jl_toplevel_eval_flex at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/toplevel.c:856
jl_toplevel_eval_flex at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/toplevel.c:856
ijl_toplevel_eval_in at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/toplevel.c:971
eval at ./boot.jl:370 [inlined]
eval_user_input at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:153
repl_backend_loop at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:249
#start_repl_backend#46 at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:234
start_repl_backend at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:231
_jl_invoke at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940
#run_repl#59 at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:379
run_repl at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:365
jfptr_run_repl_59932.clone_1 at /home/mmestre/.julia/juliaup/julia-1.9.0-rc3+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940
#1017 at ./client.jl:421
jfptr_YY.1017_33143.clone_1 at /home/mmestre/.julia/juliaup/julia-1.9.0-rc3+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940
jl_apply at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/julia.h:1879 [inlined]
jl_f__call_latest at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/builtins.c:774
#invokelatest#2 at ./essentials.jl:816 [inlined]
invokelatest at ./essentials.jl:813 [inlined]
run_main_repl at ./client.jl:405
exec_options at ./client.jl:322
_start at ./client.jl:522
jfptr__start_40741.clone_1 at /home/mmestre/.julia/juliaup/julia-1.9.0-rc3+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940
jl_apply at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/julia.h:1879 [inlined]
true_main at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/jlapi.c:573
jl_repl_entrypoint at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/src/jlapi.c:717
main at /cache/build/default-amdci4-4/julialang/julia-release-1-dot-9/cli/loader_exe.c:59
__libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line)
unknown function (ip: 0x401098)
Allocations: 20204319 (Pool: 20194208; Big: 10111); GC: 31
Segmentation fault

Specifically it fails exactly in the line that says: data = at.Table.read(file_orig).
I would really appreciate any help.

1 Like

I have made a MWE. The dev package is below:

module Cats

    using PythonCall

    @py import astropy.constants as cons

    export myfoo

    function myfoo()
        g = cons.G
        return g
    end

end

The REPL log is:

(@v1.9) pkg> activate .
  Activating project at `~/.julia/dev/Cats`

julia> using Cats
[ Info: Precompiling Cats [3f873cc4-696d-4df5-850d-2788ce7dad7b]

julia> myfoo
myfoo (generic function with 1 method)

julia> myfoo()

[941469] signal (11.1): Segmentation fault
in expression starting at REPL[4]:1
PyObject_GetAttr at /usr/local/src/conda/python-3.10.8/Objects/object.c:922
PyObject_GetAttr at /home/mmestre/.julia/packages/PythonCall/dsECZ/src/cpython/pointers.jl:299 [inlined]
macro expansion at /home/mmestre/.julia/packages/PythonCall/dsECZ/src/Py.jl:131 [inlined]
pygetattr at /home/mmestre/.julia/packages/PythonCall/dsECZ/src/abstract/object.jl:60
getproperty at /home/mmestre/.julia/packages/PythonCall/dsECZ/src/Py.jl:272
unknown function (ip: 0x7f205c9be5c6)
_jl_invoke at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2940
myfoo at /home/mmestre/.julia/dev/Cats/src/Cats.jl:10
unknown function (ip: 0x7f205c9be51f)
_jl_invoke at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2940
jl_apply at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/julia.h:1879 [inlined]
do_call at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/interpreter.c:126
eval_value at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/interpreter.c:226
eval_stmt_value at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/interpreter.c:177 [inlined]
eval_body at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/interpreter.c:624
jl_interpret_toplevel_thunk at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/interpreter.c:762
jl_toplevel_eval_flex at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/toplevel.c:912
jl_toplevel_eval_flex at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/toplevel.c:856
jl_toplevel_eval_flex at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/toplevel.c:856
jl_toplevel_eval_flex at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/toplevel.c:856
ijl_toplevel_eval_in at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/toplevel.c:971
eval at ./boot.jl:370 [inlined]
eval_user_input at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:153
repl_backend_loop at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:249
#start_repl_backend#46 at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:234
start_repl_backend at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:231
_jl_invoke at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2940
#run_repl#59 at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:379
run_repl at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:365
jfptr_run_repl_60276.clone_1 at /home/mmestre/.julia/juliaup/julia-1.9.0+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2940
#1017 at ./client.jl:421
jfptr_YY.1017_32248.clone_1 at /home/mmestre/.julia/juliaup/julia-1.9.0+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2940
jl_apply at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/julia.h:1879 [inlined]
jl_f__call_latest at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/builtins.c:774
#invokelatest#2 at ./essentials.jl:816 [inlined]
invokelatest at ./essentials.jl:813 [inlined]
run_main_repl at ./client.jl:405
exec_options at ./client.jl:322
_start at ./client.jl:522
jfptr__start_37386.clone_1 at /home/mmestre/.julia/juliaup/julia-1.9.0+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined]
ijl_apply_generic at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/gf.c:2940
jl_apply at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/julia.h:1879 [inlined]
true_main at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/jlapi.c:573
jl_repl_entrypoint at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/src/jlapi.c:717
main at /cache/build/default-amdci4-0/julialang/julia-release-1-dot-9/cli/loader_exe.c:59
__libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line)
unknown function (ip: 0x401098)
Allocations: 1902754 (Pool: 1899088; Big: 3666); GC: 3
Segmentation fault

If I convert my Cats.jl module to a script it runs well. The problem is when trying to ā€œuseā€ Cats.
Thank you very much in advance.

1 Like

Most likely a case of precompilation storing a stale pointer to a Python object. See GitHub - JuliaPy/PyCall.jl: Package to call Python functions from the Julia language for advice how to handle this with PyCall. The solution should be the same or corresponding for PythonCall.

1 Like

Thanks, you are right. I have found the equivalent of PythonCall in its documentation:

module MyModule
  using PythonCall
  const foo = PythonCall.pynew() # initially NULL
  function __init__()
    PythonCall.pycopy!(foo, pyimport("foo"))
  end
  bar() = foo.bar() # now ok
end
1 Like