Cuda.versioninfo() error "NVML could not be initialized"

sylvaticus · March 26, 2024, 8:58am

When I type CUDA.versioninfo() on a Ubuntu 22.04 box I got the following error:

julia> CUDA.versioninfo()
CUDA runtime 12.3, artifact installation
CUDA driver 12.3
┌ Error: Cannot use NVML, as it failed to initialize
│   exception =
│    NVML could not be initialized (NVML_ERROR_LIB_RM_VERSION_MISMATCH)
│    Stacktrace:
│      [1] error(s::String)
│        @ Base ./error.jl:35
│      [2] initialize_context
│        @ ~/.julia/packages/CUDA/htRwP/lib/nvml/libnvml.jl:14 [inlined]
│      [3] macro expansion
│        @ ~/.julia/packages/CUDA/htRwP/lib/nvml/NVML.jl:39 [inlined]
│      [4] macro expansion
│        @ ~/.julia/packages/CUDA/htRwP/lib/utils/memoization.jl:59 [inlined]
│      [5] has_nvml()
│        @ CUDA.NVML ~/.julia/packages/CUDA/htRwP/lib/nvml/NVML.jl:32
│      [6] versioninfo(io::Base.TTY)
│        @ CUDA ~/.julia/packages/CUDA/htRwP/src/utilities.jl:51
│      [7] versioninfo()
│        @ CUDA ~/.julia/packages/CUDA/htRwP/src/utilities.jl:42
│      [8] top-level scope
│        @ REPL[1]:1
│      [9] eval
│        @ ./boot.jl:385 [inlined]
│     [10] eval
│        @ ./Base.jl:88 [inlined]
│     [11] repleval(m::Module, code::Expr, ::String)
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/repl.jl:229
│     [12] (::VSCodeServer.var"#110#112"{Module, Expr, REPL.LineEditREPL, REPL.LineEdit.Prompt})()
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/repl.jl:192
│     [13] with_logstate(f::Function, logstate::Any)
│        @ Base.CoreLogging ./logging.jl:515
│     [14] with_logger
│        @ ./logging.jl:627 [inlined]
│     [15] (::VSCodeServer.var"#109#111"{Module, Expr, REPL.LineEditREPL, REPL.LineEdit.Prompt})()
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/repl.jl:193
│     [16] #invokelatest#2
│        @ Base ./essentials.jl:887 [inlined]
│     [17] invokelatest(::Any)
│        @ Base ./essentials.jl:884
│     [18] (::VSCodeServer.var"#62#63")()
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/eval.jl:34
└ @ CUDA.NVML ~/.julia/packages/CUDA/htRwP/lib/nvml/NVML.jl:41
Unknown NVIDIA driver, originally for CUDA 12.2

CUDA libraries: 
- CUBLAS: 12.3.4
- CURAND: 10.3.4
- CUFFT: 11.0.12
- CUSOLVER: 11.5.4
- CUSPARSE: 12.2.0
- CUPTI: 21.0.0
┌ Error: Cannot use NVML, as it failed to initialize
│   exception =
│    NVML could not be initialized (NVML_ERROR_LIB_RM_VERSION_MISMATCH)
│    Stacktrace:
│      [1] error(s::String)
│        @ Base ./error.jl:35
│      [2] initialize_context
│        @ ~/.julia/packages/CUDA/htRwP/lib/nvml/libnvml.jl:14 [inlined]
│      [3] macro expansion
│        @ ~/.julia/packages/CUDA/htRwP/lib/nvml/NVML.jl:39 [inlined]
│      [4] macro expansion
│        @ ~/.julia/packages/CUDA/htRwP/lib/utils/memoization.jl:59 [inlined]
│      [5] has_nvml()
│        @ CUDA.NVML ~/.julia/packages/CUDA/htRwP/lib/nvml/NVML.jl:32
│      [6] versioninfo(io::Base.TTY)
│        @ CUDA ~/.julia/packages/CUDA/htRwP/src/utilities.jl:69
│      [7] versioninfo()
│        @ CUDA ~/.julia/packages/CUDA/htRwP/src/utilities.jl:42
│      [8] top-level scope
│        @ REPL[1]:1
│      [9] eval
│        @ ./boot.jl:385 [inlined]
│     [10] eval
│        @ ./Base.jl:88 [inlined]
│     [11] repleval(m::Module, code::Expr, ::String)
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/repl.jl:229
│     [12] (::VSCodeServer.var"#110#112"{Module, Expr, REPL.LineEditREPL, REPL.LineEdit.Prompt})()
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/repl.jl:192
│     [13] with_logstate(f::Function, logstate::Any)
│        @ Base.CoreLogging ./logging.jl:515
│     [14] with_logger
│        @ ./logging.jl:627 [inlined]
│     [15] (::VSCodeServer.var"#109#111"{Module, Expr, REPL.LineEditREPL, REPL.LineEdit.Prompt})()
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/repl.jl:193
│     [16] #invokelatest#2
│        @ Base ./essentials.jl:887 [inlined]
│     [17] invokelatest(::Any)
│        @ Base ./essentials.jl:884
│     [18] (::VSCodeServer.var"#62#63")()
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/eval.jl:34
└ @ CUDA.NVML ~/.julia/packages/CUDA/htRwP/lib/nvml/NVML.jl:41
- NVML: missing

Julia packages: 
- CUDA: 5.2.0
- CUDA_Driver_jll: 0.7.0+1
- CUDA_Runtime_jll: 0.11.1+0

Toolchain:
- Julia: 1.10.0
- LLVM: 15.0.7

1 device:
┌ Error: Cannot use NVML, as it failed to initialize
│   exception =
│    NVML could not be initialized (NVML_ERROR_LIB_RM_VERSION_MISMATCH)
│    Stacktrace:
│      [1] error(s::String)
│        @ Base ./error.jl:35
│      [2] initialize_context
│        @ ~/.julia/packages/CUDA/htRwP/lib/nvml/libnvml.jl:14 [inlined]
│      [3] macro expansion
│        @ ~/.julia/packages/CUDA/htRwP/lib/nvml/NVML.jl:39 [inlined]
│      [4] macro expansion
│        @ ~/.julia/packages/CUDA/htRwP/lib/utils/memoization.jl:59 [inlined]
│      [5] has_nvml()
│        @ CUDA.NVML ~/.julia/packages/CUDA/htRwP/lib/nvml/NVML.jl:32
│      [6] versioninfo(io::Base.TTY)
│        @ CUDA ~/.julia/packages/CUDA/htRwP/src/utilities.jl:161
│      [7] versioninfo()
│        @ CUDA ~/.julia/packages/CUDA/htRwP/src/utilities.jl:42
│      [8] top-level scope
│        @ REPL[1]:1
│      [9] eval
│        @ ./boot.jl:385 [inlined]
│     [10] eval
│        @ ./Base.jl:88 [inlined]
│     [11] repleval(m::Module, code::Expr, ::String)
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/repl.jl:229
│     [12] (::VSCodeServer.var"#110#112"{Module, Expr, REPL.LineEditREPL, REPL.LineEdit.Prompt})()
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/repl.jl:192
│     [13] with_logstate(f::Function, logstate::Any)
│        @ Base.CoreLogging ./logging.jl:515
│     [14] with_logger
│        @ ./logging.jl:627 [inlined]
│     [15] (::VSCodeServer.var"#109#111"{Module, Expr, REPL.LineEditREPL, REPL.LineEdit.Prompt})()
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/repl.jl:193
│     [16] #invokelatest#2
│        @ Base ./essentials.jl:887 [inlined]
│     [17] invokelatest(::Any)
│        @ Base ./essentials.jl:884
│     [18] (::VSCodeServer.var"#62#63")()
│        @ VSCodeServer ~/.local/share/code-server/extensions/julialang.language-julia-1.75.2-universal/scripts/packages/VSCodeServer/src/eval.jl:34
└ @ CUDA.NVML ~/.julia/packages/CUDA/htRwP/lib/nvml/NVML.jl:41
  0: NVIDIA RTX A4500 (sm_86, 18.809 GiB / 19.705 GiB available)

However it then “seems” to work right (I am new to GPU)… what could be the issue here ?

maleadt · March 28, 2024, 10:43am

NVML is optional. It is however provided by your system (i.e., by the NVIDIA driver), so the mismatch indicates a system configuration issue. Did you perhaps upgrade your driver without rebooting?

sylvaticus · March 28, 2024, 10:48am

thanks, I’ll try by rebooting the server…