I’m new to julia reinforcementlearning, trying to run some experiments with PPO. Tried a lot of things, somehow can’t get ActorCritic to work. Maybe someone could give me a tip?
Julia Version 1.10.4
Commit 48d4fd4843 (2024-06-04 10:41 UTC)
Build Info:
Official https://julialang.org/ release
Platform Info:
OS: Windows (x86_64-w64-mingw32)
CPU: 8 × Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz
WORD_SIZE: 64
LIBM: libopenlibm
LLVM: libLLVM-15.0.7 (ORCJIT, skylake)
Threads: 1 default, 0 interactive, 1 GC (on 8 virtual cores)
Environment:
JULIA_EDITOR = code
JULIA_NUM_THREADS =
ReinforcementLearning v0.11.0
ReinforcementLearningCore v0.15.3
Flux v0.14.16
Here a mwe:
using ReinforcementLearning
using ReinforcementLearningCore
using Flux
policy_net = Chain(
Dense(1, 32, relu),
Dense(32, 32, relu),
Dense(32, 3)
)
# Critic Network
critic_net = Chain(
Dense(1, 32, relu),
Dense(32, 32, relu),
Dense(32, 1) # Output layer with 1 unit for the value estimate
)
agent = Agent(
policy=PPOPolicy(
approximator= ActorCritic(;actor = policy_net, critic = critic_net)
,
γ = 0.99, # Discount factor
clip_range = 0.2, # PPO clipping parameter
λ = 0.95, # GAE parameter
batch_size = 64, # Number of steps per update
epochs = 10, # Epochs for policy update
),
trajectory = PPOTrajectory(;capacity = 64), #replay buffer size
)
When I try to implement the agent portion of the code, I get the following error msg:
LoadError: MethodError: no method matching ActorCritic(::Chain{Tuple{Dense{typeof(relu), Matrix{Float32}, Vector{Float32}}, Dense{typeof(relu), Matrix{Float32}, Vector{Float32}}, Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}}}, ::Chain{Tuple{Dense{typeof(relu), Matrix{Float32}, Vector{Float32}}, Dense{typeof(relu), Matrix{Float32}, Vector{Float32}}, Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}}})
Stacktrace:
[1] ActorCritic(; actor::Chain{Tuple{Dense{typeof(relu), Matrix{Float32}, Vector{Float32}}, Dense{typeof(relu), Matrix{Float32}, Vector{Float32}}, Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}}}, critic::Chain{Tuple{Dense{typeof(relu), Matrix{Float32}, Vector{Float32}}, Dense{typeof(relu), Matrix{Float32}, Vector{Float32}}, Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}}})
@ ReinforcementLearningCore
[3] eval
@ .\boot.jl:385 [inlined]
[4] include_string(mapexpr::typeof(REPL.softscope), mod::Module, code::String, filename::String)
@ Base .\loading.jl:2076
[5] invokelatest(::Any, ::Any, ::Vararg{Any}; kwargs::@Kwargs{})
@ Base .\essentials.jl:892
[6] invokelatest(::Any, ::Any, ::Vararg{Any})
@ Base .\essentials.jl:889
[7] inlineeval(m::Module, code::String, code_line::Int64, code_column::Int64, file::String; softscope::Bool)
@ VSCodeServer c:\Users\X\.vscode\extensions\julialang.language-julia-1.83.2\scripts\packages\VSCodeServer\src\eval.jl:271
[8] (::VSCodeServer.var"#69#74"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams})()
@ VSCodeServer c:\Users\X\.vscode\extensions\julialang.language-julia-1.83.2\scripts\packages\VSCodeServer\src\eval.jl:181
[9] withpath(f::VSCodeServer.var"#69#74"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams}, path::String)
@ VSCodeServer c:\Users\X\.vscode\extensions\julialang.language-julia-1.83.2\scripts\packages\VSCodeServer\src\repl.jl:276
[10] (::VSCodeServer.var"#68#73"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams})()
@ VSCodeServer c:\Users\X\.vscode\extensions\julialang.language-julia-1.83.2\scripts\packages\VSCodeServer\src\eval.jl:179
[11] hideprompt(f::VSCodeServer.var"#68#73"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams})
@ VSCodeServer c:\Users\X\.vscode\extensions\julialang.language-julia-1.83.2\scripts\packages\VSCodeServer\src\repl.jl:38
[12] (::VSCodeServer.var"#67#72"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams})()
@ VSCodeServer c:\Users\X\.vscode\extensions\julialang.language-julia-1.83.2\scripts\packages\VSCodeServer\src\eval.jl:150
[13] with_logstate(f::Function, logstate::Any)
@ Base.CoreLogging .\logging.jl:515
[14] with_logger
@ .\logging.jl:627 [inlined]
[15] (::VSCodeServer.var"#66#71"{VSCodeServer.ReplRunCodeRequestParams})()
@ VSCodeServer c:\Users\X\.vscode\extensions\julialang.language-julia-1.83.2\scripts\packages\VSCodeServer\src\eval.jl:263
[16] #invokelatest#2
@ .\essentials.jl:892 [inlined]
[17] invokelatest(::Any)
@ Base .\essentials.jl:889
[18] (::VSCodeServer.var"#64#65")()
@ VSCodeServer c:\Users\X\.vscode\extensions\julialang.language-julia-1.83.2\scripts\packages\VSCodeServer\src\eval.jl:34