I want to produce a private package that fully manages and runs a series of distributed processes on private workers, but I’m seeing unstable code loading & scope issues. The kind of approach below errors most of the time with On worker 12: UndefVarError: #104#111 not defined
type messages (which seems to be a code loading scope issue with the pmap anonymous function) but sometimes it works…
The general idea is this…
Usage:
using Foo
Foo.launch() #add private remote machines to the workers and load code on each
Foo.rundist() #run distributed processes, while monitoring workers locally
module Foo
using Distributed
export dosteps
function launch()
remote_hosts = get_dns_remote_machines()
addprocs(remote_hosts)
@everwhere @eval using Foo
end
function dosteps(i, remchan) #The function that does the actual work
#do something long that produces a dict with info
put!(remchan, info_dict)
#do something else long that produces a dict with info
put!(remchan info_dict)
end
const EXIT = Ref{Bool}(false)
function rundist()
infochan = RemoteChannel(()->Channel{Dict{Symbol,Any}}(100))
@async begin #an async local worker monitoring process
while !EXIT[]
d = take!(infochaan)
message = # form a messsage from dict d
println(message)
wait(Timer(1))
end
end
try
pmap(i->dosteps(i, infochan), 1:10)
catch e
EXIT[] = true
throw(e)
end
EXIT[] = true
end
end #module