What is the recommended way to run N workers asynchronously?

General Usage
,
2

You can use “Task farm” pattern from this tutorial.

using HTTP, JSON3
using Base.Threads: @spawn

umap(f, xs; kwargs...) = umap(f, Any, xs; kwargs...)
function umap(f, TY::Type, xs::Channel; ntasks = Threads.nthreads(), buffersize = ntasks)
    return Channel{TY}(buffersize) do ys
        @sync for i in 1:ntasks
            @spawn for x in xs
                put!(ys, f(i, x))
            end
        end
    end
end

function calc(nworkers = 10, nreqs = 100)
    xs = Channel{Int}(Inf)
    for _ in 1:nreqs
        put!(xs, 1)
    end
    close(xs)

    res = umap(Int, xs; ntasks = nworkers, buffersize = nreqs) do name, n
        println("worker-$name")
        response = HTTP.request("GET", "https://qrng.anu.edu.au/API/jsonI.php?length=$(n)&type=uint16")
        json = JSON3.read(String(response.body))
        sum(json["data"])
    end

    collect(res)
end

calc() # long list of worker name and vector of numbers in the end.

Here @spawn is used, which can be an overkill, you can change it to @async, but it really doesn’t matter in this case.

2 Likes