Fastest way to fill a Sparse Matrix?

This is how I understand it. I tried to translate your problem to use sparse

using BenchmarkTools
using SparseArrays

function test(ds)
    is = Int[]
    js = Int[]
    vs = Float64[]
    for j = 1:size(ds, 2)
        d = ds[:, j]
        dis, dvs = findnz(d)
        for (i, v) in zip(dis, dvs)
            push!(is, i)
            push!(js, j)
            push!(vs, v)
        end
    end
    L = sparse(is, js, vs, size(ds, 1), size(ds, 2))
    @assert L == ds
    L
end

@btime test(ds) setup=(ds=sprand(10000, 10000, 0.05))

Does this help?