Thanks for the helpful comments to everyone who replied! After some trial and error I found my problem was in my use of the @distributed macro without an @sync. Below is a MWE highlighting the behavior I was encountering.
using Distributed
function sync_foo(T)
arr = [1,T]
@sync @distributed for i in 2:T-1
push!(arr, i)
end
return arr
end
function foo(T)
arr = [1,T]
@distributed for i in 2:T-1
push!(arr, i)
end
return arr
end
function foo_printer(T)
arr = foo(T)
println(arr)
return arr
end
function foo_noprinter(T)
arr = foo(T)
return arr
end
function sync_foo_printer(T)
arr = sync_foo(T)
println(arr)
return arr
end
function sync_foo_noprinter(T)
arr = sync_foo(T)
return arr
end
T = 100000
arr = foo_printer(T)
l1 = size(arr)
arr = foo_noprinter(T)
l2 = size(arr)
arr = sync_foo_printer(T)
l3 = size(arr)
arr = sync_foo_noprinter(T)
l4 = size(arr)
println(l1,l2,l3,l4) # (100000,), (2,), (100000,), (100000,)