Does Debian's BenchmarkGames show representative performance?

Community
19

@palli and I did some binary tree tests recently measuring influence of GC and startup times. With the MWE

using BenchmarkTools

struct Node
    l::Union{Node,Nothing}
    r::Union{Node,Nothing}
end

function make(n)
    n === 0 ? Node(nothing, nothing) : Node(make(n - 1), make(n - 1))
end

function check(node)
    node.l === nothing ? 1 : 1 + check(node.l) + check(node.r)
end

function binary_trees(io, n)
    write(io, "stretch tree of depth $(n+1)\t check: $(check(make(n+1)))\n")

    long_tree::Node = make(n)

    d = 4
    while d <= n
        niter = 1 << (n - d + 4)
        ct = Vector{Int}(undef, niter)
        GC.enable(false)
        let d = d
            Threads.@threads for i = 1:niter
                ct[i] = check(make(d))
            end
        end
        GC.enable(true)
        GC.gc(false)
        c = sum(ct)
        write(io, "$niter\t trees of depth $d\t check: $c\n")
        d += 2
    end

    write(io, "long lived tree of depth $n\t check: $(check(long_tree))\n")
end#function

isinteractive() || @time binary_trees(stdout, parse(Int, ARGS[1]))

we get something like

  3.159 s (302689310 allocations: 9.04 GiB) # no GC, measured in VS Code via @btime
TotalMilliseconds : 4656,7325 # no GC, incl. startup & compilation measured in PS
  4.545 s (302689301 allocations: 9.04 GiB) # delayed GC, measured in VS Code via @btime
TotalMilliseconds : 5303,0138 # delayed GC,  incl. startup & compilation measured in PS
  6.321 s (305666479 allocations: 9.07 GiB) # current recordholder, measured in VS Code
TotalMilliseconds : 7163,3355 # current recordholder, incl. startup & compilation measured in PS

We were asking ourselves if delaying GC is conforming to the benchmark rules?

1 Like