Strange behavior with list comprehensions using Iterators.Stateful

altre · October 10, 2019, 7:25am

I’m having trouble to understand the following behavior of Iterators.Stateful. This is as expected

julia> for a in Iterators.Stateful([1,2,3])
             print(a)
         end
123

But this seems strange:

julia> [a for a in Iterators.Stateful([1,2,3])]
2-element Array{Int64,1}:
 1
 2
julia> [a for a in Iterators.Stateful([1])]
ERROR: BoundsError: attempt to access 0-element Array{Int64,1} at index [1]

Is this a bug, or is there something I don’t understand here?

kristoffer.carlsson · October 10, 2019, 7:58am

Looking at the code

github.com

JuliaLang/julia/blob/09795cdd2e1d1af291d87122897a580c78352999/base/array.jl#L630-L643


function collect(itr::Generator)
    isz = IteratorSize(itr.iter)
    et = @default_eltype(itr)
    if isa(isz, SizeUnknown)
        return grow_to!(Vector{et}(), itr)
    else
        y = iterate(itr)
        if y === nothing
            return _array_for(et, itr.iter, isz)
        end
        v1, st = y
        collect_to_with_first!(_array_for(typeof(v1), itr.iter, isz), v1, itr, st)
    end
end

The iterate call at

github.com

JuliaLang/julia/blob/09795cdd2e1d1af291d87122897a580c78352999/base/array.jl#L636




_array_for(::Type{T}, itr, ::HasLength) where {T} = Vector{T}(undef, Int(length(itr)::Integer))
_array_for(::Type{T}, itr, ::HasShape{N}) where {T,N} = similar(Array{T,N}, axes(itr))


function collect(itr::Generator)
    isz = IteratorSize(itr.iter)
    et = @default_eltype(itr)
    if isa(isz, SizeUnknown)
        return grow_to!(Vector{et}(), itr)
    else
        y = iterate(itr)
        if y === nothing
            return _array_for(et, itr.iter, isz)
        end
        v1, st = y
        collect_to_with_first!(_array_for(typeof(v1), itr.iter, isz), v1, itr, st)
    end
end


_collect(c, itr, ::EltypeUnknown, isz::SizeUnknown) =
    grow_to!(_similar_for(c, @default_eltype(itr), itr, isz), itr)

makes length(itr) go from 1 to 0.

Then in _array_for(typeof(v1), itr.iter, isz):

github.com

JuliaLang/julia/blob/09795cdd2e1d1af291d87122897a580c78352999/base/array.jl#L627


        return quote
            if $I isa Generator && ($I).f isa Type
                ($I).f
            else
                Any
            end
        end
    end
end


_array_for(::Type{T}, itr, ::HasLength) where {T} = Vector{T}(undef, Int(length(itr)::Integer))
_array_for(::Type{T}, itr, ::HasShape{N}) where {T,N} = similar(Array{T,N}, axes(itr))


function collect(itr::Generator)
    isz = IteratorSize(itr.iter)
    et = @default_eltype(itr)
    if isa(isz, SizeUnknown)
        return grow_to!(Vector{et}(), itr)
    else
        y = iterate(itr)
        if y === nothing

a length 0 vector is allocated which is then tried to get assigned into:

github.com

JuliaLang/julia/blob/09795cdd2e1d1af291d87122897a580c78352999/base/array.jl#L657-L661


function collect_to_with_first!(dest::AbstractArray, v1, itr, st)
    i1 = first(LinearIndices(dest))
    dest[i1] = v1
    return collect_to!(dest, itr, i1+1, st)
end

I’m guessing the axis / length should be computed before the call to iterate.