(
std::chrono::system_clock::now().time_since_epoch()
);
double seconds = (double) (finish - start).count() / (double) 1000.0;
std::cout << "STDIN read time (" << chars << " chars): " << seconds << std::endl;
return 0;
}
```
The results:
```julia
$ ./jtest < 1_000_000.txt
STDIN read time (577128362 chars): 0.962
$ ./ctest < 1_000_000.txt
STDIN read time (576128362 chars): 9.689
```
The character difference is the line endings, `getline` doesn’t appear to return the \n.
EDIT: Sorry if my C++ is horrible, I haven’t played in that sandbox in years.
---
### Author:  [@Skoffer](https://discourse.julialang.org/u/Skoffer)
#### Post date: [February 21, 2021, 2:52pm UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/27 "2021-02-21T14:52:26Z")
I think simeonschaub was right, if you want to just count the number of lines you should use `\n` counter. If you want to apply some fast regex logic without allocations, then you should use something like [Automa.jl](https://github.com/BioJulia/Automa.jl). I do not know how perl works, but I guess each time Julia iterator produces the next chunk of data in `eachline`, it implicitly allocates with `String(iochunk)`, so there is an overhead.
I mean you can do fast calculations by using raw bytes, these utilities are not so bad in Julia
```julia
function myreadline(io)
buf = Vector{UInt8}(undef, 1000)
counter = 0
while !eof(io)
nb = readbytes!(io, buf)
for i in 1:nb
if buf[i] == UInt8('\n')
counter += 1
end
# Here you can do something like `String(buf[prev_i:I])` and do regexp magic
end
end
println("Julia: Number of lines: $counter")
end
myreadline(stdin)
```
On my laptop timings are the following
```bash
sh> cat 1m.txt | time ./test.pl
Perl: Number of lines: 100000000
./test.pl 7.84s user 0.34s system 99% cpu 8.240 total
sh> cat 1m.txt | time julia --startup=no test.jl
Julia: Number of lines: 100000000
julia --startup=no xxx.jl 1.65s user 0.64s system 129% cpu 1.770 total
sh> cat 1m.txt | time wc -l
100000000
wc -l 0.93s user 0.21s system 99% cpu 1.137 total
```
Here 0.6s is a startup time of julia itself, so overhead is minimal, compared to `wc -l`.
So, it’s a tradeoff between convenience of work and performance.
Also, Unicode utilities are rather slow, so if you can, it’s better to work with `Vector{Char}`(or maybe `Vector{UInt32}`.
---
### Author:  [@tferic](https://discourse.julialang.org/u/tferic)
#### Post date: [February 21, 2021, 4:10pm UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/28 "2021-02-21T16:10:18Z")
> [@pixel27](#):
>
> For me on a 100 million line file I get:
>
> ```julia
> $ ./ptest < 100_000_000.txt
> STDIN read time (1388888898 chars): 11.8912749290466
>
> $ julia test.jl < 100_000_000.txt
> STDIN read time (1388888898 lines): 12.91
>
> ```
>
> So a difference of 1 second over a runtime of 12 seconds. Perl still runs faster on this small test, no clue where the overhead is. I suspect Julia might be copying bytes out of the buffer when it converts them to a string, and Perl might not be, but that’s just a wild guess.
The Julia version runs faster when this line:
```julia
counter += length(line)
```
is replaced with:
```julia
counter += sizeof(line)
```
That’s why I initially just used a simple counter, so that it wouldn’t matter.
Next thing - invocation.
I learned that piping to STDIN **is not** equivalent to redirecting a file to STDIN.
Sorry, I was not aware of that.
```julia
$ cat test.log | ./testloopspeed2.pl
STDIN read time (4437005389 chars): 28.7752969264984
$ ./testloopspeed2.pl < test.log
STDIN read time (4437005389 chars): 9.6228129863739
$ cat test.log | ./testloopspeed2.jl
STDIN read time (4437005389 chars): 47.381
$ ./testloopspeed2.jl < test.log
STDIN read time (4437005389 chars): 8.878
```
Julia is slightly faster then Perl, when the file is passed as redirection to STDIN.
Julia is significantly slower than Perl, when the file is passed through a pipe to STDIN.
**Edit:**
I would like to add that piping to STDIN still has good use cases.
Example:
```julia
( zcat archive[0-9].log.gz ; cat current.log ) | ./script.jl
```
This is useful in rapid scripting, even if it’s performing slower.
---
### Author:  [@ToucheSir](https://discourse.julialang.org/u/ToucheSir)
#### Post date: [February 21, 2021, 4:33pm UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/29 "2021-02-21T16:33:16Z")
Whilst trying to search for why the pipe case would be much slower, I found [this discourse topic](https://discourse.julialang.org/t/stdin-stdout-10x-slower-than-file-open/17233/11) which lead to [this GH issue](https://github.com/JuliaLang/julia/issues/29960) and ultimately [this](https://github.com/JuliaLang/julia/pull/30638) (failed) PR. Perhaps it’s worth rebooting given stdin/stdout buffering has come up a a number of times in the past year.
---
### Author:  [@Skoffer](https://discourse.julialang.org/u/Skoffer)
#### Post date: [February 21, 2021, 6:52pm UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/30 "2021-02-21T18:52:51Z")
It looks indeed like a buffer issue.
If we consider the following implementation
```julia
import Base: iterate
struct Eachline{T <: IO}
io::T
buf::Vector{UInt8}
end
Eachline(io::IO, n::Integer) = Eachline(io, Vector{UInt8}(undef, n))
Eachline(io::IO) = Eachline(io, 10000)
function iterate(iter::Eachline, state = (1, 0))
iprev, nb = state
(eof(iter.io) & (iprev > nb)) && return nothing
s = ""
while true
if iprev > nb
nb = readbytes!(iter.io, iter.buf)
iprev = 1
end
for i in iprev:nb
if iter.buf[i] == UInt8('\n')
if (iprev == 1) & !isempty(s)
s *= String(iter.buf[1:i])
else
s = String(iter.buf[iprev:i])
end
return (s, (i + 1, nb))
end
end
s *= String(iter.buf[iprev:nb])
eof(iter.io) && return (s, (1, 0))
iprev = nb + 1
end
end
```
Then with the following char counting implementation
```julia
# iterators.jl
function main(io)
counter = 0
for s in Eachline(io)
counter += sizeof(s)
end
println("Julia: Number of chars: $counter")
end
main(stdin)
```
I get the following times
```julia
sh> cat 1m.txt | time julia --startup=no iterators.jl
Julia: Number of chars: 1400000000
julia --startup=no iterators.jl 7.53s user 1.05s system 103% cpu 8.314 total
sh> cat 1m.txt | time ./test2.pl
STDIN read time (1400000000 chars): 9.1769061088562
./test2.pl 8.86s user 0.32s system 99% cpu 9.201 total
```
(here test2.pl is the same as in pixel27 snippet).
Can anyone compare it on other datasets? Because it looks like in this implementation Julia is faster than Perl, despite enormous string allocations. Of course Automa.jl implementation can be less allocating and as a result much much faster.
---
### Author:  [@rikh](https://discourse.julialang.org/u/rikh)
#### Post date: [February 21, 2021, 7:53pm UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/31 "2021-02-21T19:53:31Z")
Thanks to your code, I got reminded of a relevant method in Julia base:
```julia
julia> countlines("file.txt")
1000
```
---
### Author:  [@tim.holy](https://discourse.julialang.org/u/tim.holy)
#### Post date: [February 21, 2021, 10:39pm UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/32 "2021-02-21T22:39:54Z")
If you like to live dangerously, you can get a performance boost by turning off locking:
```julia
julia> function iterlines(io)
counter = 0
for line in eachline(io)
counter += 1
end
return counter
end
iterlines (generic function with 1 method)
julia> precompile(iterlines, (IOStream,))
true
julia> open("/tmp/textfile.txt") do io
@time iterlines(io)
end
0.195130 seconds (1.01 M allocations: 107.697 MiB, 12.18% gc time)
1000000
julia> open("/tmp/textfile.txt") do io
io._dolock = false # turn off locking
@time iterlines(io)
end
0.086630 seconds (1.01 M allocations: 107.697 MiB, 26.46% gc time)
1000000
```
If you turn off locking, don’t even think about threading.
---
### Author:  [@tferic](https://discourse.julialang.org/u/tferic)
#### Post date: [February 22, 2021, 10:51am UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/33 "2021-02-22T10:51:23Z")
> [@tim.holy](#):
>
> If you like to live dangerously, you can get a performance boost by turning off locking:
What kind of risk would you expect when reading from STDIN?
I would like to mention that I am working with logfiles, where a server process is holding a write filehandle on it. But I don’t see a conflict there.
---
### Author:  [@tferic](https://discourse.julialang.org/u/tferic)
#### Post date: [February 22, 2021, 11:25pm UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/34 "2021-02-22T23:25:11Z")
Hello @Skoffer
Just curious, did you try to write a more efficient version of `eachline()` with your code?
I used your code in the following tests (`testloopspeed3.jl`):
```julia
$ time (cat test.log | ./testloopspeed2.pl)
STDIN read time (4437005389 chars): 30.2816960811615
real 0m30.326s
user 0m19.187s
sys 0m16.329s
$ time (cat test.log | julia --startup=no testloopspeed3.jl)
Julia: Number of chars: 4437005389
real 0m40.995s
user 0m23.752s
sys 0m17.695s
```
I think your version is significantly faster than `eachline()`, but still 25% slower than Perl in my test.
The flag `--startup=no` does not give significant advantage.
---
### Author:  [@Henrique\_Becker](https://discourse.julialang.org/u/Henrique_Becker)
#### Post date: [February 23, 2021, 12:13am UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/35 "2021-02-23T00:13:30Z")
> [@tferic](#):
>
> The flag `--startup=no` does not give significant advantage.
The flag just disables any startup files that you have configured yourself, if you have never configured any, then it has no effect.
---
### Author:  [@tferic](https://discourse.julialang.org/u/tferic)
#### Post date: [March 3, 2021, 8:51am UTC](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701/36 "2021-03-03T08:51:18Z")
As @ToucheSir pointed out, there is a similar issue in a related thread:
> [@Slow gzip streaming in julia but not in python](https://discourse.julialang.org/t/slow-gzip-streaming-in-julia-but-not-in-python/56346):
>
> Hi there, I am new to Julia, and come to Julia for performance with fewer codes. I have been using R, and encountered a simple task: read from a gziped file and do something to each line. It was slow in R, of course. So I went to Perl first. Here is the code: perl -e ' open( DATA, "zcat ~/in2/swxx/sj/db/ncbi/gene/gene2accession.gz|"); while( \) { @F = split( /\t/); } close( DATA);' I found the gzip command was using about 48% CPU of a thread. Then I came to Julia, and here is the code: …
[Previous page](https://discourse.julialang.org/t/bad-performance-of-eachline-on-stdin/55701.md?page=1)