Stdio tee (copy without redirection)

General Usage
1

Dear all,

I would like to make something like Base.redirect_stdio, but which copies elsewhere the stdout and stderr without suppressing them, like the tee command.

Would you know how to do this ?
Thanks a lot,

2

Please check this similar thread and this.

1 Like
3

Many thanks, but I have this error (private file names redacted):

ERROR: LoadError: MethodError: no method matching (::Base.RedirectStdStream)(::Tee{Tuple{Base.TTY, Base.TTY}})
The function Base.RedirectStdStream(2, true) exists, but no method is defined for this combination of argument types.

Closest candidates are:
(::Base.RedirectStdStream)()
@ Base stream.jl:1293
(::Base.RedirectStdStream)(::Pipe)
@ Base stream.jl:1285
(::Base.RedirectStdStream)(::Base.DevNull)
@ Base stream.jl:1271
…

Stacktrace:
[1] redirect_stdio(; stdin::Nothing, stderr::Tee{Tuple{Base.TTY, Base.TTY}}, stdout::Tee{Tuple{Base.TTY, Base.TTY}})
@ Base ./stream.jl:1355
[2] redirect_stdio(f::var"#logstdio##0#logstdio##1"{var"#11#12"}; stdin::Nothing, stderr::Tee{Tuple{Base.TTY, Base.TTY}}, stdout::Tee{Tuple{Base.TTY, Base.TTY}})
@ Base ./stream.jl:1445
[3] logstdio(f::var"#11#12")
@ Main XXXXXXXXXXXXXXX
[4] top-level scope
@ XXXXXXXXXXXXXX
[5] include(mod::Module, _path::String)
@ Base ./Base.jl:306
[6] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:317
[7] _start()
@ Base ./client.jl:550
in expression starting at XXXXXXXXXXX

I will dig a little by myself and return here in a few hours.

4

Update: it seems the redirect_stdio function does not work with more complex structures than fd integers:

struct RedirectStdStream <: Function
    unix_fd::Int
    writable::Bool
end
for (f, writable, unix_fd) in
        ((:redirect_stdin, false, 0),
         (:redirect_stdout, true, 1),
         (:redirect_stderr, true, 2))
    @eval const ($f) = RedirectStdStream($unix_fd, $writable)
end

function redirect_stdio(;stdin=nothing, stderr=nothing, stdout=nothing)
    stdin  === nothing || redirect_stdin(stdin)
    stderr === nothing || redirect_stderr(stderr)
    stdout === nothing || redirect_stdout(stdout)
end
5

EDIT: I eventually found a workaround.

6

We are very happy for you, and it is likely that the Julia community would be even happier if you were to share the workaround with them.

1 Like
7

The workaround is rather specific to our needs and code, not sure of general applicability.
Anyway, here is a minimal working example made by stripping project specific parts:

#!/usr/bin/env -S julia -t auto

using Logging
logging_io = Base.BufferStream()
global_logger(Base.SimpleLogger(logging_io))

function _capture_pkg_output(f, logging_io)
    # Here, BufferStream don't work, need true file descriptors
    # Short output, no need to multi-thread
    pipe = Pipe()
    redirect_stdio(; stdout=pipe, stderr=pipe) do
        f()
    end
    close(pipe.in)
    while !eof(pipe)
        r = readavailable(pipe)
        write(stdout, r)
        write(logging_io, r)
    end
end

using Pkg
_capture_pkg_output(logging_io) do
    Pkg.activate(@__DIR__)
end

using Base: link_pipe!

function cdshow(f, path)
    @info "cd $f"
    cd(path) do
        f()
    end
end

function mkpathshow(path)
    @info "mkpath $path"
    mkpath(path)
end

function _run_redirect_stdio(command::Cmd)
    stdout_pipe = Pipe()
    link_pipe!(stdout_pipe)
    stderr_pipe = Pipe()
    link_pipe!(stderr_pipe)
    run_th = Threads.@spawn begin
        run(pipeline(command; stdout=stdout_pipe, stderr=stderr_pipe))
        close(stdout_pipe.in)
        close(stderr_pipe.in)
    end
    stdout_th = Threads.@spawn begin
        while !eof(stdout_pipe)
            r = readavailable(stdout_pipe)
            write(stdout, r)
            write(logging_io, r)
        end
    end
    stderr_th = Threads.@spawn begin
        while !eof(stderr_pipe)
            r = readavailable(stderr_pipe)
            write(stderr, r)
            write(logging_io, r)
        end
    end
    waitall([
        run_th,
        stdout_th,
        stderr_th
    ])
end

function runshow(command::Cmd)
    @info command
    return _run_redirect_stdio(command)
end

function readchompshow(command::Cmd)
    @info command
    res = readchomp(command)
    @info res
    return res
end

function catchprocessfailure(f)
    try
        f()
    catch e
        if e isa ProcessFailedException
        else
            rethrow(e)
        end
    end
end

# Results dir
res_dir = joinpath(pwd(), "mwe_results")
mkpathshow(res_dir)
# Infos dir
info_dir = joinpath(res_dir, "info")
mkpathshow(info_dir)
# Start draining log buffer into file
log_path = joinpath(realpath(info_dir), "output.log")
log_th = Threads.@spawn begin
    open(log_path, "w") do f
        @info "Saving logs into $log_path"
        while !eof(logging_io)
            r = readavailable(logging_io)
            write(f, r)
            # Needed to observe the file in real time
            flush(f)
        end
    end
end
# Some test command
runshow(`cat $(expanduser("~/.bashrc"))`)
# Check final dir
cdshow(res_dir) do
    runshow(`ls -alh`)
end
# End of logging thread
closewrite(logging_io)
wait(log_th)
2 Likes