`Literate.markdown` not thread-safe?

[giordano]

using Literate

@sync for script_file in ("one.jl", "two.jl", "three.jl")
    Threads.@spawn :interactive begin
        script_path = joinpath(mktempdir(), script_file)
        write(script_path, """
# # Rational numbers
#
# In julia rational numbers can be constructed with the `//` operator.
# Lets define two rational numbers, `x` and `y`:

## Define variable x and y
x = 1//3
y = 2//5

# When adding `x` and `y` together we obtain a new rational number:

z = x + y
""")
        Literate.markdown(script_path, mktempdir();
                          flavor = Literate.DocumenterFlavor(),
                          execute = true)
    end
end

Run with

julia --project -t 1,4 script.jl

This fails in multiple different ways, including

ERROR: LoadError: TaskFailedException

    nested task error: BoundsError: attempt to access 2-element Vector{AbstractDisplay} at index [3]
    Stacktrace:
      [1] throw_boundserror(A::Vector{AbstractDisplay}, I::Tuple{Int64})
        @ Base ./essentials.jl:13
      [2] checkbounds
        @ ./essentials.jl:385 [inlined]
      [3] getindex
        @ ./essentials.jl:964 [inlined]
      [4] splice!(a::Vector{AbstractDisplay}, i::Int64, ins::Vector{Any})
        @ Base ./array.jl:2040
      [5] splice!
        @ ./array.jl:2040 [inlined]
      [6] popdisplay(d::Literate.LiterateDisplay)
        @ Base.Multimedia ./multimedia.jl:297
      [7] execute_block(sb::Module, block::String; inputfile::String, fake_source::String, softscope::Bool, continue_on_error::Bool)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:963
      [8] execute_block
        @ ~/.julia/packages/Literate/On3EJ/src/Literate.jl:935 [inlined]
      [9] execute_markdown!(io::IOBuffer, sb::Module, block::String, outputdir::String; inputfile::String, fake_source::String, flavor::Literate.DocumenterFlavor, image_formats::Vector{Tuple{MIME, String}}, file_prefix::String, softscope::Bool, continue_on_error::Bool)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:683
     [10] kwcall(::@NamedTuple{inputfile::String, fake_source::String, flavor::Literate.DocumenterFlavor, image_formats::Vector{Tuple{MIME, String}}, file_prefix::String, softscope::Bool, continue_on_error::Bool}, ::typeof(Literate.execute_markdown!), io::IOBuffer, sb::Module, block::String, outputdir::String)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:676
     [11] (::Literate.var"#42#43"{Dict{String, Any}, IOBuffer, Module, Literate.CodeChunk, Int64})()
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:651
     [12] cd(f::Literate.var"#42#43"{Dict{String, Any}, IOBuffer, Module, Literate.CodeChunk, Int64}, dir::String)
        @ Base.Filesystem ./file.jl:112
     [13] markdown(inputfile::String, outputdir::String; config::Dict{Any, Any}, kwargs::@Kwargs{flavor::Literate.DocumenterFlavor, execute::Bool})
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:650
     [14] markdown
        @ ~/.julia/packages/Literate/On3EJ/src/Literate.jl:614 [inlined]

and

ERROR: LoadError: TaskFailedException

    nested task error: KeyError: key Literate.LiterateDisplay(Any[]) not found
    Stacktrace:
      [1] popdisplay(d::Literate.LiterateDisplay)
        @ Base.Multimedia ./multimedia.jl:300
      [2] execute_block(sb::Module, block::String; inputfile::String, fake_source::String, softscope::Bool, continue_on_error::Bool)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:963
      [3] execute_block
        @ ~/.julia/packages/Literate/On3EJ/src/Literate.jl:935 [inlined]
      [4] execute_markdown!(io::IOBuffer, sb::Module, block::String, outputdir::String; inputfile::String, fake_source::String, flavor::Literate.DocumenterFlavor, image_formats::Vector{Tuple{MIME, String}}, file_prefix::String, softscope::Bool, continue_on_error::Bool)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:683
      [5] kwcall(::@NamedTuple{inputfile::String, fake_source::String, flavor::Literate.DocumenterFlavor, image_formats::Vector{Tuple{MIME, String}}, file_prefix::String, softscope::Bool, continue_on_error::Bool}, ::typeof(Literate.execute_markdown!), io::IOBuffer, sb::Module, block::String, outputdir::String)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:676
      [6] (::Literate.var"#42#43"{Dict{String, Any}, IOBuffer, Module, Literate.CodeChunk, Int64})()
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:651
      [7] cd(f::Literate.var"#42#43"{Dict{String, Any}, IOBuffer, Module, Literate.CodeChunk, Int64}, dir::String)
        @ Base.Filesystem ./file.jl:112
      [8] markdown(inputfile::String, outputdir::String; config::Dict{Any, Any}, kwargs::@Kwargs{flavor::Literate.DocumenterFlavor, execute::Bool})
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:650
      [9] markdown
        @ ~/.julia/packages/Literate/On3EJ/src/Literate.jl:614 [inlined]
     [10] (::var"#2#3"{String})()
        @ Main /tmp/foo.jl:20

...and 1 more exception.

and

ERROR: LoadError: TaskFailedException

    nested task error: UndefRefError: access to undefined reference
    Stacktrace:
      [1] getindex
        @ ./essentials.jl:965 [inlined]
      [2] popdisplay(d::Literate.LiterateDisplay)
        @ Base.Multimedia ./multimedia.jl:296
      [3] execute_block(sb::Module, block::String; inputfile::String, fake_source::String, softscope::Bool, continue_on_error::Bool)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:963
      [4] execute_block
        @ ~/.julia/packages/Literate/On3EJ/src/Literate.jl:935 [inlined]
      [5] execute_markdown!(io::IOBuffer, sb::Module, block::String, outputdir::String; inputfile::String, fake_source::String, flavor::Literate.DocumenterFlavor, image_formats::Vector{Tuple{MIME, String}}, file_prefix::String, softscope::Bool, continue_on_error::Bool)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:683
      [6] kwcall(::@NamedTuple{inputfile::String, fake_source::String, flavor::Literate.DocumenterFlavor, image_formats::Vector{Tuple{MIME, String}}, file_prefix::String, softscope::Bool, continue_on_error::Bool}, ::typeof(Literate.execute_markdown!), io::IOBuffer, sb::Module, block::String, outputdir::String)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:676
      [7] (::Literate.var"#42#43"{Dict{String, Any}, IOBuffer, Module, Literate.CodeChunk, Int64})()
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:651
      [8] cd(f::Literate.var"#42#43"{Dict{String, Any}, IOBuffer, Module, Literate.CodeChunk, Int64}, dir::String)
        @ Base.Filesystem ./file.jl:112
      [9] markdown(inputfile::String, outputdir::String; config::Dict{Any, Any}, kwargs::@Kwargs{flavor::Literate.DocumenterFlavor, execute::Bool})
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:650
     [10] markdown
        @ ~/.julia/packages/Literate/On3EJ/src/Literate.jl:614 [inlined]
     [11] (::var"#2#3"{String})()
        @ Main /tmp/foo.jl:20

and

ERROR: LoadError: TaskFailedException

    nested task error: SystemError: dup: Bad file descriptor
    Stacktrace:
      [1] systemerror(p::String, errno::Int32; extrainfo::Nothing)
        @ Base ./error.jl:186
      [2] systemerror
        @ ./error.jl:185 [inlined]
      [3] dup
        @ ./libc.jl:47 [inlined]
      [4] _redirect_io_libc
        @ ./stream.jl:1248 [inlined]
      [5] (::Base.RedirectStdStream)(handle::Base.PipeEndpoint)
        @ Base ./stream.jl:1266
      [6] RedirectStdStream
        @ ./stream.jl:1282 [inlined]
      [7] (::IOCapture.var"#12#13"{Type{Union{}}, Literate.var"#59#60"{String, Bool, Module, String}, IOContext{Base.PipeEndpoint}, IOContext{Base.PipeEndpoint}, IOContext{Base.PipeEndpoint}, IOContext{Base.PipeEndpoint}})()
        @ IOCapture ~/.julia/packages/IOCapture/MR051/src/IOCapture.jl:177
      [8] with_logstate(f::IOCapture.var"#12#13"{Type{Union{}}, Literate.var"#59#60"{String, Bool, Module, String}, IOContext{Base.PipeEndpoint}, IOContext{Base.PipeEndpoint}, IOContext{Base.PipeEndpoint}, IOContext{Base.PipeEndpoint}}, logstate::Base.CoreLogging.LogState)
        @ Base.CoreLogging ./logging/logging.jl:540
      [9] with_logger(f::Function, logger::Base.CoreLogging.ConsoleLogger)
        @ Base.CoreLogging ./logging/logging.jl:651
     [10] capture(f::Literate.var"#59#60"{String, Bool, Module, String}; rethrow::Type, color::Bool, passthrough::Bool, capture_buffer::IOBuffer, io_context::Vector{Any})
        @ IOCapture ~/.julia/packages/IOCapture/MR051/src/IOCapture.jl:167
     [11] capture
        @ ~/.julia/packages/IOCapture/MR051/src/IOCapture.jl:100 [inlined]
     [12] execute_block(sb::Module, block::String; inputfile::String, fake_source::String, softscope::Bool, continue_on_error::Bool)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:953
     [13] execute_block
        @ ~/.julia/packages/Literate/On3EJ/src/Literate.jl:935 [inlined]
     [14] execute_markdown!(io::IOBuffer, sb::Module, block::String, outputdir::String; inputfile::String, fake_source::String, flavor::Literate.DocumenterFlavor, image_formats::Vector{Tuple{MIME, String}}, file_prefix::String, softscope::Bool, continue_on_error::Bool)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:683
     [15] kwcall(::@NamedTuple{inputfile::String, fake_source::String, flavor::Literate.DocumenterFlavor, image_formats::Vector{Tuple{MIME, String}}, file_prefix::String, softscope::Bool, continue_on_error::Bool}, ::typeof(Literate.execute_markdown!), io::IOBuffer, sb::Module, block::String, outputdir::String)
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:676
     [16] (::Literate.var"#42#43"{Dict{String, Any}, IOBuffer, Module, Literate.CodeChunk, Int64})()
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:651
     [17] cd(f::Literate.var"#42#43"{Dict{String, Any}, IOBuffer, Module, Literate.CodeChunk, Int64}, dir::String)
        @ Base.Filesystem ./file.jl:112
     [18] markdown(inputfile::String, outputdir::String; config::Dict{Any, Any}, kwargs::@Kwargs{flavor::Literate.DocumenterFlavor, execute::Bool})
        @ Literate ~/.julia/packages/Literate/On3EJ/src/Literate.jl:650
     [19] markdown
        @ ~/.julia/packages/Literate/On3EJ/src/Literate.jl:614 [inlined]
     [20] (::var"#2#3"{String})()
        @ Main /tmp/foo.jl:20

Is there a global state?

[giordano]

Is there a global state?

A couple of the errors above point to

Literate.jl/src/Literate.jl

Line 963 in 17e8ef5

popdisplay(disp) # IOCapture.capture has a try-catch so should always end up here

which brought me to https://github.com/JuliaLang/julia/blob/8fd607a7df9e65ed060c88f32d801c95aa8a573f/base/multimedia.jl#L272-L301. This stuff is definitely dealing with a global state, sigh.