Postprocessing fails with BLOSC error

[felixcremer]

When I am trying to run the postprocessing I get a BLOSC decompression error but I am not sure whether this comes from having not enough disk space.

[danlooo]

Maybe related to an issue @dpabon has right now?

[dpabon]

hard to tell as I'm not familiar with the postprocessing script. From my side, the error is triggered when I save a zarr dataset and then try to open the file in python with the latest zarr version. At the end I exported the dataset as netcdf in julia and it works without problem in python

[felixcremer]

This seems to be unrelated to the missing disk space since other tiles are saved without problems.
This is only triggered be the european tile N048E027.

This is the error I get:

ERROR: LoadError: Blosc decompress error, output buffer is not large enough
Stacktrace:
  [1] error(s::String)
    @ Base ./error.jl:35
  [2] decompress!(dest::Vector{Float32}, src::Vector{UInt8})
    @ Blosc ~/.julia/packages/Blosc/jk4Np/src/Blosc.jl:189
  [3] zuncompress!
    @ ~/.julia/packages/Zarr/3QSdj/src/Compressors.jl:68 [inlined]
  [4] zuncompress!
    @ ~/.julia/packages/Zarr/3QSdj/src/Compressors.jl:14 [inlined]
  [5] uncompress_raw!(a::Matrix{…}, z::ZArray{…}, curchunk::Vector{…})
    @ Zarr ~/.julia/packages/Zarr/3QSdj/src/ZArray.jl:261
  [6] uncompress_to_output!(aout::Matrix{…}, output_base_offsets::Tuple{…}, z::ZArray{…}, chunk_compressed::Vector{…}, current_chunk_offsets::Tuple{…}, a::Matrix{…}, indranges::Tuple{…})
    @ Zarr ~/.julia/packages/Zarr/3QSdj/src/ZArray.jl:270
  [7] readblock!(aout::Matrix{…}, z::ZArray{…}, r::CartesianIndices{…})
    @ Zarr ~/.julia/packages/Zarr/3QSdj/src/ZArray.jl:178
  [8] readblock!
    @ ~/.julia/packages/Zarr/3QSdj/src/ZArray.jl:247 [inlined]
  [9] readblock_checked!
    @ /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:290 [inlined]
 [10] getindex_disk_nobatch!(out::Nothing, a::ZArray{…}, i::Tuple{…})
    @ DiskArrays /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:102
 [11] getindex_disk!
    @ /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:65 [inlined]
 [12] getindex_disk
    @ /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:58 [inlined]
 [13] getindex
    @ /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:308 [inlined]
 [14] readblock!
    @ /mnt/felix1/worldmap/dev/DiskArrayTools/src/DiskArrayTools.jl:266 [inlined]
 [15] #4
    @ /mnt/felix1/worldmap/dev/DiskArrayTools/src/DiskArrayTools.jl:39 [inlined]
 [16] foreach(f::DiskArrayTools.var"#4#6"{DiskArrayStack{…}, Array{…}, Tuple{…}, Tuple{…}}, itr::Base.Iterators.Zip{Tuple{…}})
    @ Base ./abstractarray.jl:3187
 [17] readblock!(::DiskArrayStack{…}, ::Array{…}, ::UnitRange{…}, ::UnitRange{…}, ::UnitRange{…})
    @ DiskArrayTools /mnt/felix1/worldmap/dev/DiskArrayTools/src/DiskArrayTools.jl:36
 [18] readblock_checked!(::DiskArrayStack{…}, ::Array{…}, ::UnitRange{…}, ::Vararg{…})
    @ DiskArrays /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:290
 [19] getindex_disk_nobatch!(out::Nothing, a::DiskArrayStack{…}, i::Tuple{…})
    @ DiskArrays /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:102
 [20] getindex_disk!
    @ /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:65 [inlined]
 [21] getindex_disk
    @ /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:58 [inlined]
 [22] getindex
    @ /mnt/felix1/worldmap/dev/DiskArrays/src/indexing.jl:308 [inlined]
 [23] read_range(r::Tuple{…}, ia::DiskArrayEngine.InputArray{…}, buffer::Array{…})
    @ DiskArrayEngine /mnt/felix1/worldmap/dev/DiskArrayEngine/src/buffers.jl:130
 [24] _broadcast_getindex_evalf(::typeof(DiskArrayEngine.read_range), ::Tuple{…}, ::DiskArrayEngine.InputArray{…}, ::Array{…})
    @ Base.Broadcast ./broadcast.jl:678
 [25] _broadcast_getindex(bc::Base.Broadcast.Broadcasted{…}, I::Int64)
    @ Base.Broadcast ./broadcast.jl:651
 [26] #17
    @ ./broadcast.jl:1102 [inlined]
 [27] ntuple
    @ ./ntuple.jl:48 [inlined]
 [28] copy(bc::Base.Broadcast.Broadcasted{Base.Broadcast.Style{…}, Nothing, typeof(DiskArrayEngine.read_range), Tuple{…}})
    @ Base.Broadcast ./broadcast.jl:1102
 [29] materialize(bc::Base.Broadcast.Broadcasted{…})
    @ Base.Broadcast ./broadcast.jl:872
 [30] default_loopbody(inow::Tuple{…}, op::DiskArrayEngine.GMDWop{…}, inbuffers_pure::Tuple{…}, outbuffers::Tuple{…}, threaded::Bool, outars::Tuple{…}, cb::Tuple{…}, runfilter::Tuple{})
    @ DiskArrayEngine /mnt/felix1/worldmap/dev/DiskArrayEngine/src/runner.jl:174
 [31] run_loop(::DiskArrayEngine.LocalRunner, op::DiskArrayEngine.GMDWop{…}, inbuffers_pure::Tuple{…}, outbuffers::Tuple{…}, threaded::Bool, outars::Tuple{…}, loopranges::DiskArrayEngine.ProductArray{…}, cb::Tuple{…}, runfilter::Tuple{}; groupspecs::Tuple{…})
    @ DiskArrayEngine /mnt/felix1/worldmap/dev/DiskArrayEngine/src/runner.jl:185
 [32] run_loop(runner::DiskArrayEngine.LocalRunner, loopranges::DiskArrayEngine.ProductArray{…}; groupspecs::Tuple{…})
    @ DiskArrayEngine /mnt/felix1/worldmap/dev/DiskArrayEngine/src/runner.jl:166
 [33] run_loop
    @ /mnt/felix1/worldmap/dev/DiskArrayEngine/src/runner.jl:165 [inlined]
 [34] run_group(::DiskArrayEngine.DiskEngineScheduler{…}, ::Tuple{…})
    @ DiskArrayEngine /mnt/felix1/worldmap/dev/DiskArrayEngine/src/distribute.jl:127
 [35] run_group(::DiskArrayEngine.DiskEngineScheduler{…}, ::Nothing)
    @ DiskArrayEngine /mnt/felix1/worldmap/dev/DiskArrayEngine/src/distribute.jl:145
 [36] run(runner::DiskArrayEngine.LocalRunner)
    @ DiskArrayEngine /mnt/felix1/worldmap/dev/DiskArrayEngine/src/distribute.jl:153
 [37] postprocess(a::YAXArray{…}, target_array::YAXArray{…}, orbitcombine::Function; minsize::Int64, max_cache::Float64)
    @ RQADeforestation /mnt/felix1/worldmap/dev/RQADeforestation/src/cluster.jl:50
 [38] postprocess
    @ /mnt/felix1/worldmap/dev/RQADeforestation/src/cluster.jl:38 [inlined]
 [39] postprocess(a::YAXArray{…}, target_array::YAXArray{…})
    @ RQADeforestation /mnt/felix1/worldmap/dev/RQADeforestation/src/cluster.jl:38
 [40] macro expansion
    @ ./timing.jl:581 [inlined]
 [41] postprocess(tile::SubString{String}, indir::String, outdir::String)
    @ RQADeforestation /mnt/felix1/worldmap/dev/RQADeforestation/src/cluster.jl:68
 [42] macro expansion
    @ ./timing.jl:581 [inlined]
 [43] top-level scope
    @ /mnt/felix1/worldmap/scripts/cluster_europe.jl:19
in expression starting at /mnt/felix1/worldmap/scripts/cluster_europe.jl:17
Some type information was truncated. Use `show(err)` to see complete types.