From e027e092917c6f64877856920cda2d31d493442f Mon Sep 17 00:00:00 2001 From: Beforerr Date: Thu, 11 Jun 2026 09:04:04 -0700 Subject: [PATCH] feat: add close(::CDFDataset) There was no way to release the mmap (which locks the file on Windows) before GC. close finalizes the backing map; compressed files already live in memory, and their original mmap is now unmapped eagerly right after decompression instead of lingering until GC. Co-Authored-By: Claude Fable 5 --- src/dataset.jl | 4 ++++ src/decompress.jl | 2 +- test/runtests.jl | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/dataset.jl b/src/dataset.jl index a48c0de..823a38f 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -39,13 +39,17 @@ end function _load_dataset(fname, buffer, ::Type{FieldSizeType}) where {FieldSizeType} compression = NoCompression if is_compressed(read_be(buffer, 5, UInt32)) + mmapped = buffer buffer, compression = decompress_bytes(buffer, FieldSizeType) + finalize(mmapped) end cdr = CDR(buffer, 8, FieldSizeType) gdr = GDR(buffer, Int(cdr.gdr_offset), FieldSizeType) return CDFDataset{FieldSizeType}(fname, cdr, gdr, buffer, compression) end +Base.close(cdf::CDFDataset) = (finalize(parent(cdf)); nothing) + is_big_endian_encoding(cdf::CDFDataset) = is_big_endian_encoding(cdf.cdr.encoding) is_compressed(magic_numbers::UInt32) = magic_numbers != 0x0000FFFF diff --git a/src/decompress.jl b/src/decompress.jl index 757efed..20d3a9e 100644 --- a/src/decompress.jl +++ b/src/decompress.jl @@ -49,7 +49,7 @@ function decompress_bytes!(decompressor, dest, doffs, src::AbstractVector{UInt8} n_out = N * sizeof(eltype(dest)) GC.@preserve dest src begin out = _unsafe_gzip_decompress!(decompressor, pointer(dest, doffs), n_out, pointer(src, soffs), n_in) - out isa LibDeflateError && throw(ArgumentError("gzip decompression failed: $out")) + out isa LibDeflateError && throw(ArgumentError("gzip decompression failed")) end elseif compression == RLECompression n_out = N * sizeof(eltype(dest)) diff --git a/test/runtests.jl b/test/runtests.jl index 097ba19..92c5d67 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -38,6 +38,16 @@ end @test string(Epoch(-1.0e31)) == "FILLVAL" end +@testset "close" begin + ds = CDFDataset(data_path("a_cdf.cdf")) + v = ds["var"][:] + @test isnothing(close(ds)) + cds = CDFDataset(data_path("a_compressed_cdf.cdf")) + @test cds["var"][:] == v # compressed data lives in memory; close is a no-op + @test isnothing(close(cds)) + @test cds["var"][:] == v +end + @testset "Uncompressed cdf file" begin file = data_path("a_cdf.cdf") ds = CDFDataset(file)