From ce974255f82289882a44a2f83a355df085724b2c Mon Sep 17 00:00:00 2001 From: MetaIO Maintainers Date: Fri, 4 Sep 2026 21:39:53 -0400 Subject: [PATCH] MetaIO 2026-09-04 (11606dcd) Code extracted from: https://github.com/Kitware/MetaIO.git at commit 11606dcd6b720c4eea7beb1bf2e5a8d90413db47 (master). --- src/metaUtils.cxx | 40 ++++++++- src/metaUtils.h | 9 ++ src/tests/CMakeLists.txt | 1 + .../testMeta15UncompressChunkBoundary.cxx | 84 +++++++++++++++++++ 4 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 src/tests/testMeta15UncompressChunkBoundary.cxx diff --git a/src/metaUtils.cxx b/src/metaUtils.cxx index e37635bf36a..670893a467b 100644 --- a/src/metaUtils.cxx +++ b/src/metaUtils.cxx @@ -57,7 +57,22 @@ bool META_DEBUG = false; static char MET_SeperatorChar = '='; -constexpr static std::streamoff MET_MaxChunkSize = 1024 * 1024 * 1024; +static std::streamoff MET_MaxChunkSize = 1024 * 1024 * 1024; + +void +MET_SetMaxChunkSize(std::streamoff chunkSize) +{ + if (chunkSize > 0) + { + MET_MaxChunkSize = chunkSize; + } +} + +std::streamoff +MET_GetMaxChunkSize() +{ + return MET_MaxChunkSize; +} MET_FieldRecordType * MET_GetFieldRecord(const char * _fieldName, std::vector * _fields) @@ -890,6 +905,29 @@ MET_PerformUncompression(const unsigned char * sourceCompressed, } } while (d_stream.avail_out == 0); } while (err != Z_STREAM_END && err >= 0); + // The output buffer can fill before the trailer arrives in a later input + // chunk; keep feeding input so zlib can reach the CRC and report stream end. + unsigned char trailerScratch[1]; + while (err == Z_BUF_ERROR && dest_pos == uncompressedDataSize) + { + if (d_stream.avail_in == 0) + { + if (source_pos >= sourceCompressedSize) + { + break; + } + d_stream.next_in = const_cast(sourceCompressed + source_pos); + d_stream.avail_in = static_cast(std::min(sourceCompressedSize - source_pos, max_chunk_size)); + source_pos += d_stream.avail_in; + } + d_stream.next_out = trailerScratch; + d_stream.avail_out = 1; + err = inflate(&d_stream, Z_NO_FLUSH); + if (d_stream.avail_out == 0) + { + break; + } + } inflateEnd(&d_stream); if (err != Z_STREAM_END) { diff --git a/src/metaUtils.h b/src/metaUtils.h index 2d2ad50ef20..d8a0e11a6d8 100644 --- a/src/metaUtils.h +++ b/src/metaUtils.h @@ -339,6 +339,15 @@ MET_PerformCompression(const unsigned char * source, std::streamoff * compressedDataSize, int compressionLevel); +// Size of the input and output pieces the (de)compression loops work in. +METAIO_EXPORT +void +MET_SetMaxChunkSize(std::streamoff chunkSize); + +METAIO_EXPORT +std::streamoff +MET_GetMaxChunkSize(); + METAIO_EXPORT bool MET_PerformUncompression(const unsigned char * sourceCompressed, diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index c0f470d1124..164d17f38e1 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -28,3 +28,4 @@ MetaAddTest(testMeta11Form) MetaAddTest(testMeta12Array) MetaAddTest(testMeta13ImageList) MetaAddTest(testMeta14ImageCompressed) +MetaAddTest(testMeta15UncompressChunkBoundary) diff --git a/src/tests/testMeta15UncompressChunkBoundary.cxx b/src/tests/testMeta15UncompressChunkBoundary.cxx new file mode 100644 index 00000000000..5b2d8d4485d --- /dev/null +++ b/src/tests/testMeta15UncompressChunkBoundary.cxx @@ -0,0 +1,84 @@ +#include +#include +#include + +#include + +// A chunk boundary that falls inside the gzip trailer leaves the trailer in +// the next input piece after the output buffer is already full. +static int +TestTrailerInLaterChunk(const std::vector & raw, const std::vector & compressed) +{ + const std::streamoff savedChunkSize = MET_GetMaxChunkSize(); + MET_SetMaxChunkSize(static_cast(compressed.size()) - 4); + + std::vector destination(raw.size(), 0); + const bool accepted = MET_PerformUncompression(compressed.data(), + static_cast(compressed.size()), + destination.data(), + static_cast(raw.size())); + MET_SetMaxChunkSize(savedChunkSize); + + if (!accepted) + { + std::cerr << "FAILED: valid stream rejected when the trailer lands in a later input chunk\n"; + return 1; + } + if (destination != raw) + { + std::cerr << "FAILED: decompressed content does not match the original\n"; + return 1; + } + return 0; +} + +static int +TestCorruptTrailerStillRejected(const std::vector & raw, std::vector compressed) +{ + compressed[compressed.size() - 1] ^= 0xFF; + + std::vector destination(raw.size(), 0); + std::cerr << "--- expect an uncompression failure message below ---\n"; + const bool accepted = MET_PerformUncompression(compressed.data(), + static_cast(compressed.size()), + destination.data(), + static_cast(raw.size())); + if (accepted) + { + std::cerr << "FAILED: stream with a corrupt CRC trailer was accepted\n"; + return 1; + } + return 0; +} + +int +main(int, char *[]) +{ + std::vector raw(64 * 1024); + for (size_t i = 0; i < raw.size(); ++i) + { + raw[i] = static_cast((i * 7 + (i >> 3)) & 0xFF); + } + + std::streamoff compressedSize = 0; + unsigned char * compressedBuffer = + MET_PerformCompression(raw.data(), static_cast(raw.size()), &compressedSize, 6); + if (compressedBuffer == nullptr || compressedSize <= 8) + { + std::cerr << "FAILED: compression did not produce a usable stream\n"; + delete[] compressedBuffer; + return 1; + } + const std::vector compressed(compressedBuffer, compressedBuffer + compressedSize); + delete[] compressedBuffer; + + int result = 0; + result += TestTrailerInLaterChunk(raw, compressed); + result += TestCorruptTrailerStillRejected(raw, compressed); + + if (result == 0) + { + std::cout << "testMeta15UncompressChunkBoundary passed\n"; + } + return result; +}