Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 39 additions & 1 deletion Modules/ThirdParty/MetaIO/src/MetaIO/src/metaUtils.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,22 @@ bool META_DEBUG = false;

static char MET_SeperatorChar = '=';

constexpr static std::streamoff MET_MaxChunkSize = 1024 * 1024 * 1024;
static std::streamoff MET_MaxChunkSize = 1024 * 1024 * 1024;

void
MET_SetMaxChunkSize(std::streamoff chunkSize)
{
if (chunkSize > 0)
{
MET_MaxChunkSize = chunkSize;
Comment thread
hjmjohnson marked this conversation as resolved.
}
Comment thread
hjmjohnson marked this conversation as resolved.
}

std::streamoff
MET_GetMaxChunkSize()
{
return MET_MaxChunkSize;
}

MET_FieldRecordType *
MET_GetFieldRecord(const char * _fieldName, std::vector<MET_FieldRecordType *> * _fields)
Expand Down Expand Up @@ -890,6 +905,29 @@ MET_PerformUncompression(const unsigned char * sourceCompressed,
}
} while (d_stream.avail_out == 0);
} while (err != Z_STREAM_END && err >= 0);
// The output buffer can fill before the trailer arrives in a later input
// chunk; keep feeding input so zlib can reach the CRC and report stream end.
Comment thread
hjmjohnson marked this conversation as resolved.
unsigned char trailerScratch[1];
while (err == Z_BUF_ERROR && dest_pos == uncompressedDataSize)
{
if (d_stream.avail_in == 0)
{
if (source_pos >= sourceCompressedSize)
{
break;
}
d_stream.next_in = const_cast<unsigned char *>(sourceCompressed + source_pos);
d_stream.avail_in = static_cast<uInt>(std::min(sourceCompressedSize - source_pos, max_chunk_size));
source_pos += d_stream.avail_in;
}
d_stream.next_out = trailerScratch;
d_stream.avail_out = 1;
err = inflate(&d_stream, Z_NO_FLUSH);
if (d_stream.avail_out == 0)
{
break;
}
}
inflateEnd(&d_stream);
if (err != Z_STREAM_END)
{
Expand Down
9 changes: 9 additions & 0 deletions Modules/ThirdParty/MetaIO/src/MetaIO/src/metaUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,15 @@ MET_PerformCompression(const unsigned char * source,
std::streamoff * compressedDataSize,
int compressionLevel);

// Size of the input and output pieces the (de)compression loops work in.
METAIO_EXPORT
void
MET_SetMaxChunkSize(std::streamoff chunkSize);

METAIO_EXPORT
std::streamoff
MET_GetMaxChunkSize();

METAIO_EXPORT
bool
MET_PerformUncompression(const unsigned char * sourceCompressed,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,4 @@ MetaAddTest(testMeta11Form)
MetaAddTest(testMeta12Array)
MetaAddTest(testMeta13ImageList)
MetaAddTest(testMeta14ImageCompressed)
MetaAddTest(testMeta15UncompressChunkBoundary)
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#include <cstring>
#include <iostream>
#include <vector>

#include <metaUtils.h>

// A chunk boundary that falls inside the gzip trailer leaves the trailer in
// the next input piece after the output buffer is already full.
static int
TestTrailerInLaterChunk(const std::vector<unsigned char> & raw, const std::vector<unsigned char> & compressed)
{
const std::streamoff savedChunkSize = MET_GetMaxChunkSize();
MET_SetMaxChunkSize(static_cast<std::streamoff>(compressed.size()) - 4);

std::vector<unsigned char> destination(raw.size(), 0);
const bool accepted = MET_PerformUncompression(compressed.data(),
static_cast<std::streamoff>(compressed.size()),
destination.data(),
static_cast<std::streamoff>(raw.size()));
MET_SetMaxChunkSize(savedChunkSize);

if (!accepted)
{
std::cerr << "FAILED: valid stream rejected when the trailer lands in a later input chunk\n";
return 1;
}
if (destination != raw)
{
std::cerr << "FAILED: decompressed content does not match the original\n";
return 1;
}
return 0;
}

static int
TestCorruptTrailerStillRejected(const std::vector<unsigned char> & raw, std::vector<unsigned char> compressed)
{
compressed[compressed.size() - 1] ^= 0xFF;
Comment thread
hjmjohnson marked this conversation as resolved.

std::vector<unsigned char> destination(raw.size(), 0);
std::cerr << "--- expect an uncompression failure message below ---\n";
const bool accepted = MET_PerformUncompression(compressed.data(),
static_cast<std::streamoff>(compressed.size()),
destination.data(),
static_cast<std::streamoff>(raw.size()));
if (accepted)
{
std::cerr << "FAILED: stream with a corrupt CRC trailer was accepted\n";
return 1;
}
return 0;
}

int
main(int, char *[])
{
std::vector<unsigned char> raw(64 * 1024);
for (size_t i = 0; i < raw.size(); ++i)
{
raw[i] = static_cast<unsigned char>((i * 7 + (i >> 3)) & 0xFF);
}

std::streamoff compressedSize = 0;
unsigned char * compressedBuffer =
MET_PerformCompression(raw.data(), static_cast<std::streamoff>(raw.size()), &compressedSize, 6);
if (compressedBuffer == nullptr || compressedSize <= 8)
{
std::cerr << "FAILED: compression did not produce a usable stream\n";
delete[] compressedBuffer;
return 1;
}
const std::vector<unsigned char> compressed(compressedBuffer, compressedBuffer + compressedSize);
delete[] compressedBuffer;

int result = 0;
result += TestTrailerInLaterChunk(raw, compressed);
result += TestCorruptTrailerStillRejected(raw, compressed);

if (result == 0)
{
std::cout << "testMeta15UncompressChunkBoundary passed\n";
}
return result;
}
Loading