From 8894756e7903b329dbdbf3d4cddd94edcb613dfe Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:30:42 +0000 Subject: [PATCH 1/4] Limit zip file entries to 100MB by default Ports sillsdev/machine#481. Co-Authored-By: Claude Sonnet 5.5 Co-authored-by: Eli C. Lowry <83078660+Enkidu93@users.noreply.github.com> --- machine/corpora/dbl_bundle_text_corpus.py | 5 ++- machine/corpora/zip_entry_stream_container.py | 4 +- .../zip_paratext_project_file_handler.py | 4 +- machine/utils/bounded_stream.py | 33 ++++++++++++++ machine/utils/zip_entry_utils.py | 30 +++++++++++++ tests/utils/test_zip_entry_utils.py | 45 +++++++++++++++++++ 6 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 machine/utils/bounded_stream.py create mode 100644 machine/utils/zip_entry_utils.py create mode 100644 tests/utils/test_zip_entry_utils.py diff --git a/machine/corpora/dbl_bundle_text_corpus.py b/machine/corpora/dbl_bundle_text_corpus.py index 33aa62cd..87cc3ae3 100644 --- a/machine/corpora/dbl_bundle_text_corpus.py +++ b/machine/corpora/dbl_bundle_text_corpus.py @@ -7,6 +7,7 @@ from ..scripture import ENGLISH_VERSIFICATION from ..scripture.verse_ref import Versification from ..utils.typeshed import StrPath +from ..utils.zip_entry_utils import open_bounded_stream from .scripture_text_corpus import ScriptureTextCorpus from .usx_zip_text import UsxZipText @@ -16,7 +17,7 @@ class DblBundleTextCorpus(ScriptureTextCorpus): def __init__(self, filename: StrPath) -> None: with ZipFile(filename, "r") as archive: - with archive.open("metadata.xml", "r") as stream: + with open_bounded_stream(archive, "metadata.xml") as stream: doc = ElementTree.parse(stream) version = doc.getroot().get("version", "2.0") parts = version.split(".", maxsplit=3) @@ -27,7 +28,7 @@ def __init__(self, filename: StrPath) -> None: (zi for zi in archive.filelist if os.path.basename(zi.filename) == "versification.vrs"), None ) if versification_entry is not None: - with archive.open(versification_entry, "r") as stream: + with open_bounded_stream(archive, versification_entry) as stream: abbr = doc.getroot().findtext("./identification/abbreviation", "") versification = Versification.parse( TextIOWrapper(stream, encoding="utf-8-sig"), "versification.vrs", fallback_name=abbr diff --git a/machine/corpora/zip_entry_stream_container.py b/machine/corpora/zip_entry_stream_container.py index a7f40a70..f719a95d 100644 --- a/machine/corpora/zip_entry_stream_container.py +++ b/machine/corpora/zip_entry_stream_container.py @@ -5,6 +5,7 @@ from zipfile import ZipFile from ..utils.typeshed import StrPath +from ..utils.zip_entry_utils import open_bounded_stream from .stream_container import StreamContainer @@ -20,7 +21,8 @@ def __exit__(self, type: Any, value: Any, traceback: Any) -> None: self.close() def open_stream(self) -> BinaryIO: - return BytesIO(self._archive.read(self._entry)) + with open_bounded_stream(self._archive, self._entry) as stream: + return BytesIO(stream.read()) def close(self) -> None: self._archive.close() diff --git a/machine/corpora/zip_paratext_project_file_handler.py b/machine/corpora/zip_paratext_project_file_handler.py index 3f430048..ca075c97 100644 --- a/machine/corpora/zip_paratext_project_file_handler.py +++ b/machine/corpora/zip_paratext_project_file_handler.py @@ -4,6 +4,7 @@ from typing import BinaryIO, Optional, cast from zipfile import ZipFile +from ..utils.zip_entry_utils import open_bounded_stream from .paratext_project_file_handler import ParatextProjectFileHandler from .usfm_stylesheet import UsfmStylesheet @@ -27,7 +28,8 @@ def find(self, extension: str) -> Optional[str]: def open(self, file_name: str) -> Optional[BinaryIO]: for actual_entry_name in self._archive.namelist(): if actual_entry_name.lower() == file_name.lower(): - return BytesIO(self._archive.read(actual_entry_name)) + with open_bounded_stream(self._archive, actual_entry_name) as stream: + return BytesIO(stream.read()) return None def create_stylesheet(self, file_name: str) -> UsfmStylesheet: diff --git a/machine/utils/bounded_stream.py b/machine/utils/bounded_stream.py new file mode 100644 index 00000000..d673f578 --- /dev/null +++ b/machine/utils/bounded_stream.py @@ -0,0 +1,33 @@ +from io import RawIOBase +from typing import BinaryIO + + +class BoundedStream(RawIOBase): + """A read-only stream that raises an `OSError` once more than `max_size` bytes have been read. + + Closing the stream closes the inner stream. + """ + + def __init__(self, inner_stream: BinaryIO, max_size: int) -> None: + if max_size < 0: + raise ValueError("max_size must not be negative.") + self._inner_stream = inner_stream + self._max_size = max_size + self._total_bytes_read = 0 + + def readable(self) -> bool: + return True + + def readinto(self, buffer) -> int: # pyright: ignore[reportIncompatibleMethodOverride] + data = self._inner_stream.read(len(buffer)) + self._total_bytes_read += len(data) + if self._total_bytes_read > self._max_size: + raise OSError(f"Stream operation aborted. Exceeded maximum limit of {self._max_size} bytes.") + buffer[: len(data)] = data + return len(data) + + def close(self) -> None: + try: + self._inner_stream.close() + finally: + super().close() diff --git a/machine/utils/zip_entry_utils.py b/machine/utils/zip_entry_utils.py new file mode 100644 index 00000000..e3ec0476 --- /dev/null +++ b/machine/utils/zip_entry_utils.py @@ -0,0 +1,30 @@ +from typing import Union +from zipfile import BadZipFile, ZipFile, ZipInfo + +from .bounded_stream import BoundedStream + +DEFAULT_MAX_UNCOMPRESSED_SIZE = 100 * 1024 * 1024 # 100 MB +DEFAULT_MAX_COMPRESSION_RATIO = 100.0 # 100:1 ratio limit + + +def open_bounded_stream( + archive: ZipFile, + entry: Union[str, ZipInfo], + max_uncompressed_size: int = DEFAULT_MAX_UNCOMPRESSED_SIZE, + max_compression_ratio: float = DEFAULT_MAX_COMPRESSION_RATIO, +) -> BoundedStream: + """Opens a zip entry for reading, refusing entries that are too large or too highly compressed. + + Raises `BadZipFile` if the entry's declared size or compression ratio exceeds the limits. Reading more than + `max_uncompressed_size` bytes from the returned stream raises an `OSError`. + """ + if isinstance(entry, str): + entry = archive.getinfo(entry) + + if entry.file_size > max_uncompressed_size: + raise BadZipFile("Entry uncompressed size exceeds maximum allowed limit.") + + if entry.compress_size > 0 and entry.file_size / entry.compress_size > max_compression_ratio: + raise BadZipFile("Compression ratio exceeds safe threshold.") + + return BoundedStream(archive.open(entry, "r"), max_uncompressed_size) diff --git a/tests/utils/test_zip_entry_utils.py b/tests/utils/test_zip_entry_utils.py new file mode 100644 index 00000000..52ce4726 --- /dev/null +++ b/tests/utils/test_zip_entry_utils.py @@ -0,0 +1,45 @@ +from io import BytesIO +from zipfile import ZIP_DEFLATED, BadZipFile, ZipFile + +from pytest import raises + +from machine.utils.bounded_stream import BoundedStream +from machine.utils.zip_entry_utils import open_bounded_stream + + +def create_zip(name: str, content: bytes) -> ZipFile: + buffer = BytesIO() + with ZipFile(buffer, "w", ZIP_DEFLATED) as archive: + archive.writestr(name, content) + return ZipFile(buffer, "r") + + +def test_open_bounded_stream_returns_readable_stream() -> None: + with create_zip("test.txt", b"Hello World") as archive: + with open_bounded_stream(archive, "test.txt", max_uncompressed_size=100) as stream: + assert stream.read() == b"Hello World" + + +def test_open_bounded_stream_empty_entry() -> None: + with create_zip("empty.txt", b"") as archive: + with open_bounded_stream(archive, "empty.txt") as stream: + assert stream.read() == b"" + + +def test_open_bounded_stream_header_size_exceeded() -> None: + with create_zip("large.txt", bytes(200)) as archive: + with raises(BadZipFile): + open_bounded_stream(archive, "large.txt", max_uncompressed_size=100) + + +def test_open_bounded_stream_compression_ratio_exceeded() -> None: + with create_zip("bomb.txt", bytes(10_000)) as archive: + with raises(BadZipFile): + open_bounded_stream(archive, "bomb.txt", max_uncompressed_size=20_000, max_compression_ratio=2.0) + + +def test_bounded_stream_runtime_expansion_exceeds_limit() -> None: + with BoundedStream(BytesIO(b"1234567890"), max_size=5) as stream: + assert stream.read(4) == b"1234" + with raises(OSError): + stream.read(4) From b6123f1deb60958bc02d51d9d7d8dba6b1bb17ec Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:07:29 +0000 Subject: [PATCH 2/4] Fix pyright errors in bounded stream usage Type the inner stream as IO[bytes] and wrap the bounded stream in a BufferedReader before handing it to TextIOWrapper. Co-Authored-By: Claude Sonnet 5.5 Co-authored-by: Eli C. Lowry <83078660+Enkidu93@users.noreply.github.com> --- machine/corpora/dbl_bundle_text_corpus.py | 6 ++++-- machine/utils/bounded_stream.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/machine/corpora/dbl_bundle_text_corpus.py b/machine/corpora/dbl_bundle_text_corpus.py index 87cc3ae3..5b75823a 100644 --- a/machine/corpora/dbl_bundle_text_corpus.py +++ b/machine/corpora/dbl_bundle_text_corpus.py @@ -1,5 +1,5 @@ import os -from io import TextIOWrapper +from io import BufferedReader, TextIOWrapper from typing import List from xml.etree import ElementTree from zipfile import ZipFile @@ -31,7 +31,9 @@ def __init__(self, filename: StrPath) -> None: with open_bounded_stream(archive, versification_entry) as stream: abbr = doc.getroot().findtext("./identification/abbreviation", "") versification = Versification.parse( - TextIOWrapper(stream, encoding="utf-8-sig"), "versification.vrs", fallback_name=abbr + TextIOWrapper(BufferedReader(stream), encoding="utf-8-sig"), + "versification.vrs", + fallback_name=abbr, ) else: versification = ENGLISH_VERSIFICATION diff --git a/machine/utils/bounded_stream.py b/machine/utils/bounded_stream.py index d673f578..5c946131 100644 --- a/machine/utils/bounded_stream.py +++ b/machine/utils/bounded_stream.py @@ -1,5 +1,5 @@ from io import RawIOBase -from typing import BinaryIO +from typing import IO class BoundedStream(RawIOBase): @@ -8,7 +8,7 @@ class BoundedStream(RawIOBase): Closing the stream closes the inner stream. """ - def __init__(self, inner_stream: BinaryIO, max_size: int) -> None: + def __init__(self, inner_stream: IO[bytes], max_size: int) -> None: if max_size < 0: raise ValueError("max_size must not be negative.") self._inner_stream = inner_stream From 5756633e27fd4bdcada69b1f4f286209a61385a3 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:51:07 +0000 Subject: [PATCH 3/4] Correct open_bounded_stream docstring on understated headers Co-Authored-By: Claude Sonnet 5.5 Co-authored-by: Eli C. Lowry <83078660+Enkidu93@users.noreply.github.com> --- machine/utils/zip_entry_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machine/utils/zip_entry_utils.py b/machine/utils/zip_entry_utils.py index e3ec0476..5f6c4554 100644 --- a/machine/utils/zip_entry_utils.py +++ b/machine/utils/zip_entry_utils.py @@ -15,8 +15,8 @@ def open_bounded_stream( ) -> BoundedStream: """Opens a zip entry for reading, refusing entries that are too large or too highly compressed. - Raises `BadZipFile` if the entry's declared size or compression ratio exceeds the limits. Reading more than - `max_uncompressed_size` bytes from the returned stream raises an `OSError`. + Raises `BadZipFile` if the entry's declared size or compression ratio exceeds the limits. `ZipFile` never yields + more than the declared size, so an entry whose header understates its size fails the CRC check with `BadZipFile`. """ if isinstance(entry, str): entry = archive.getinfo(entry) From ba37fcb39a09eed6d5f79c0e505a273277cec394 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:03:56 +0000 Subject: [PATCH 4/4] Drop BoundedStream and rely on the zip header size check ZipFile never yields more than the declared size, so the runtime limit could not fire. open_bounded_stream now returns the zip entry stream. Co-Authored-By: Claude Sonnet 5.5 Co-authored-by: Eli C. Lowry <83078660+Enkidu93@users.noreply.github.com> --- machine/corpora/dbl_bundle_text_corpus.py | 6 ++--- machine/utils/bounded_stream.py | 33 ----------------------- machine/utils/zip_entry_utils.py | 8 +++--- tests/utils/test_zip_entry_utils.py | 8 ------ 4 files changed, 5 insertions(+), 50 deletions(-) delete mode 100644 machine/utils/bounded_stream.py diff --git a/machine/corpora/dbl_bundle_text_corpus.py b/machine/corpora/dbl_bundle_text_corpus.py index 5b75823a..87cc3ae3 100644 --- a/machine/corpora/dbl_bundle_text_corpus.py +++ b/machine/corpora/dbl_bundle_text_corpus.py @@ -1,5 +1,5 @@ import os -from io import BufferedReader, TextIOWrapper +from io import TextIOWrapper from typing import List from xml.etree import ElementTree from zipfile import ZipFile @@ -31,9 +31,7 @@ def __init__(self, filename: StrPath) -> None: with open_bounded_stream(archive, versification_entry) as stream: abbr = doc.getroot().findtext("./identification/abbreviation", "") versification = Versification.parse( - TextIOWrapper(BufferedReader(stream), encoding="utf-8-sig"), - "versification.vrs", - fallback_name=abbr, + TextIOWrapper(stream, encoding="utf-8-sig"), "versification.vrs", fallback_name=abbr ) else: versification = ENGLISH_VERSIFICATION diff --git a/machine/utils/bounded_stream.py b/machine/utils/bounded_stream.py deleted file mode 100644 index 5c946131..00000000 --- a/machine/utils/bounded_stream.py +++ /dev/null @@ -1,33 +0,0 @@ -from io import RawIOBase -from typing import IO - - -class BoundedStream(RawIOBase): - """A read-only stream that raises an `OSError` once more than `max_size` bytes have been read. - - Closing the stream closes the inner stream. - """ - - def __init__(self, inner_stream: IO[bytes], max_size: int) -> None: - if max_size < 0: - raise ValueError("max_size must not be negative.") - self._inner_stream = inner_stream - self._max_size = max_size - self._total_bytes_read = 0 - - def readable(self) -> bool: - return True - - def readinto(self, buffer) -> int: # pyright: ignore[reportIncompatibleMethodOverride] - data = self._inner_stream.read(len(buffer)) - self._total_bytes_read += len(data) - if self._total_bytes_read > self._max_size: - raise OSError(f"Stream operation aborted. Exceeded maximum limit of {self._max_size} bytes.") - buffer[: len(data)] = data - return len(data) - - def close(self) -> None: - try: - self._inner_stream.close() - finally: - super().close() diff --git a/machine/utils/zip_entry_utils.py b/machine/utils/zip_entry_utils.py index 5f6c4554..9d1c69cc 100644 --- a/machine/utils/zip_entry_utils.py +++ b/machine/utils/zip_entry_utils.py @@ -1,8 +1,6 @@ -from typing import Union +from typing import IO, Union from zipfile import BadZipFile, ZipFile, ZipInfo -from .bounded_stream import BoundedStream - DEFAULT_MAX_UNCOMPRESSED_SIZE = 100 * 1024 * 1024 # 100 MB DEFAULT_MAX_COMPRESSION_RATIO = 100.0 # 100:1 ratio limit @@ -12,7 +10,7 @@ def open_bounded_stream( entry: Union[str, ZipInfo], max_uncompressed_size: int = DEFAULT_MAX_UNCOMPRESSED_SIZE, max_compression_ratio: float = DEFAULT_MAX_COMPRESSION_RATIO, -) -> BoundedStream: +) -> IO[bytes]: """Opens a zip entry for reading, refusing entries that are too large or too highly compressed. Raises `BadZipFile` if the entry's declared size or compression ratio exceeds the limits. `ZipFile` never yields @@ -27,4 +25,4 @@ def open_bounded_stream( if entry.compress_size > 0 and entry.file_size / entry.compress_size > max_compression_ratio: raise BadZipFile("Compression ratio exceeds safe threshold.") - return BoundedStream(archive.open(entry, "r"), max_uncompressed_size) + return archive.open(entry, "r") diff --git a/tests/utils/test_zip_entry_utils.py b/tests/utils/test_zip_entry_utils.py index 52ce4726..e22e3e1d 100644 --- a/tests/utils/test_zip_entry_utils.py +++ b/tests/utils/test_zip_entry_utils.py @@ -3,7 +3,6 @@ from pytest import raises -from machine.utils.bounded_stream import BoundedStream from machine.utils.zip_entry_utils import open_bounded_stream @@ -36,10 +35,3 @@ def test_open_bounded_stream_compression_ratio_exceeded() -> None: with create_zip("bomb.txt", bytes(10_000)) as archive: with raises(BadZipFile): open_bounded_stream(archive, "bomb.txt", max_uncompressed_size=20_000, max_compression_ratio=2.0) - - -def test_bounded_stream_runtime_expansion_exceeds_limit() -> None: - with BoundedStream(BytesIO(b"1234567890"), max_size=5) as stream: - assert stream.read(4) == b"1234" - with raises(OSError): - stream.read(4)