diff --git a/docs/error-codes.md b/docs/error-codes.md index 6b1aefc5..7cf3395a 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -493,7 +493,7 @@ export CACHEKIT_CONNECTION_POOL_SIZE=100 **Exception**: none raised to a `@cache`-decorated caller: the result is returned but not stored in the backend. The exception is `@cache(interop=...)`: there an unsupported return value raises `InteropError` (a `ValueError` subclass, `from cachekit.interop import InteropError`) by design, so a cross-SDK entry is never silently skipped. Calling the serializer directly raises `TypeError`, e.g. `StandardSerializer does not support custom classes (Python-specific types). ...` or `StandardSerializer does not support pandas DataFrames or Series (Python-specific types). ...` -**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. +**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 of those that share one hash value, or two frozensets that share one, or two tuples that share one in a set where a tuple holds a frozenset (`AutoSerializer cannot store this set: it holds ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). **When it occurs**: ```python @@ -549,6 +549,8 @@ def get_json_data(): **Message** (logged): `L2 cache decrypt/integrity failure (corruption) for ...: ...`, or `(envelope_shape)` in place of `(corruption)` for the envelope-shape refusal described below +With `serializer='auto'`, a stored set or frozenset over the [Set Size Bound](serializers/auto.md#set-size-bound) is handled the same way. Under `@cache` the log line ends in `SerializationError` and names no bound, because the log carries only the exception's class; the full text, `Invalid set format: ... a hash-flood bound, not a framing error. ...`, appears only when you call `deserialize()` directly. A forged entry, an entry written by a cachekit version without the bound, or a set whose elements read back equal (pandas `Timestamp`s that differ only in nanoseconds, or an aware `datetime` at `fold=1`) causes it; the function recomputes as for any corrupt entry. + **Exception**: none under `@cache` — cachekit attempts to evict the entry (best effort; a failed delete is logged and the entry stays) and the function recomputes. Calling a serializer's `deserialize()` directly raises `SerializationError` (there is no separate `DeserializationError` class). **Cause**: Cached data is corrupted, or was written by an incompatible serializer/config. This is corruption *detection*, not tamper detection: the plaintext checksum is unkeyed xxHash3-64, which anyone with backend write access can recompute. Tamper detection requires encryption — see *Decryption failed* above. diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index cc9d18f2..ee9f8884 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -67,6 +67,18 @@ These markers are Python-specific — other language SDKs (Rust, TypeScript, PHP Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is installed but fails to import, the first DataFrame write or Arrow read on each `AutoSerializer` logs a warning on the `cachekit.serializers.auto_serializer` logger. New DataFrames are then written in the columnar format, and reading an Arrow entry raises `SerializationError`. +## Set Size Bound + +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, at most 8 of those that share one hash value, and the empty string `""` and empty bytes `b""` at most once each. No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. + +A set whose elements change their hash on the round trip can cross these bounds only once read back, so the writer accepts it and every read refuses it. pandas `Timestamp`s that differ only in nanoseconds read back as one `datetime`, so more than 8 of them, or two frozensets of them, are refused, while up to 8 bare ones collapse into one element without an error. An aware `datetime` at `fold=1` reads back with its offset but without `fold`, so it can come back equal to another element of the set. + +The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose colliding element hashes that make one read quadratic in the set's size. A stored set is a list, so it can also repeat an element, and every copy counts toward both bounds. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies, except `""` and `b""`, which hash to 0 in every process. The secret holds while Python's hash randomization is on, its default: a process run with `PYTHONHASHSEED` set to a fixed value hashes strings and bytes predictably, so a forged entry could then flood a set of them too. + +Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance, because comparing two of them costs more with every level they nest. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1, frozenset()), (-2, frozenset())}`, two tuples on one hash in a set whose tuples hold a frozenset. Tuples that hold only tuples and scalars keep the allowance: `{((0, -1), (0, -2)), ((0, -2), (0, -1))}` round-trips. + +Checking a set adds cost to every read and every write of it. For a set of more than 8 ints, floats, UUIDs or other elements that are not strings, sorting their hashes dominates, and it can make decoding a large set several times slower. A set of only strings or only bytes pays a smaller per-element check. To cache more elements than the bounds admit, cache them as a list. + ## When to Use **Use `serializer='auto'` when:** diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 7e57a0fa..6963bcc2 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -27,10 +27,13 @@ import importlib import logging +import operator import sys +from collections.abc import Iterable from datetime import date, datetime, time from functools import cached_property from importlib.util import find_spec +from itertools import chain, compress, repeat from typing import TYPE_CHECKING, Any, ClassVar, Optional, cast from uuid import UUID @@ -73,6 +76,14 @@ # never travel in an envelope. serialize() always writes the field, so an absent one is rot too. _HEADER_FORMATS = _ENVELOPE_FORMATS | {"numpy", "arrow"} +# Bounds on one stored set or frozenset, so a forged entry cannot make set() quadratic by choosing its elements' +# hashes (why, and what each bound admits: docs/serializers/auto.md#set-size-bound). _set_bound_error applies them to +# the reader's list and to what _auto_default writes. +_MAX_NON_STRING_SET_ELEMENTS = 1024 +_MAX_SAME_HASH_SET_ELEMENTS = 8 +_STR_TYPES = frozenset({str, bytes}) +_EMPTY_TWICE = 'the empty str or bytes ("" or b"") more than once' + # Error message constants for unsupported types PYDANTIC_ERROR_MESSAGE = ( "AutoSerializer does not support Pydantic models. Use .model_dump() to convert to dict: result = model.model_dump()" @@ -317,7 +328,20 @@ def _auto_default(obj: Any) -> Any: # NEW: set/frozenset support (type-safe roundtrip) if isinstance(obj, (set, frozenset)): - return {"__set__": True, "value": list(obj), "frozen": isinstance(obj, frozenset)} + value = list(obj) + types = list(map(type, value)) + exposed = len(types) - types.count(str) - types.count(bytes) + if exposed > _MAX_NON_STRING_SET_ELEMENTS: + # A str or bytes subclass reads back as str or bytes: recount as the reader will see them. Below the + # bound a subclass counts as exposed, which can only make the writer stricter. + exposed = sum(not isinstance(x, (str, bytes)) for x in value) + reason = _set_bound_error(value, types, exposed) + if reason: + raise TypeError( + f"AutoSerializer cannot store this {type(obj).__name__}: it holds {reason}, which its reader refuses " + "as a hash-flood bound. Cache the elements as a list instead." + ) + return {"__set__": True, "value": value, "frozen": isinstance(obj, frozenset)} # NumPy array support (nested in dicts/lists via msgpack custom encoder) np_loaded = sys.modules.get("numpy") @@ -340,6 +364,89 @@ def _auto_default(obj: Any) -> Any: raise TypeError(f"Object of type {type(obj).__name__} is not MessagePack serializable") +def _frozenset_in_tuples(values: list[Any], types: list[type]) -> bool: + """Whether a frozenset sits at any depth inside a tuple among ``values``. + + Walked a level at a time in C passes: linear in the tuples' size, and no recursion to overflow. + """ + while True: + values = list(chain.from_iterable(compress(values, map(issubclass, types, repeat(tuple))))) + if not values: + return False + types = list(map(type, values)) + if any(issubclass(t, frozenset) for t in set(types)): + return True + + +def _hash_run_over(values: Iterable[Any], limit: int) -> bool: + """Whether more than ``limit`` of ``values`` share one hash. + + The hashes are sorted, never tallied in a dict or set: a table keyed by them is the very build being bounded. + """ + hashes = sorted(map(hash, values)) + return any(map(operator.eq, hashes, hashes[limit:])) + + +def _empty_count(values: list[Any], types: list[type], kind: type) -> int: + """How many of ``values`` are an empty ``kind`` (str or bytes), counted in C passes.""" + return sum(map(operator.not_, compress(values, map(operator.is_, types, repeat(kind))))) + + +def _may_compare_frozensets(values: list[Any], types: list[type], kinds: set[type]) -> bool: + """Whether set() over ``values`` could compare two frozensets: two of them share a hash, or two tuples do while a + frozenset sits inside one of the set's tuples. ``kinds`` is set(types). + """ + checked: list[type] = [frozenset] if any(issubclass(t, frozenset) for t in kinds) else [] + if any(issubclass(t, tuple) for t in kinds) and _frozenset_in_tuples(values, types): + checked.append(tuple) + return any(_hash_run_over(compress(values, map(issubclass, types, repeat(kind))), 1) for kind in checked) + + +def _set_bound_error(values: list[Any], types: list[type], exposed: int | None = None) -> str | None: + """Why set() over ``values`` could be quadratic, or None (see docs/serializers/auto.md#set-size-bound). + + ``types`` holds each value's type. ``exposed`` is how many values are neither str nor bytes, by default counted + from ``types``. Every pass runs in C, and a pass only runs when its types are present: a Python loop over every + element would slow honest sets several-fold. + """ + # Distinct types are few, and a type hashes by identity, so this set is safe to build. + kinds = set(types) + if len(kinds) == 1 and kinds <= _STR_TYPES: + # All str or all bytes, the most common large set: nothing is exposed, and list.count finds the empty value. + return _EMPTY_TWICE if values.count(types[0]()) > 1 else None + strings = str in kinds or bytes in kinds + if exposed is None: + exposed = len(types) - types.count(str) - types.count(bytes) if strings else len(types) + if exposed > _MAX_NON_STRING_SET_ELEMENTS: + return f"more than {_MAX_NON_STRING_SET_ELEMENTS} elements that are neither str nor bytes" + # Every other str and bytes hashes under a per-process secret, but "" and b"" hash to 0 in every process, and a + # stored list can repeat them for 1 or 2 bytes a copy: each copy would walk whatever forged elements pile on hash + # 0's probe path. An honest set holds each at most once. + if any(kind in kinds and _empty_count(values, types, kind) > 1 for kind in (str, bytes)): + return _EMPTY_TWICE + if _may_compare_frozensets(values, types, kinds): + return "two frozensets, or two tuples in a set where a tuple holds a frozenset, that share one hash" + # The per-hash bound counts the elements that are neither str nor bytes (only one "" and one b"" can join their + # run on hash 0), so fewer of them than the bound cannot exceed it. + if exposed <= _MAX_SAME_HASH_SET_ELEMENTS: + return None + others = compress(values, map(operator.not_, map(_STR_TYPES.__contains__, types))) if strings else values + if _hash_run_over(others, _MAX_SAME_HASH_SET_ELEMENTS): + return f"more than {_MAX_SAME_HASH_SET_ELEMENTS} elements that are neither str nor bytes and share one hash" + return None + + +def _check_untrusted_set(values: list[Any]) -> None: + """Refuse a stored set's elements if set() over them could be quadratic, before set() builds it.""" + reason = _set_bound_error(values, list(map(type, values))) + if reason: + raise SerializationError( + f"Invalid set format: {reason}: a hash-flood bound, not a framing error. Either the entry is forged, a " + "cachekit version without this bound wrote it, or, for a shared hash, its elements read back equal (such " + "as pandas Timestamps that differ only in nanoseconds)." + ) + + def _auto_object_hook(obj: Any) -> Any: """Custom decoder for types encoded by _auto_default(). @@ -401,6 +508,7 @@ def _auto_object_hook(obj: Any) -> Any: value_list = obj["value"] if not isinstance(value_list, list): raise SerializationError(f"Invalid set format: expected list, got {type(value_list).__name__}") + _check_untrusted_set(value_list) if obj.get("frozen"): return frozenset(value_list) diff --git a/tests/unit/test_auto_interop_decode.py b/tests/unit/test_auto_interop_decode.py index 2e6ff5e3..2a5ff3a1 100644 --- a/tests/unit/test_auto_interop_decode.py +++ b/tests/unit/test_auto_interop_decode.py @@ -414,7 +414,7 @@ def test_buffer_inputs_decode_like_bytes() -> None: def _keys_tested_by(hook: Callable[..., Any]) -> set[str]: """The marker keys ``hook`` tests: the dunder string literals in its source.""" - return set(re.findall(r'"(__[a-z]+__)"', inspect.getsource(hook))) + return set(re.findall(r'"(__\w+__)"', inspect.getsource(hook))) def test_each_readers_rust_marker_keys_are_its_hooks() -> None: diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index a9a98e78..0a855049 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -11,7 +11,9 @@ from __future__ import annotations +import itertools from collections.abc import Callable +from datetime import datetime, timedelta from uuid import UUID import msgpack @@ -20,7 +22,7 @@ from hypothesis import strategies as st from cachekit._rust_serializer import ByteStorage -from cachekit.serializers.auto_serializer import AutoSerializer +from cachekit.serializers.auto_serializer import _MAX_NON_STRING_SET_ELEMENTS, AutoSerializer from cachekit.serializers.base import SerializationError from tests.utils.hypothesis_settings import slow_first_draw @@ -206,6 +208,231 @@ def test_frozenset_type_preserved(self): assert deserialized == test_frozenset +def _uuid_marker(i: int) -> dict: + return {"__uuid__": True, "value": str(UUID(int=i))} + + +def _set_doc(values: list, frozen: bool) -> bytes: + return msgpack.packb({"__set__": True, "value": values, "frozen": frozen}, use_bin_type=True) + + +# UUID.__hash__ is hash(self.int), and an int hashes mod 2**61 - 1: these all hash alike. +_SAME_HASH = [12345 + k * (2**61 - 1) for k in range(40)] + + +class _Str(str): + pass + + +class TestAutoSerializerSetHashFloodBound: + """A stored set holds at most _MAX_NON_STRING_SET_ELEMENTS elements that are neither str nor bytes, at most + _MAX_SAME_HASH_SET_ELEMENTS on one hash, and no two frozensets on one hash (nor two tuples, once a frozenset sits in + a tuple of the set). Each refusal comes before set() builds the set, so a forged one never pays for a quadratic + build. The writer refuses the same sets, so an honest one fails at write, not on every read (unless its elements + only share a hash once read back, as pandas Timestamps that differ only in nanoseconds do).""" + + @pytest.fixture + def counts(self, monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: + counts = {"hash": 0, "eq": 0} + uuid_hash, uuid_eq = UUID.__hash__, UUID.__eq__ + + def counting_hash(self: UUID) -> int: + counts["hash"] += 1 + return uuid_hash(self) + + def counting_eq(self: UUID, other: object) -> bool: + counts["eq"] += 1 + return uuid_eq(self, other) + + monkeypatch.setattr(UUID, "__hash__", counting_hash) + monkeypatch.setattr(UUID, "__eq__", counting_eq) + return counts + + @pytest.mark.parametrize("frozen", [False, True]) + @pytest.mark.parametrize( + "element", + [_uuid_marker, lambda i: {"__tuple__": True, "value": [_uuid_marker(i), 0]}], + ids=["uuid", "tuple"], + ) + def test_a_same_hash_set_is_refused_before_any_comparison( + self, counts: dict[str, int], frozen: bool, element: Callable[[int], dict] + ) -> None: + data = _set_doc([element(i) for i in _SAME_HASH], frozen) + with pytest.raises( + SerializationError, + match="more than 8 elements that are neither str nor bytes and share one hash: a hash-flood bound", + ): + AutoSerializer(enable_integrity_checking=False).deserialize(data) + # Hashed once each, by the sort; set() would have compared each to every earlier one. + assert counts == {"hash": len(_SAME_HASH), "eq": 0} + + def test_timestamps_that_read_back_equal_are_refused_on_read_past_the_bounds(self) -> None: + """The writer sees distinct nanosecond Timestamps; the reader sees equal datetimes, so equal hashes.""" + pd = pytest.importorskip("pandas") + stamps = [pd.Timestamp("2026-01-01") + pd.Timedelta(i, "ns") for i in range(9)] + serializer = AutoSerializer() + for value in (set(stamps), {frozenset(stamps[:1]), frozenset(stamps[1:2])}): + with pytest.raises(SerializationError, match="for a shared hash, its elements read back equal"): + serializer.deserialize(*serializer.serialize(value)) + assert serializer.deserialize(*serializer.serialize(set(stamps[:8]))) == {datetime(2026, 1, 1)} + + def test_eight_elements_may_share_a_hash_but_not_nine(self) -> None: + """hash(-1) == hash(-2), so 3-D offsets over -1 and -2 put 8 honest tuples on one hash; 4-D offsets put 16.""" + serializer = AutoSerializer() + offsets = set(itertools.product(range(-2, 3), repeat=3)) + assert len({hash(t) for t in itertools.product((-1, -2), repeat=3)}) == 1 + assert serializer.deserialize(*serializer.serialize(offsets)) == offsets + with pytest.raises(TypeError, match="it holds more than 8 elements that are neither str nor bytes and share one hash"): + serializer.serialize(set(itertools.product(range(-2, 3), repeat=4))) + with pytest.raises(SerializationError, match="more than 8 elements that are neither str nor bytes and share one hash"): + serializer.deserialize(_set_doc([_uuid_marker(i) for i in _SAME_HASH[:9]], False), None) + assert serializer.deserialize(_set_doc([_uuid_marker(i) for i in _SAME_HASH[:8]], False), None) == { + UUID(int=i) for i in _SAME_HASH[:8] + } + + def test_ints_floats_and_other_types_on_one_hash_count_together(self) -> None: + """No type is exempt: 202 distinct floats can share one hash, and a UUID's Python __eq__ runs against each.""" + p = 2**61 - 1 + serializer = AutoSerializer(enable_integrity_checking=False) + for mixed in ( + [*(2.0 ** (61 * k) for k in range(9))], # hash 1 + [*(k * p for k in range(-4, 5))], # hash 0 + [*(2.0 ** (61 * k) for k in range(5)), *(_uuid_marker(1 + j * p) for j in range(4))], # hash 1 + ): + for frozen in (False, True): + with pytest.raises( + SerializationError, match="more than 8 elements that are neither str nor bytes and share one hash" + ): + serializer.deserialize( + _set_doc([*mixed, "a"], frozen) + ) # a str forces the branch that filters strings out before the sort + + @pytest.mark.parametrize("frozen", [False, True]) + def test_duplicates_count_toward_the_hash_bound(self, frozen: bool) -> None: + """A stored list can repeat an element, which the set dedups only after each copy has walked its probe path, + so copies count like any elements on one hash. An honest writer never repeats one.""" + serializer = AutoSerializer(enable_integrity_checking=False) + with pytest.raises(SerializationError, match="more than 8 elements that are neither str nor bytes and share one hash"): + serializer.deserialize(_set_doc([*range(100), *[25] * 8], frozen)) + assert serializer.deserialize(_set_doc([*range(100), *[25] * 7], frozen)) == set(range(100)) + + @pytest.mark.parametrize("frozen", [False, True]) + @pytest.mark.parametrize("empty", ["", b""]) + def test_an_empty_str_or_bytes_may_appear_once(self, frozen: bool, empty: str | bytes) -> None: + """Both hash to 0 in every process, so copies of one would all walk hash 0's probe path, and they count + toward neither bound.""" + assert hash(empty) == 0 + serializer = AutoSerializer(enable_integrity_checking=False) + with pytest.raises(SerializationError, match=r"the empty str or bytes \(\"\" or b\"\"\) more than once"): + serializer.deserialize(_set_doc([0, empty, empty], frozen)) + # All str or all bytes takes its own branch, which counts the empty value with list.count. + a, b = ("a", "b") if empty == "" else (b"a", b"b") + with pytest.raises(SerializationError, match="more than once"): + serializer.deserialize(_set_doc([a, empty, b, empty], frozen)) + assert serializer.deserialize(_set_doc([a, empty, b], frozen)) == {a, empty, b} + honest = {0, "a", "", b""} + assert AutoSerializer().deserialize(*AutoSerializer().serialize(honest)) == honest + + @pytest.mark.parametrize("frozen", [False, True]) + @pytest.mark.parametrize("wrap", [lambda m: m, lambda m: {"__tuple__": True, "value": [m]}], ids=["bare", "in a tuple"]) + def test_frozensets_on_one_hash_are_refused_before_any_comparison( + self, counts: dict[str, int], frozen: bool, wrap: Callable[[dict], dict] + ) -> None: + """Two frozensets compare by looking each element of one up in the other, and each lookup can compare + same-hash frozensets one level down, so nested same-hash frozensets cost more with every level. Even two on + one hash are refused, bare or in a tuple, before set() compares them.""" + inner = [{"__set__": True, "value": [_uuid_marker(i)], "frozen": True} for i in _SAME_HASH[:2]] + with pytest.raises( + SerializationError, match="two frozensets, or two tuples in a set where a tuple holds a frozenset, that share" + ): + AutoSerializer(enable_integrity_checking=False).deserialize(_set_doc([wrap(m) for m in inner], frozen)) + assert counts["eq"] == 0 + + def test_the_writer_refuses_same_hash_frozensets_and_tuples_holding_them(self) -> None: + assert hash(frozenset({-1})) == hash(frozenset({-2})) + serializer = AutoSerializer() + # Tuples holding only tuples and scalars compare in linear time, so they may share a hash. + edges = {((0, -1), (0, -2)), ((0, -2), (0, -1)), (-1, ((1, 2),)), (-2, ((1, 2),))} + assert serializer.deserialize(*serializer.serialize(edges)) == edges + for value in ( + {frozenset({-1}), frozenset({-2})}, + {(-1, frozenset()), (-2, frozenset())}, + {(((frozenset({-1}),),),), (((frozenset({-2}),),),)}, # however deep the frozenset sits + {(-1, "x"), (-2, "x"), (0, (frozenset(),))}, # once any tuple holds one, every tuple counts + ): + with pytest.raises( + TypeError, match="two frozensets, or two tuples in a set where a tuple holds a frozenset, that share" + ): + serializer.serialize(value) + + @pytest.mark.parametrize("frozen", [False, True]) + def test_a_set_over_the_bound_is_refused_before_any_hash(self, counts: dict[str, int], frozen: bool) -> None: + data = _set_doc([_uuid_marker(i) for i in range(_MAX_NON_STRING_SET_ELEMENTS + 1)], frozen) + with pytest.raises(SerializationError, match="more than 1024 elements that are neither str nor bytes"): + AutoSerializer(enable_integrity_checking=False).deserialize(data) + assert counts == {"hash": 0, "eq": 0} + + def test_the_bound_counts_only_elements_that_are_neither_str_nor_bytes(self) -> None: + serializer = AutoSerializer(enable_integrity_checking=False) + strings = [*map(str, range(2000)), *(b"%d" % i for i in range(2000))] + assert serializer.deserialize(_set_doc([*strings, *range(1024)], False)) == {*strings, *range(1024)} + with pytest.raises(SerializationError, match="more than 1024 elements"): + serializer.deserialize(_set_doc([*strings, *range(1025)], False)) + only_bytes = [b"%d" % i for i in range(2000)] + assert serializer.deserialize(_set_doc([*only_bytes, *range(1024)], False)) == {*only_bytes, *range(1024)} + + def test_a_set_at_the_bound_hashes_each_element_twice(self, counts: dict[str, int]) -> None: + value = frozenset(UUID(int=i) for i in range(_MAX_NON_STRING_SET_ELEMENTS)) + serializer = AutoSerializer() + data, metadata = serializer.serialize(value) + counts.update(hash=0, eq=0) + result = serializer.deserialize(data, metadata) + # Once by the sort, once by frozenset(); no element shares a hash, so nothing is compared. + assert counts == {"hash": 2 * len(value), "eq": 0} + assert result == value + + @pytest.mark.parametrize( + "value", + [ + set(range(_MAX_NON_STRING_SET_ELEMENTS)), + frozenset(range(-(2**63), -(2**63) + _MAX_NON_STRING_SET_ELEMENTS)), + {i / 3 for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {UUID(int=i * 2**64) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {(i, str(i)) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {frozenset({i, -i - 10}) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {datetime(2026, 1, 1) + timedelta(seconds=i) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {*range(2, _MAX_NON_STRING_SET_ELEMENTS - 1), True, None, 1.5}, # True == 1, so no 1 here + {*range(_MAX_NON_STRING_SET_ELEMENTS), *map(str, range(5000)), *(b"%d" % i for i in range(5000))}, + {*range(_MAX_NON_STRING_SET_ELEMENTS), *(_Str(i) for i in range(5000))}, + {((i, -i), (frozenset({i}),), str(i)) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + ], + ids=[ + "ints", + "int64 floor", + "floats", + "uuids", + "tuples", + "frozensets", + "datetimes", + "mixed", + "strs", + "str subclass", + "nested tuples", + ], + ) + @pytest.mark.parametrize("kind", [set, frozenset]) + def test_every_set_shape_at_the_bound_round_trips(self, value: set, kind: type) -> None: + serializer = AutoSerializer() + result = serializer.deserialize(*serializer.serialize({"v": kind(value)}))["v"] + assert type(result) is kind + assert result == value + + @pytest.mark.parametrize("kind", [set, frozenset]) + def test_the_writer_refuses_a_set_over_the_count(self, kind: type) -> None: + with pytest.raises(TypeError, match=rf"cannot store this {kind.__name__}: it holds more than 1024 elements"): + AutoSerializer().serialize({"v": [kind(range(_MAX_NON_STRING_SET_ELEMENTS + 1))]}) + + class TestAutoSerializerComplexNesting: """Test nested complex objects with new types."""