diff --git a/docs/source/pcapkit/corekit/infoclass.rst b/docs/source/pcapkit/corekit/infoclass.rst index ffe8563b96..e7e4b1e43a 100644 --- a/docs/source/pcapkit/corekit/infoclass.rst +++ b/docs/source/pcapkit/corekit/infoclass.rst @@ -23,6 +23,8 @@ classes :class:`~pcapkit.corekit.infoclass.MultiInfo` and :no-value: .. autoattribute:: __excluded__ :no-value: + .. autoattribute:: __included__ + :no-value: .. autoattribute:: __map__ .. autoattribute:: __map_reverse__ diff --git a/docs/source/pcapkit/protocols/index.rst b/docs/source/pcapkit/protocols/index.rst index 36918052f3..b076ee2130 100644 --- a/docs/source/pcapkit/protocols/index.rst +++ b/docs/source/pcapkit/protocols/index.rst @@ -268,3 +268,6 @@ Data Model Please refer to :func:`Protocol._decode_next_layer ` for more information with the inserted names to be excluded. + + .. autoattribute:: __included__ + :no-value: diff --git a/pcapkit/corekit/infoclass.py b/pcapkit/corekit/infoclass.py index 7964d6a7be..d7d80ee2ea 100644 --- a/pcapkit/corekit/infoclass.py +++ b/pcapkit/corekit/infoclass.py @@ -21,7 +21,7 @@ import sys import types import typing -from typing import TYPE_CHECKING, Generic, TypeVar +from typing import TYPE_CHECKING, Generic, TypeVar, cast from pcapkit.corekit.enum import EnumLookup from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict, iter_multi_items @@ -30,7 +30,8 @@ from pcapkit.utilities.warnings import InfoWarning, warn if TYPE_CHECKING: - from typing import Any, ItemsView, Iterable, Iterator, NoReturn, Optional, Type + from typing import (Any, Collection, ItemsView, Iterable, Iterator, NoReturn, Optional, + SupportsIndex, Type) from typing_extensions import Self @@ -255,11 +256,13 @@ class InfoMeta(abc.ABCMeta): :class:`Info` class. It can be useful to reduce runtime generation cost as well as caching already generated attributes. - * :attr:`Info.__additional__` and :attr:`Info.__excluded__` are - lists of additional and excluded field names, which are used to - determine certain names to be included or excluded from the field - dictionary. They will be automatically populated from the class - attributes of the :class:`Info` class and its base classes. + * :attr:`Info.__additional__`, :attr:`Info.__excluded__` and + :attr:`Info.__included__` are lists of additional, excluded and included + field names, which are used to determine certain names to be treated as + built-in names, left out of ``repr`` and the dumps, or carried as + metadata by :meth:`Info.to_dict`. They will be automatically populated + from the class attributes of the :class:`Info` class and its base + classes. .. note:: @@ -286,8 +289,9 @@ class C(B): """ def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]', **kwargs: 'Any') -> 'Type[Info]': - """Create the class, extending its :attr:`Info.__additional__` and - :attr:`Info.__excluded__` with every base's. + """Create the class, extending its :attr:`Info.__additional__`, + :attr:`Info.__excluded__` and :attr:`Info.__included__` with every + base's. Args: name: Class name. @@ -300,6 +304,8 @@ def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]' attrs['__additional__'] = [] if '__excluded__' not in attrs: attrs['__excluded__'] = [] + if '__included__' not in attrs: + attrs['__included__'] = [] for base in bases: if hasattr(base, '__additional__'): @@ -307,6 +313,8 @@ def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]' name for name in base.__additional__ if name not in attrs['__additional__']) if hasattr(base, '__excluded__'): attrs['__excluded__'].extend(name for name in base.__excluded__ if name not in attrs['__excluded__']) + if hasattr(base, '__included__'): + attrs['__included__'].extend(name for name in base.__included__ if name not in attrs['__included__']) return super().__new__(cls, name, bases, attrs, **kwargs) # type: ignore[return-value] @@ -331,6 +339,19 @@ class Info(Mapping[str, VT], Generic[VT], metaclass=InfoMeta): Iterating an instance skips the names listed in :attr:`__excluded__`. + A key whose name is a dunder, such as ``__next_type__``, is metadata. + Iterating an instance, ``len()`` and :meth:`to_dict` leave such a key out + unless :attr:`__included__` names it, and carry every other key; + :attr:`__excluded__` names the keys that ``repr``, ``str`` and the dumps of + :mod:`pcapkit.dumpkit` leave out besides. A subclass extends either list + with its own names, as for :attr:`__additional__`. + + .. versionchanged:: 1.5.0 + + :attr:`__excluded__` names the keys left out of ``repr``, ``str`` and + the dumps only, and :attr:`__included__` the dunder keys that iterating + an instance, ``len()`` and :meth:`to_dict` carry (:issue:`2003`). + """ if TYPE_CHECKING: @@ -352,8 +373,20 @@ class Info(Mapping[str, VT], Generic[VT], metaclass=InfoMeta): #: List of additional built-in names. __additional__: 'list[str]' = [] - #: List of names to be excluded from :obj:`dict` conversion. + #: List of names of keys left out of ``repr``, ``str`` and the dumps of + #: :mod:`pcapkit.dumpkit` -- metadata such as ``__next_type__``, which + #: :meth:`to_dict` may still carry, see :attr:`__included__`. + #: Finalisation adds the names in :attr:`__builtin__`. __excluded__: 'list[str]' = [] + #: List of names of dunder keys that iterating an instance, ``len()`` and + #: :meth:`to_dict` carry -- metadata such as ``__next_type__``, which a + #: rebuild from the :obj:`dict` reads. Every other dunder key is dropped + #: from all three, though indexing still reads it; a key that is no + #: dunder is carried regardless. A subclass extends the list with its own + #: names, as for :attr:`__excluded__`, so a custom :class:`Info` holding + #: dunder keys of its own must list them here for :meth:`to_dict` to keep + #: them (:issue:`2003`). + __included__: 'list[str]' = [] def __init_subclass__(cls, /, *args: 'Any', **kwargs: 'Any') -> 'None': """Refuse to derive from a finalised info class. @@ -446,9 +479,9 @@ class is finalised on its first construction, whatever its # ``info_final`` writes ``BASE`` or ``FINAL`` onto the class it # finalises, and a subclass reading that state through inheritance was # never finalised itself -- it would keep the ``__excluded__`` it copied - # at declaration, without the names ``info_final`` adds, and leak - # ``__map__``, ``__map_reverse__`` and ``__multi__`` from ``to_dict()`` - # (GitHub issue #1490). The inner one: an ancestor hand-marked bare + # at declaration, without the names ``info_final`` adds, which once + # leaked ``__map__``, ``__map_reverse__`` and ``__multi__`` from + # ``to_dict()`` (GitHub issue #1490). The inner one: an ancestor hand-marked bare # ``@final`` passes its marker to every subclass declared before it # landed, and ``getattr`` would blame such a subclass for a mismarking # that is its ancestor's. ``OwnDictRuleTests.test_a_subclass_that_ @@ -579,13 +612,14 @@ def __repr__(self) -> 'str': def __len__(self) -> 'int': # NOTE: count exactly the keys :meth:`__iter__` yields, so the - # bookkeeping attributes in ``__excluded__`` are not counted. - return sum(1 for key in self.__dict__ if key not in self.__excluded__) + # bookkeeping attributes and the dunder keys ``__included__`` does not + # name are not counted, while the keys in ``__excluded__`` are (#2003). + return sum(1 for key in self.__dict__ if not self.__hidden(key)) def __iter__(self) -> 'Iterator[str]': for key in self.__dict__: - if key in self.__excluded__: - continue + if self.__hidden(key): + continue yield self.__map_reverse__.get(key, key) def __getitem__(self, name: 'str') -> 'VT': @@ -609,10 +643,22 @@ def items(self, multi: 'bool' = False) -> 'ItemsView[str, VT] | Iterator[tuple[s def __items_multi(self) -> 'Iterator[tuple[str, VT]]': """Every ``(key, value)`` pair, with the transformed keys, in order.""" for (key, value) in self.__dict__.items(): - if key not in self.__excluded__: + if not self.__hidden(key): yield key, value yield from self.__multi__.get(key, ()) + def __hidden(self, key: 'str') -> 'bool': + """Whether iteration, ``len()`` and :meth:`to_dict` leave ``key`` out. + + That is a bookkeeping attribute in :attr:`__builtin__`, or a dunder key + :attr:`__included__` does not name (:issue:`2003`). + + """ + if key in self.__builtin__: + return True + return (isinstance(key, str) and key.startswith('__') and key.endswith('__') + and key not in self.__included__) + def __setattr__(self, name: 'str', value: 'VT') -> 'NoReturn': raise UnsupportedCall("can't set attribute") @@ -719,10 +765,14 @@ def to_dict(self) -> 'OrderedMultiDict[str, VT]': Returns an :class:`~pcapkit.corekit.multidict.OrderedMultiDict` rather than a :obj:`dict`, so a key held more than once keeps every - value (:issue:`1484`). + value (:issue:`1484`). It leaves out the dunder keys + :attr:`__included__` does not name, carries the keys in + :attr:`__excluded__`, and records those as its own ``__excluded__`` + for the dumpers to leave out (:issue:`2003`). """ - dict_ = _OrderedMultiDict() # type: OrderedMultiDict[str, Any] + dict_ = _OrderedMultiDict() # type: _OrderedMultiDict[str, Any] + dict_.__excluded__ = self.__excluded__ for (key, value) in self.__items_multi(): out_key = self.__map_reverse__.get(key, key) if isinstance(value, Info): @@ -775,6 +825,11 @@ class _OrderedMultiDict(OrderedMultiDict[KT, VT]): __ne__ = _ne + #: Names of the keys the dumpers leave out: the :attr:`Info.__excluded__` + #: of the info exported, kept by copies, deep copies and pickles; none for + #: an export of a multi-mapping (:issue:`2003`). + __excluded__ = () # type: Collection[str] + def __repr__(self) -> 'str': """Write the export as the :class:`~pcapkit.corekit.multidict.OrderedMultiDict` it is, with every pair. @@ -782,6 +837,29 @@ def __repr__(self) -> 'str': """ return f'{OrderedMultiDict.__name__}({list(self.items(multi=True))!r})' + def copy(self) -> 'Self': + """Return a shallow copy, with the same ``__excluded__``.""" + new = super().copy() + new.__excluded__ = self.__excluded__ + return new + + def deepcopy(self, memo: 'Optional[dict]' = None) -> 'Self': + """Return a deep copy, with the same ``__excluded__``.""" + new = cast('Self', super().deepcopy(memo)) + new.__excluded__ = self.__excluded__ + return new + + def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'Any': + return _restore_export, (type(self), list(self.items(multi=True)), list(self.__excluded__)) + + +def _restore_export(cls: 'Type[_OrderedMultiDict]', pairs: 'list[tuple[Any, Any]]', + excluded: 'list[str]') -> '_OrderedMultiDict': + """Unpickle an :class:`_OrderedMultiDict`, with its ``__excluded__``.""" + new = cls(pairs) + new.__excluded__ = excluded + return new + class _MultiInfo(Info[VT], Generic[KT, VT]): """An :class:`Info` that holds its values as a multi-mapping. diff --git a/pcapkit/dumpkit/common.py b/pcapkit/dumpkit/common.py index d025491560..adbf92a705 100644 --- a/pcapkit/dumpkit/common.py +++ b/pcapkit/dumpkit/common.py @@ -41,7 +41,7 @@ if TYPE_CHECKING: - from typing import Any, DefaultDict, Iterator, Optional, TextIO, Type + from typing import Any, Collection, DefaultDict, Iterator, Optional, TextIO, Type from dictdumper.dumper import Dumper as ABCDumper from typing_extensions import Literal @@ -409,6 +409,26 @@ def escape_key(key: 'Any') -> 'Any': return json.dumps(format(key, ''))[1:-1] return key + def excluded(info: 'Info | MultiDict[Any, Any]') -> 'Collection[Any]': + """Return the keys of ``info`` the writer leaves out. + + Args: + info: An :class:`~pcapkit.corekit.infoclass.Info`, or a multi-mapping + such as the export its :meth:`~pcapkit.corekit.infoclass.Info.to_dict` + returns. + + Returns: + The :attr:`~pcapkit.corekit.infoclass.Info.__excluded__` of a field + model, or of the field model an export was made from -- metadata + such as ``__next_type__``, which only a rebuild reads + (:issue:`2003`). None for an option list, whose keys are its + options. + + """ + if isinstance(info, Info) and isinstance(info, MultiDict): + return () + return getattr(info, '__excluded__', ()) + def fields(info: 'Info | MultiDict[Any, Any]') -> 'dict[Any, Any]': """Return the fields of ``info`` as the :class:`dict` the writer is handed. @@ -423,10 +443,12 @@ def fields(info: 'Info | MultiDict[Any, Any]') -> 'dict[Any, Any]': held more than once, such as a repeated IPv6 extension header, maps to the :class:`list` of all its values, in order (GitHub issue :issue:`1484`). The values are left as they are, so each comes back - through :meth:`DictDumper.object_hook` when written. + through :meth:`DictDumper.object_hook` when written. The keys + :func:`excluded` names are left out. """ - pairs = list(info.items(multi=True)) + skip = excluded(info) + pairs = [(key, val) for key, val in info.items(multi=True) if key not in skip] counts = collections.Counter(key for key, _ in pairs) result = {} # type: dict[Any, Any] for key, val in pairs: @@ -540,10 +562,13 @@ def object_hook(self, o: 'Any') -> 'Any': # by name could only gather the repeats of a key under its first # occurrence (GitHub issue #1263). The keys are escaped when the # writer hands each mapping back to this hook, through the - # :class:`dict` branch below. + # :class:`dict` branch below. The keys an export of an + # :class:`Info` excludes are left out, as :func:`fields` leaves + # them out (#2003). + skip = excluded(o) result = [ {render_enum(key) if isinstance(key, (enum.Enum, aenum.Enum)) else key: val} - for key, val in o.items(multi=True) + for key, val in o.items(multi=True) if key not in skip ] elif isinstance(o, MultiDict): # NOTE: a :class:`MultiDict` keeps no order across keys, only diff --git a/pcapkit/protocols/data/application/httpv2.py b/pcapkit/protocols/data/application/httpv2.py index ec1e8ff360..088fc102fb 100644 --- a/pcapkit/protocols/data/application/httpv2.py +++ b/pcapkit/protocols/data/application/httpv2.py @@ -31,6 +31,10 @@ class Flags(Data): """Data model for HTTP/2 flags.""" + # NOTE: ``__value__`` is a dunder key, which ``to_dict`` carries only so + # named (#2003); the rebuild reads it, and the dumps write it, as before. + __included__ = ['__value__'] + if TYPE_CHECKING: #: Flags as in combination value, bits the frame type leaves #: undefined included. diff --git a/pcapkit/protocols/data/data.py b/pcapkit/protocols/data/data.py index 569c29dbf5..1f6ecebef3 100644 --- a/pcapkit/protocols/data/data.py +++ b/pcapkit/protocols/data/data.py @@ -14,9 +14,28 @@ class Data(Info): - """Base class for data models.""" - - __excluded__ = ['__next_name__', '__next_type__'] + """Base class for data models. + + The ``__next_name__``, ``__next_type__`` and ``__short_read__`` keys are + metadata, listed in :attr:`~pcapkit.corekit.infoclass.Info.__included__` + and :attr:`~pcapkit.corekit.infoclass.Info.__excluded__`: + :meth:`Protocol.from_data ` + dispatches the payload on the first two and on nothing else, and cuts a + short read back on the third. :meth:`to_dict` carries them, so that + ``from_data(info.to_dict())`` rebuilds as ``from_data(info)`` does, while + the dumpers and ``repr`` leave them out. A hand-written :obj:`dict` must + carry them as well -- and, for IPv6, the extension header chain under + ``__exthdr__`` -- to be rebuilt the same way: without the first two, no + payload layer is dispatched, as for a protocol with no next layer, and + without ``__exthdr__`` no IPv6 extension header is rebuilt + (:issue:`2003`). + + """ + + #: Metadata keys that :meth:`to_dict` carries, for a rebuild to read. + __included__ = ['__next_name__', '__next_type__', '__short_read__'] + #: Metadata keys that ``repr``, ``str`` and the dumps leave out. + __excluded__ = ['__next_name__', '__next_type__', '__short_read__'] if TYPE_CHECKING: #: Next field name, i.e., the name of the payload field. @@ -24,8 +43,8 @@ class Data(Info): #: Next field type, i.e., the type of the payload field. __next_type__: 'Type[ProtocolBase]' #: Name of the header field the data ended inside, and how many of its - #: octets were there; present only then. Not excluded, so that - #: :meth:`to_dict` carries it and ``from_data(info.to_dict())`` cuts - #: the rebuild back as ``from_data(info)`` does. See + #: octets were there; present only then. :meth:`to_dict` carries it, + #: so that ``from_data(info.to_dict())`` cuts the rebuild back as + #: ``from_data(info)`` does. See #: :func:`~pcapkit.protocols.schema.schema.keep_short_read`. __short_read__: 'tuple[str, int]' diff --git a/pcapkit/protocols/data/internet/ipv6.py b/pcapkit/protocols/data/internet/ipv6.py index 4181e9e13b..73bd0e61d7 100644 --- a/pcapkit/protocols/data/internet/ipv6.py +++ b/pcapkit/protocols/data/internet/ipv6.py @@ -33,6 +33,9 @@ class IPv6(Protocol): """ + # NOTE: ``__exthdr__`` is metadata, as the keys ``Data`` names are: + # ``to_dict`` carries it, the dumps and ``repr`` leave it out (#2003). + __included__ = ['__exthdr__'] __excluded__ = ['__exthdr__'] #: Version. @@ -65,8 +68,11 @@ class IPv6(Protocol): #: past the Jumbo Payload Length -- present only when there are any. trailer: 'bytes' #: Extension header chain, in wire order, as pairs of the parser - #: class and its info. Kept apart from the per-header keys above, - #: which a repeated header overwrites. + #: class and its info. The per-header keys hold the same infos, but in + #: the order of the keys of this :class:`~pcapkit.corekit.infoclass.Info`, + #: which differs on PyPy, so the chain is rebuilt from this alone; the + #: parser class keeps the IPv6_Ext a header fell back to + #: (:issue:`1473`, :issue:`2003`). __exthdr__: 'tuple[tuple[Type[ProtocolBase], Data], ...]' def __new__(cls, *args: 'Any', **kwargs: 'Any') -> 'IPv6': diff --git a/pcapkit/protocols/data/misc/pcapng.py b/pcapkit/protocols/data/misc/pcapng.py index b5787fc601..0e23025f74 100644 --- a/pcapkit/protocols/data/misc/pcapng.py +++ b/pcapkit/protocols/data/misc/pcapng.py @@ -65,6 +65,11 @@ class PCAPNG(Protocol): """Data model for PCAP-NG file blocks.""" + # NOTE: ``__truncated_raw__`` is a dunder key, which ``to_dict`` carries + # only so named (#2003); the rebuild reads it, and the dumps write it, as + # before. + __included__ = ['__truncated_raw__'] + #: Block type. type: 'Enum_BlockType' #: Block total length. diff --git a/pcapkit/protocols/internet/internet.py b/pcapkit/protocols/internet/internet.py index 8c4f3863bb..dfc7dee394 100644 --- a/pcapkit/protocols/internet/internet.py +++ b/pcapkit/protocols/internet/internet.py @@ -325,7 +325,7 @@ def _rebuilt_exthdr_depth(cls) -> 'int': return _exthdr_rebuilt.get() if issubclass(cls, IPv6_Ext) else 0 @classmethod - def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'ProtocolBase': + def _make_payload(cls, data: 'Data') -> 'ProtocolBase': """Create payload from ``data`` for protocol construction. As :meth:`ProtocolBase._make_payload @@ -338,9 +338,6 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc Args: data: protocol data - exclude: keys of ``data`` that are not the payload, see - :meth:`ProtocolBase._make_payload - ` Returns: Payload for protocol construction. @@ -352,7 +349,7 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc # where it does not parse), so no rebuilt upper layer reads this place. token = _exthdr_rebuilt.set(cls._rebuilt_exthdr_depth() + 1) try: - return super()._make_payload(data, exclude) + return super()._make_payload(data) finally: _exthdr_rebuilt.reset(token) @@ -430,8 +427,9 @@ def _decode_next_layer(self, dict_: '_PT', proto: 'Optional[int]' = None, # pyl Notes: ``dict_`` gains the keys ``__next_type__`` (next layer protocol - type) and ``__next_name__`` (next layer protocol name). Neither - is included when :meth:`Info.to_dict ` is called. + type) and ``__next_name__`` (next layer protocol name), which + :meth:`Info.to_dict ` + carries and the dumpers leave out. """ next_ = cast('ProtocolBase', # type: ignore[redundant-cast] diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index 75122f5a90..38dabf7d69 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -32,8 +32,7 @@ from pcapkit.const.ipv6.extension_header import ExtensionHeader as Enum_ExtensionHeader from pcapkit.const.reg.transtype import TransType as Enum_TransType from pcapkit.corekit.decorators import beholder -from pcapkit.corekit.infoclass import Info -from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict +from pcapkit.corekit.multidict import OrderedMultiDict from pcapkit.corekit.protochain import ProtoChain from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6 from pcapkit.protocols.internet.ip import IP @@ -410,20 +409,21 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over Key-value pairs for protocol construction. """ + # NOTE: The chain is rebuilt from ``__exthdr__`` alone, which + # :meth:`Info.to_dict ` + # carries; without it, no extension header is rebuilt, and none is + # guessed from the keys of ``data``, whose order is not part of the + # contract of an :class:`~pcapkit.corekit.infoclass.Info` and differs + # on PyPy (#2003). exthdr = data.get('__exthdr__') # type: Optional[tuple[tuple[Type[ProtocolBase], Any], ...]] # pylint: disable=line-too-long - exclude = () # type: tuple[str, ...] - if exthdr is None: - # NOTE: ``data`` was rebuilt from :meth:`Info.to_dict`, which leaves - # ``__exthdr__`` out, so the chain is recovered from the keys the - # extension headers were written under; those are no payload. - exclude, exthdr = cls._lookup_exthdr(data) - upper = cls._make_payload(data, exclude=exclude) + chain = tuple(exthdr) if isinstance(exthdr, (tuple, list)) else () + upper = cls._make_payload(data) payload = upper # type: bytes | ProtocolBase # NOTE: The extension headers sit between this header and the upper # layer payload, so they are rebuilt in wire order ahead of it. - if exthdr: - payload = b''.join(proto.from_data(info).data for proto, info in exthdr) + upper.data + if chain: + payload = b''.join(proto.from_data(info).data for proto, info in chain) + upper.data return { 'traffic_class': data['class'], @@ -457,69 +457,6 @@ def _exthdr_key(alias: 'str') -> 'str': """ return alias.lstrip('IPv6-').lower() - @classmethod - def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[str, ...], tuple[tuple[Type[ProtocolBase], Any], ...]]': # pylint: disable=line-too-long - """Recover the extension header chain of ``data`` without ``__exthdr__``. - - :meth:`_decode_next_layer` writes each extension header's info under a - key of its own, in wire order, ahead of the upper layer. Walking the - ``next`` field from :attr:`data.next ` names the parser - of each in turn, exactly as the walk that parsed them did. - - Args: - data: protocol data, without ``__exthdr__`` - - Returns: - The keys the extension headers are under, and the chain in the - shape of ``__exthdr__``, i.e. the parser class and info of each. - - A header repeated in the chain is held under the same key as its - predecessor, and ``data.items(multi=True)`` yields each of them in turn. - - """ - from pcapkit.protocols.internet.ipv6_ext import \ - IPv6_Ext # isort: skip # pylint: disable=import-outside-toplevel - - names = [] # type: list[str] - chain = [] # type: list[tuple[Type[ProtocolBase], Any]] - proto = data.get('next') - nested = cls._lookup_nested(data) - pairs = data.items(multi=True) if isinstance(data, (Info, MultiDict)) else data.items() - for (key, info) in pairs: - # NOTE: ``fragment`` is declared only for type checkers, so it reads - # as nested, but it is the packet split at the Fragment header and - # sits between that header and the next one. - if key not in nested or key == 'fragment': - continue - try: - ex_proto = Enum_ExtensionHeader(proto) - except ValueError: - break - - # NOTE: As in :meth:`_decode_next_layer`, a code with no parser - # carrying ``next`` ends the chain, and so does a Fragment header - # other than a first fragment's. - if 'next' not in info: - break - - # NOTE: A header whose dedicated parser raised was parsed by - # :class:`~pcapkit.protocols.internet.ipv6_ext.IPv6_Ext` instead, - # see :meth:`_import_next_layer`, and written under that class's - # key rather than the dedicated one's. Rebuilding it with the - # dedicated class would look for fields only that class parses - # (#1473), so the key decides the parser. - if key == cls._exthdr_key(IPv6_Ext.alias.fget(None)): # type: ignore[attr-defined] - parser = IPv6_Ext # type: Type[ProtocolBase] - else: - parser = cls._lookup_next_layer(cls.__proto__, int(ex_proto)) - names.append(key) - chain.append((parser, info)) - - proto = info['next'] - if ex_proto == Enum_ExtensionHeader.IPv6_Frag and not cls._is_first_fragment(info): - break - return tuple(names), tuple(chain) - @staticmethod def _is_first_fragment(info: 'Mapping[str, Any]') -> 'bool': """Whether a Fragment header's info is that of a first fragment. @@ -586,7 +523,7 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, hdr_len = self.length # header length raw_len = ipv6.payload if length is None else length # payload length _protos = [] # type: list[ProtocolBase] # ProtoChain buffer - _exthdr = [] # (parser class, info) per extension header + _exthdr = [] # type: list[tuple[Type[ProtocolBase], Any]] # (parser class, info) per extension header split = None # type: Optional[tuple[int, int]] # at first Fragment header opaque = False # walk ended at a non-zero Fragment Offset depth = _frame_depth.get() # layers of the frame up to this one @@ -657,7 +594,7 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, # NOTE: Added through a :class:`MultiDict`, so a repeated header # keeps its predecessor's info rather than overwriting it, and # :meth:`Info.to_dict ` - # writes every one of them, in wire order (#1453). + # writes every one of them (#1453). name = self._exthdr_key(next_.alias) ipv6.__update__(OrderedMultiDict([(name, info)])) @@ -718,7 +655,8 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, 'protocol': proto, # extension header chain in wire order, with the parser of each, - # for ``from_data`` + # for ``from_data``; the parser keeps the IPv6_Ext a header fell + # back to (#1473) '__exthdr__': tuple(_exthdr), }) diff --git a/pcapkit/protocols/misc/pcap/frame.py b/pcapkit/protocols/misc/pcap/frame.py index 11af4bc37d..994da73fa7 100644 --- a/pcapkit/protocols/misc/pcap/frame.py +++ b/pcapkit/protocols/misc/pcap/frame.py @@ -633,8 +633,9 @@ def _decode_next_layer(self, dict_: 'Data_Frame', proto: 'Optional[int]' = None, Notes: We added a new key ``__next_type__`` to ``dict_`` to store the next layer protocol type, and a new key ``__next_name__`` to - store the next layer protocol name. These keys will **NOT** - be included when :meth:`Info.to_dict ` is called. + store the next layer protocol name. :meth:`Info.to_dict + ` carries both, and the + dumpers leave them out. We also added a new key ``protocols`` to ``dict_`` to store the protocol chain of the current packet (frame). diff --git a/pcapkit/protocols/misc/pcapng.py b/pcapkit/protocols/misc/pcapng.py index 40f703f396..3e72a165e8 100644 --- a/pcapkit/protocols/misc/pcapng.py +++ b/pcapkit/protocols/misc/pcapng.py @@ -2142,8 +2142,9 @@ def _decode_next_layer(self, dict_: 'Data_PCAPNG', proto: 'Optional[int]' = None Notes: We added a new key ``__next_type__`` to ``dict_`` to store the next layer protocol type, and a new key ``__next_name__`` to - store the next layer protocol name. These keys will **NOT** - be included when :meth:`Info.to_dict ` is called. + store the next layer protocol name. :meth:`Info.to_dict + ` carries both, and the + dumpers leave them out. We also added a new key ``protocols`` to ``dict_`` to store the protocol chain of the current packet (frame). diff --git a/pcapkit/protocols/protocol.py b/pcapkit/protocols/protocol.py index 7fb13784ed..82f2bcfb21 100644 --- a/pcapkit/protocols/protocol.py +++ b/pcapkit/protocols/protocol.py @@ -59,7 +59,7 @@ from pcapkit.protocols.schema.schema import Schema from pcapkit.utilities import config from pcapkit.utilities.chardet import detect -from pcapkit.utilities.compat import cached_property, get_annotations +from pcapkit.utilities.compat import cached_property from pcapkit.utilities.exceptions import (BaseError, ProtocolError, ProtocolNotFound, ProtocolNotImplemented, RegistryError, StructError, UnsupportedCall, stacklevel) @@ -1129,6 +1129,16 @@ def from_schema(cls, schema: '_ST | dict[str, Any]') -> 'Self': def from_data(cls, data: '_PT | dict[str, Any]', **kwargs: 'Any') -> 'Self': """Create protocol instance from data. + ``data`` may be the :obj:`dict` that :meth:`Info.to_dict + ` returns, which carries the + metadata the rebuild reads: ``__next_name__`` and ``__next_type__``, + which the payload is dispatched on, ``__short_read__``, and, for + :class:`~pcapkit.protocols.internet.ipv6.IPv6`, the extension header + chain under ``__exthdr__``. A hand-written :obj:`dict` must carry them + too, or no payload (nor IPv6 extension header) is rebuilt and no short + read is cut back, see :meth:`_make_payload`; nothing is guessed from + its other keys (:issue:`2003`). + Args: data: Protocol data. **kwargs: Construction keywords that ``data`` does not carry, such as @@ -2100,19 +2110,22 @@ def _make_data(cls, data: 'Data') -> 'dict[str, Any]': return data.to_dict() @classmethod - def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'ProtocolBase': + def _make_payload(cls, data: 'Data') -> 'ProtocolBase': """Create payload from ``data`` for protocol construction. This method uses ``__next_type__`` and ``__next_name__`` to - determine the payload type and name. If either of them is - :data:`None`, a :class:`~pcapkit.protocols.misc.null.NoPayload` - instance will be returned. Otherwise, the payload will be - constructed by :meth:`Protocol.from_data `. - - :meth:`Info.to_dict ` leaves - both keys out, so ``data`` rebuilt from such a :obj:`dict` carries - neither. The payload is then found from the structure of ``data`` - instead, by ``_lookup_payload()``. + determine the payload type and name. If either of them is missing or + :data:`None`, or ``data`` holds nothing under ``__next_name__``, a + :class:`~pcapkit.protocols.misc.null.NoPayload` instance will be + returned, exactly as for a protocol with no next layer. Otherwise, the + payload will be constructed by :meth:`Protocol.from_data + `. + + :meth:`Info.to_dict ` carries + both keys, so ``data`` rebuilt from such a :obj:`dict` dispatches as + the info did. A hand-written :obj:`dict` must carry them too, as it + must ``__short_read__`` and, for IPv6, ``__exthdr__``: the payload is + never guessed from the other keys of ``data`` (:issue:`2003`). The payload is built one layer further into the frame than this protocol, as :meth:`_parse_next_layer` counts it, since each layer built @@ -2124,34 +2137,15 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc Args: data: protocol data - exclude: keys of ``data`` that are not the payload, though they may - look like it, passed on to ``_lookup_payload()`` Returns: Payload for protocol construction. """ - if '__next_type__' not in data and '__next_name__' not in data: - proto, name = cls._lookup_payload(data, exclude) - if proto is None or name is None: - from pcapkit.protocols.misc.null import \ - NoPayload # pylint: disable=import-outside-toplevel - return NoPayload() - token, rebuilding = _frame_depth.set(_frame_depth.get() + 1), _rebuilding.set(True) - try: - return proto.from_data(data[name]) - finally: - _rebuilding.reset(rebuilding) - _frame_depth.reset(token) - proto = cast('Optional[Type[Protocol]]', data.get('__next_type__')) - if proto is None or not (isinstance(proto, type) and issubclass(proto, ProtocolBase)): - from pcapkit.protocols.misc.null import \ - NoPayload # pylint: disable=import-outside-toplevel - return NoPayload() - name = cast('Optional[str]', data.get('__next_name__')) - if name is None: + if (proto is None or not (isinstance(proto, type) and issubclass(proto, ProtocolBase)) + or name is None or name not in data): from pcapkit.protocols.misc.null import \ NoPayload # pylint: disable=import-outside-toplevel return NoPayload() @@ -2163,74 +2157,6 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc _rebuilding.reset(rebuilding) _frame_depth.reset(token) - @classmethod - def _lookup_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'tuple[Optional[Type[ProtocolBase]], Optional[str]]': # pylint: disable=line-too-long - """Find the payload of ``data`` that carries no ``__next_type__``. - - :meth:`_decode_next_layer` writes the payload's info under the payload's - :attr:`info_name`, after every field of the data model, so the payload is - the *last* key that the data model does not declare and whose value is a - mapping. Its protocol is the one of :attr:`__proto__` -- or its fallback - -- whose :attr:`info_name` the key is. That is the lowercased class name - by default, but a protocol may override it: every version of - :class:`~pcapkit.protocols.link.l2tp.L2TP` is ``l2tp``. - - Args: - data: protocol data - exclude: keys of ``data`` to pass over, such as the IPv6 extension - headers, which are written like a payload but are not one - - Returns: - The payload protocol and the key of its info in ``data``, or - :data:`None` for both if ``data`` has no payload this protocol can - dispatch to. - - """ - keys = [key for key in cls._lookup_nested(data) if key not in exclude] - if not keys: - return None, None - name = keys[-1] - - registry = cls.__proto__ - candidates = list(registry.values()) - if registry.default_factory is not None: - candidates.append(registry.default_factory()) - for entry in candidates: - proto = entry.klass if isinstance(entry, ModuleDescriptor) else entry - if not (isinstance(proto, type) and issubclass(proto, ProtocolBase)): - continue - - # NOTE: :attr:`info_name` is an instance property, so it is read off - # a bare instance; ``__new__`` only sets up the per-instance caches. - # An abstract class cannot be instantiated, and is never a payload. - try: - label = proto.__new__(proto).info_name - except TypeError: - continue - if label == name: - return proto, name - return None, None - - @staticmethod - def _lookup_nested(data: 'Data') -> 'list[str]': - """Keys of ``data`` that its data model does not declare and that hold a mapping. - - These are where the next layer protocols wrote their info, in the order - they wrote it -- the payload, and the IPv6 extension headers. - - Args: - data: protocol data - - Returns: - The keys, in the order of ``data``. - - """ - declared = set() # type: set[str] - for klass in type(data).mro(): - declared.update(get_annotations(klass)) - return [key for key in data - if key not in declared and isinstance(data[key], collections.abc.Mapping)] - @staticmethod def _lookup_registry(registry: 'DefaultDict[Any, _VT]', code: 'Any') -> '_VT': """Look up a dispatch :term:`registry` entry without recording a miss. @@ -2328,9 +2254,10 @@ def _decode_next_layer(self, dict_: '_PT', proto: 'int', length: 'Optional[int]' Notes: We added a new key ``__next_type__`` to ``dict_`` to store the next layer protocol type, and a new key ``__next_name__`` to - store the next layer protocol name. These keys will **NOT** - be included when :meth:`Info.to_dict ` is called; - ``_lookup_payload()`` finds the payload of such a :obj:`dict` instead. + store the next layer protocol name. :meth:`Info.to_dict + ` carries both, so that + :meth:`_make_payload` dispatches a :obj:`dict` rebuild on them, + while the dumpers and ``repr`` leave them out (:issue:`2003`). """ next_ = cast('ProtocolBase', self._import_next_layer(proto, length, packet=packet)) # type: ignore[misc,call-arg,redundant-cast] diff --git a/tests/corekit/test_info_roundtrip_unit.py b/tests/corekit/test_info_roundtrip_unit.py index 374e389f79..36ce428db8 100644 --- a/tests/corekit/test_info_roundtrip_unit.py +++ b/tests/corekit/test_info_roundtrip_unit.py @@ -329,6 +329,9 @@ class Leaf(Info): class Node(Info): """Nested, optional, renamed and plain keys together.""" + # NOTE: ``to_dict`` carries a dunder key only if named here (#2003). + __included__ = ['__short_read__', '__truncated_raw__'] + leaf: Leaf maybe: Optional[Leaf] items: int diff --git a/tests/corekit/test_infoclass.py b/tests/corekit/test_infoclass.py index 41e8a41b72..60f7e11cc8 100644 --- a/tests/corekit/test_infoclass.py +++ b/tests/corekit/test_infoclass.py @@ -62,20 +62,48 @@ class Bag(Info): bag = Bag({'items': 'conflict', 'visible': 1, 'hidden': 2}, extra=3) self.assertEqual(bag['items'], 'conflict') - self.assertEqual(list(bag), ['items', 'visible', 'extra']) + # NOTE: GitHub issue #2003 -- ``__excluded__`` keys are left out of + # ``repr`` and ``str`` only; iteration and ``to_dict`` carry them. + self.assertEqual(list(bag), ['items', 'visible', 'hidden', 'extra']) # NOTE: GitHub issue #1296 -- ``len`` counts exactly the keys - # iteration yields, so the ``__excluded__`` ``hidden`` key and the - # internal bookkeeping attributes are not counted. - self.assertEqual(len(bag), 3) + # iteration yields, so the internal bookkeeping attributes are not + # counted. + self.assertEqual(len(bag), 4) self.assertEqual(str(bag), 'Bag(items=conflict, visible=1, extra=3)') self.assertIn('items=', repr(bag)) - self.assertEqual(list(bag.to_dict().items(multi=True)), [('items', 'conflict'), ('visible', 1), ('extra', 3)]) + self.assertNotIn('hidden', repr(bag)) + self.assertEqual(list(bag.to_dict().items(multi=True)), + [('items', 'conflict'), ('visible', 1), ('hidden', 2), ('extra', 3)]) with self.assertRaises(UnsupportedCall): bag.visible = 9 with self.assertRaises(UnsupportedCall): del bag.visible + def test_to_dict_carries_only_the_included_dunder_keys(self) -> None: + """GitHub issue #2003: a dunder key is metadata, carried only if ``__included__`` names it.""" + from pcapkit.corekit.infoclass import Info + + class Plain(Info): + pass + + class Meta(Info): + __included__ = ['__meta__'] + + class More(Meta): + __included__ = ['__more__'] + + self.assertEqual(More.__included__, ['__more__', '__meta__']) + pairs = {'a': 1, '__meta__': 2, '__more__': 3} + for cls, kept in ((Plain, ['a']), (Meta, ['a', '__meta__']), (More, ['a', '__meta__', '__more__'])): + with self.subTest(cls=cls.__name__): + info = cls.from_dict(pairs) + self.assertEqual([key for key, _ in info.to_dict().items(multi=True)], kept) + self.assertEqual(list(info), kept) + self.assertEqual(len(info), len(kept)) + # Still held, and read by indexing. + self.assertEqual(info['__meta__'], 2) + def test_from_dict_iterables_nested_info_and_inherited_metadata(self) -> None: from pcapkit.corekit.infoclass import Info @@ -100,8 +128,10 @@ class Child(Base): converted = outer.to_dict() self.assertEqual(list(converted['inner'].items(multi=True)), [('answer', 42)]) self.assertEqual(converted['kept'], 'yes') - self.assertNotIn('base_hidden', converted) - self.assertNotIn('child_hidden', converted) + # Carried by ``to_dict``, left out of ``repr`` (#2003). + self.assertEqual(converted['base_hidden'], 'skip') + self.assertEqual(converted['child_hidden'], 'skip') + self.assertNotIn('hidden', repr(outer)) self.assertEqual(Info.from_dict({'a': 1}, b=2).to_dict().to_dict(), {'a': 1, 'b': 2}) self.assertEqual(Info.from_dict(c=3).to_dict().to_dict(), {'c': 3}) @@ -193,11 +223,9 @@ class BareChild(Bare): def test_annotations_are_read_through_the_compat_shim(self) -> None: """GitHub issue #1709: one version-gated reader, from :mod:`pcapkit.utilities.compat`.""" from pcapkit.corekit import infoclass - from pcapkit.protocols import protocol from pcapkit.utilities import compat self.assertIs(infoclass.get_annotations, compat.get_annotations) - self.assertIs(protocol.get_annotations, compat.get_annotations) def test_the_pre_3_10_reader_gives_the_same_signatures(self) -> None: # NOTE: a stand-in for ``compat.get_annotations`` below 3.10 -- the diff --git a/tests/corekit/test_multiinfo_1484_unit.py b/tests/corekit/test_multiinfo_1484_unit.py index 736f775677..ef8efb2fdf 100644 --- a/tests/corekit/test_multiinfo_1484_unit.py +++ b/tests/corekit/test_multiinfo_1484_unit.py @@ -408,11 +408,16 @@ def test_to_dict_returns_a_plain_ordered_multi_dict_at_every_level(self) -> None exported = info.to_dict() self.assertIs(type(exported), _OrderedMultiDict) self.assertIsInstance(exported, OrderedMultiDict) - self.assertEqual([key for key, _ in exported.items(multi=True)], ['nested', 'ordered', 'grouped', 'nested']) - self.assertEqual([type(value) for _, value in exported.items(multi=True)], - [_OrderedMultiDict, _OrderedMultiDict, MultiDict, _OrderedMultiDict]) + # NOTE: The order across keys is that of the Info's own keys, which is + # not part of its contract and differs on PyPy (#2003); the values of + # one key keep theirs. + self.assertEqual(sorted(key for key, _ in exported.items(multi=True)), ['grouped', 'nested', 'nested', 'ordered']) + self.assertEqual({key: [type(value) for value in exported.getlist(key)] for key in exported}, + {'nested': [_OrderedMultiDict, _OrderedMultiDict], 'ordered': [_OrderedMultiDict], + 'grouped': [MultiDict]}) self.assertEqual([value['x'] for value in exported.getlist('nested')], [1, 2]) self.assertEqual(list(exported['ordered'].items(multi=True)), PAIRS) + self.assertEqual(list(exported['grouped'].items(multi=True)), list(MultiDict(PAIRS).items(multi=True))) # The export is a copy the caller may edit. exported['ordered'].add('z', 9) diff --git a/tests/dumpkit/test_capture_dump_runtime.py b/tests/dumpkit/test_capture_dump_runtime.py index 5d426cbe84..17b4c52887 100644 --- a/tests/dumpkit/test_capture_dump_runtime.py +++ b/tests/dumpkit/test_capture_dump_runtime.py @@ -170,7 +170,9 @@ def test_json_frame_sections_carry_every_info_key_and_the_chain(self) -> None: for number, frame in enumerate(extractor.frame, start=1): section = report[f'Frame {number}'] with self.subTest(capture=name, frame=number): - self.assertEqual(list(section), list(frame.info.to_dict())) + # Every key but those the dumps leave out (#2003). + self.assertEqual(list(section), [key for key in frame.info.to_dict() + if key not in frame.info.__excluded__]) self.assertEqual(section['protocols'], str(frame.protochain)) def test_plist_dump_reads_back_through_plistlib(self) -> None: diff --git a/tests/dumpkit/test_repeated_field_dump_1484_unit.py b/tests/dumpkit/test_repeated_field_dump_1484_unit.py index fd5f26d61e..f3abbf2fdc 100644 --- a/tests/dumpkit/test_repeated_field_dump_1484_unit.py +++ b/tests/dumpkit/test_repeated_field_dump_1484_unit.py @@ -120,7 +120,8 @@ def test_the_writer_takes_the_info_or_its_export(self) -> None: tops.append(json.load(file)['Frame 1']) self.assertEqual(pads(tops[0]['ethernet']['ipv6']['opts']), [f'000000{mark:02x}' for mark in MARKS]) self.assertEqual(list(tops[1]), list(tops[0])) - self.assertEqual(list(tops[0]), list(frame.info.to_dict())) + # Every key but those the dumps leave out (#2003). + self.assertEqual(list(tops[0]), [key for key in frame.info.to_dict() if key not in frame.info.__excluded__]) if __name__ == '__main__': diff --git a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py index ba3124476d..6dc492d5d6 100644 --- a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py +++ b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py @@ -112,7 +112,7 @@ def test_dict_rebuild_keeps_the_generic_parser(self) -> None: with self.subTest(cut=cut): ip6 = IPv6(data, len(data)) self.assertIs(type(ip6._exthdr[ExtensionHeader.HOPOPT]), IPv6_Ext) - _, chain = IPv6._lookup_exthdr(ip6.info.to_dict()) + chain = ip6.info.to_dict()['__exthdr__'] self.assertEqual([parser for parser, _ in chain], [IPv6_Ext]) self.assertEqual(IPv6.from_data(ip6.info.to_dict()).data, data) diff --git a/tests/protocols/internet/test_ipv6_from_data_ext_unit.py b/tests/protocols/internet/test_ipv6_from_data_ext_unit.py index 19ffcaa589..d7b930f5d6 100644 --- a/tests/protocols/internet/test_ipv6_from_data_ext_unit.py +++ b/tests/protocols/internet/test_ipv6_from_data_ext_unit.py @@ -82,13 +82,18 @@ def test_repeated_header_keeps_both_instances(self) -> None: ) self.assertEqual(IPv6.from_data(parsed.info).data, packet) - def test_chain_is_hidden_from_info_views(self) -> None: + def test_chain_is_carried_by_to_dict_and_hidden_from_repr(self) -> None: from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.internet.ipv6_route import IPv6_Route packet = PACKETS['routing'] info = IPv6(packet, len(packet)).info - self.assertNotIn('__exthdr__', list(info)) - self.assertNotIn('__exthdr__', info.to_dict()) + self.assertNotIn('__exthdr__', repr(info)) + self.assertNotIn('__exthdr__', str(info)) + # The parser and the info of each, as ``to_dict`` carries it. + exported = info.to_dict()['__exthdr__'] + self.assertEqual(exported, info.__exthdr__) + self.assertEqual([parser for parser, _ in exported], [IPv6_Route]) if __name__ == '__main__': diff --git a/tests/protocols/test_lookup_by_name_2003_unit.py b/tests/protocols/test_lookup_by_name_2003_unit.py new file mode 100644 index 0000000000..1f3bd7f9d3 --- /dev/null +++ b/tests/protocols/test_lookup_by_name_2003_unit.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +"""A dict rebuild dispatches as the info's, whatever the key order. + +GitHub issue #2003: on PyPy, an :class:`~pcapkit.corekit.infoclass.Info` +lays its keys out as an earlier instance of the same class did, so its key +order is not CPython's. Two lookups of a rebuild from ``to_dict()`` read +structure off that order: the payload was taken to be the *last* nested key, +and the IPv6 extension header chain was walked in key order. A cut frame +rebuilt short, and so did a Hop-by-Hop, Destination, Destination chain once +``hopopt`` had moved past the upper layer. + +The order of an :class:`~pcapkit.corekit.infoclass.Info` is not part of its +contract, so nothing is read off it any more: ``to_dict()`` carries the +``__next_type__`` and ``__next_name__`` metadata the payload is dispatched on, +and ``__exthdr__``, the extension header chain as a tuple of parser and info +pairs in wire order. A dict without the metadata is not guessed at. These tests reorder an +instance's :attr:`~object.__dict__` as PyPy does, so CPython checks the same +thing. + +Every case builds its own octets and reads no capture. Classes are imported +inside each test, after :func:`~tests._support.reimport_once_per_class`. + +""" + +import copy +import struct +import unittest +import warnings + +from tests._support import reimport_once_per_class + +#: Test tier (#1958): a hand-picked round trip, pinning that a rebuild does not +#: depend on key order, which differs by interpreter. +__tier__ = 'ALL' + +HOP, DST, FRAG, UDP = 0, 60, 44, 17 +SRC6 = bytes.fromhex('20010db8' + '00' * 11 + '01') +DST6 = bytes.fromhex('20010db8' + '00' * 11 + '02') +DATAGRAM = struct.pack('!HHHH', 40000, 40000, 12, 0) + b'ping' + + +def options(nxt: int, mark: int) -> bytes: + """An 8-octet options header carrying a PadN; ``mark`` tells repeats apart.""" + return bytes([nxt, 0, 1, 4, 0, 0, 0, mark]) + + +def ipv6(first: int, body: bytes) -> bytes: + """An IPv6 header whose Next Header is ``first``, followed by ``body``.""" + return struct.pack('!IHBB16s16s', 6 << 28, len(body), first, 64, SRC6, DST6) + body + + +#: Hop-by-Hop, Destination, Destination, then UDP. +HOP_DST_DST = ipv6(HOP, options(DST, 0) + options(DST, 1) + options(UDP, 2) + DATAGRAM) + +#: An atomic Fragment header -- offset zero, no more fragments -- then UDP, so +#: the walk goes on to the upper layer and ``fragment`` is held beside it. +ATOMIC = ipv6(FRAG, struct.pack('!BBHI', UDP, 0, 0, 0x2003) + DATAGRAM) + + +def moved(info, key: str, before=None): # type: ignore[no-untyped-def] + """A copy of ``info`` with ``key`` moved before ``before``, or last, in its ``__dict__``. + + That is the reordering PyPy makes when an earlier instance of the class + laid its attributes out differently. PyPy may lay the copy out its own way + again, so the order is asserted of nothing but the rebuild. + + """ + clone = copy.copy(info) + layout = clone.__dict__ + value = layout.pop(key) + items = list(layout.items()) + index = len(items) if before is None else [name for name, _ in items].index(before) + items.insert(index, (key, value)) + layout.clear() + layout.update(items) + return clone + + +class LookupByNameTests(unittest.TestCase): + """Pin that the dict rebuild reads no structure off the key order.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + quiet = warnings.catch_warnings() + quiet.__enter__() # pylint: disable=unnecessary-dunder-call + self.addCleanup(quiet.__exit__, None, None, None) + warnings.simplefilter('ignore') + + def test_extension_headers_follow_exthdr_whatever_the_key_order(self) -> None: + from pcapkit.protocols.internet.hopopt import HOPOPT + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts + + ip6 = IPv6(HOP_DST_DST) + self.assertEqual(len(ip6.data), len(HOP_DST_DST)) + + # As PyPy laid it out: ``hopopt`` after every other key; and the + # chain first. + for key, before in (('hopopt', None), ('__exthdr__', 'version')): + with self.subTest(moved=key): + exported = moved(ip6.info, key, before=before).to_dict() + chain = exported['__exthdr__'] + self.assertIsInstance(chain, tuple) + self.assertEqual([parser for parser, _ in chain], [HOPOPT, IPv6_Opts, IPv6_Opts]) + self.assertEqual([info for _, info in chain], + [ext.info for _, ext in ip6.extension_headers.items(multi=True)]) + self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) + + def test_payload_follows_the_metadata_whatever_the_key_order(self) -> None: + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.transport.udp import UDP as UDP_ + + ip6 = IPv6(ATOMIC) + self.assertIn('fragment', ip6.info) + self.assertEqual(len(ip6.data), len(ATOMIC)) + + # ``fragment`` is a mapping too, and not the payload; laid out after + # the payload, it is the last of them, and the metadata first. + info = moved(moved(ip6.info, 'udp', before='fragment'), '__next_name__', before='version') + exported = info.to_dict() + self.assertEqual(exported['__next_name__'], 'udp') + self.assertIs(exported['__next_type__'], UDP_) + rebuilt = IPv6.from_data(exported) + self.assertIsInstance(rebuilt.payload, UDP_) + self.assertEqual(rebuilt.data, ATOMIC) + + def test_cut_upper_layer_is_found_ahead_of_the_headers(self) -> None: + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.misc.raw import Raw + from pcapkit.utilities import config + + limit = config.EXTENSION_HEADER_LIMIT + self.addCleanup(setattr, config, 'EXTENSION_HEADER_LIMIT', limit) + config.EXTENSION_HEADER_LIMIT = 2 + + # The third header is cut, and kept raw as the upper layer. + ip6 = IPv6(HOP_DST_DST) + self.assertIn('raw', ip6.info) + self.assertEqual(len(ip6.data), len(HOP_DST_DST)) + + # As on PyPy for #1610: ``opts`` after the upper layer. + info = moved(moved(ip6.info, 'raw', before='opts'), 'hopopt', before='raw') + exported = info.to_dict() + self.assertEqual(len(exported['__exthdr__']), 2) + rebuilt = IPv6.from_data(exported) + self.assertIsInstance(rebuilt.payload, Raw) + self.assertEqual(rebuilt.data, HOP_DST_DST) + + def test_dict_without_the_dispatch_keys_is_not_guessed_at(self) -> None: + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.misc.null import NoPayload + + ip6 = IPv6(HOP_DST_DST) + exported = ip6.info.to_dict() + for key in ('__next_type__', '__next_name__'): + exported.poplist(key) + self.assertIn('udp', exported) + + # The extension headers are rebuilt from ``__exthdr__``, and the + # upper layer, named by nothing, is not. + rebuilt = IPv6.from_data(exported) + self.assertIsInstance(rebuilt.payload, NoPayload) + self.assertEqual(rebuilt.data, HOP_DST_DST[:40 + 3 * 8]) + + def test_dict_without_exthdr_rebuilds_no_extension_header(self) -> None: + from pcapkit.protocols.internet.ipv6 import IPv6 + + ip6 = IPv6(HOP_DST_DST) + exported = ip6.info.to_dict() + exported.poplist('__exthdr__') + self.assertIn('hopopt', exported) + self.assertEqual(len(exported.getlist('opts')), 2) + + # The headers are still there under their keys, but nothing names + # the chain, so only the upper layer follows the IPv6 header. + self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST[:40] + DATAGRAM) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/test_short_read_roundtrip_unit.py b/tests/protocols/test_short_read_roundtrip_unit.py index 3ea9ce238a..b5af8cc7c8 100644 --- a/tests/protocols/test_short_read_roundtrip_unit.py +++ b/tests/protocols/test_short_read_roundtrip_unit.py @@ -189,10 +189,36 @@ def test_record_rides_in_to_dict_only_when_cut(self) -> None: again = Ethernet.from_data(cut.info.to_dict()) self.assertEqual(again.info['__short_read__'], ('type', 1)) self.assertEqual(Ethernet.from_data(again.info.to_dict()).data, bytes(13)) - # a dumper writes the record as a list, which replays the same + # a hand-written dict may carry the record as a list, which replays + # the same record = dict(cut.info.to_dict(), __short_read__=['type', 1]) self.assertEqual(Ethernet.from_data(record).data, bytes(13)) + def test_record_is_left_out_of_repr_and_dumps(self) -> None: + import json + import os + import tempfile + + import dictdumper.json + + from pcapkit.dumpkit.common import make_dumper + from pcapkit.protocols.link.ethernet import Ethernet + + # GitHub issue #2003: metadata, which ``__excluded__`` names. + info = Ethernet(bytes(13), 13).info + self.assertIn('__short_read__', info.to_dict()) + self.assertNotIn('__short_read__', repr(info)) + self.assertNotIn('__short_read__', str(info)) + with tempfile.TemporaryDirectory() as tmp: + for index, value in enumerate((info, info.to_dict())): + with self.subTest(dumped=type(value).__name__): + path = os.path.join(tmp, f'{index}.json') + make_dumper(dictdumper.json.JSON)(path)(value, name='Ethernet') + with open(path, encoding='utf-8') as file: + dumped = json.load(file)['Ethernet'] + self.assertNotIn('__short_read__', dumped) + self.assertIn('dst', dumped) + class TestSchemaShortRead(unittest.TestCase): """Pin the schema-level record and the pack that honours it.""" diff --git a/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py b/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py index edc9cfd4e0..afb72a88bd 100644 --- a/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py +++ b/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py @@ -8,8 +8,9 @@ its 14-octet header. IPv6 lost its extension headers the same way, with ``__exthdr__``, and HTTP raised ``ProtocolError('invalid HTTP data: Raw')``. -The dict itself is left as it was, since the dumps are written from it; the -payload is found from the dict's structure instead. +``to_dict`` now carries both keys, so the dict rebuild dispatches exactly as +the info's, and the dumps leave every dunder key out (#2003). A dict without +them is not guessed at: no payload is rebuilt. Every case builds its own octets and reads no capture. Classes are imported inside each test, after :func:`~tests._support.reimport_once_per_class`. @@ -135,7 +136,7 @@ def test_ipv6_extension_headers_rebuild_from_its_dict(self) -> None: ipv6 = self.frame('ipv6-exthdr-udp')[IPv6] dict_ = ipv6.info.to_dict() - self.assertNotIn('__exthdr__', dict_) + self.assertIn('__exthdr__', dict_) rebuilt = IPv6.from_data(dict_) self.assertEqual(rebuilt.data, ipv6.data) @@ -144,7 +145,7 @@ def test_ipv6_extension_headers_rebuild_from_its_dict(self) -> None: ['HOPOPT', 'IPv6_Opts'], ) - def test_payload_is_found_by_info_name_not_class_name(self) -> None: + def test_payload_is_found_under_its_info_name(self) -> None: from pcapkit.protocols.link.l2tpv2 import L2TPv2 from pcapkit.protocols.transport.udp import UDP @@ -178,15 +179,59 @@ def test_dict_without_a_payload_key_rebuilds_the_header_alone(self) -> None: self.assertIsInstance(rebuilt.payload, NoPayload) self.assertEqual(rebuilt.data, FRAMES['ipv4-udp'][:14]) - def test_to_dict_still_leaves_the_dispatch_keys_out(self) -> None: - # The dumps are written from ``to_dict``, so the fix must not add to it. + def test_dict_without_the_dispatch_keys_rebuilds_the_header_alone(self) -> None: + from pcapkit.protocols.link.ethernet import Ethernet + from pcapkit.protocols.misc.null import NoPayload + + # The payload is still there under its name, but nothing names it. + eth = Ethernet(FRAMES['ipv4-udp']) + dict_ = eth.info.to_dict() + for key in ('__next_type__', '__next_name__'): + dict_.poplist(key) + self.assertIn('ipv4', dict_) + + rebuilt = Ethernet.from_data(dict_) + self.assertIsInstance(rebuilt.payload, NoPayload) + self.assertEqual(rebuilt.data, FRAMES['ipv4-udp'][:14]) + + def test_repr_and_str_leave_the_dispatch_keys_out(self) -> None: + for name in FRAMES: + info = self.frame(name).info + self.assertIn('__next_type__', info) + with self.subTest(frame=name): + self.assertNotIn('__next', repr(info)) + self.assertNotIn('__next', str(info)) + + def test_to_dict_carries_the_dispatch_keys_and_a_dump_none(self) -> None: + import json + import os + import tempfile + + import dictdumper.json + + from pcapkit.dumpkit.common import make_dumper + for name in FRAMES: + self.assertIn('__next_type__', self.frame(name).info.to_dict()) for layer in self.layers(name): with self.subTest(frame=name, layer=type(layer).__name__): dict_ = layer.info.to_dict() - for key in ('__next_type__', '__next_name__', '__exthdr__'): - self.assertNotIn(key, dict_) - + for key in ('__next_type__', '__next_name__'): + self.assertEqual(key in dict_, key in layer.info) + if key in dict_: + self.assertIs(dict_[key], layer.info[key]) + + frame = self.frame(name) + with tempfile.TemporaryDirectory() as tmp: + for index, value in enumerate((frame.info, frame.info.to_dict())): + with self.subTest(frame=name, dumped=type(value).__name__): + path = os.path.join(tmp, f'{index}.json') + make_dumper(dictdumper.json.JSON)(path)(value, name='Frame 1') + with open(path, encoding='utf-8') as file: + text = file.read() + json.loads(text) + for key in ('__next_type__', '__next_name__', '__short_read__', '__exthdr__'): + self.assertNotIn(f'"{key}"', text) if __name__ == '__main__': unittest.main() diff --git a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py index 6a1b89d082..24fb5b6b8b 100644 --- a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py +++ b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py @@ -7,9 +7,11 @@ Destination Options, Destination Options, UDP rebuilt 56 of 64 octets. ``to_dict`` returns an :class:`~pcapkit.corekit.multidict.OrderedMultiDict` -(#1484): indexing sees the first value of each key, and ``items(multi=True)`` -yields every header in wire order for the rebuild. The dumpers write every -header too. +(#1484): indexing sees the first value of each key, and ``getlist`` gives every +header of a type in wire order. The order across types is that of the keys of +the :class:`~pcapkit.corekit.infoclass.Info`, which is not part of its contract +and differs on PyPy, so the rebuild walks the ``next`` fields instead (#2003). +The dumpers write every header too. Every case builds its own octets and reads no capture. Classes are imported inside each test, after :func:`~tests._support.reimport_once_per_class`. @@ -133,7 +135,7 @@ def test_rebuilt_chain_has_the_parsed_headers(self) -> None: self.assertEqual([ext.data for ext in rebuilt.extension_headers.values()], [ext.data for ext in ip6.extension_headers.values()]) - def test_to_dict_keeps_every_header_in_wire_order(self) -> None: + def test_to_dict_keeps_every_header_of_a_type_in_wire_order(self) -> None: from pcapkit.corekit.multidict import OrderedMultiDict from pcapkit.protocols.internet.ipv6 import IPv6 @@ -143,8 +145,10 @@ def test_to_dict_keeps_every_header_in_wire_order(self) -> None: dict_ = ip6.info.to_dict() self.assertIsInstance(dict_, OrderedMultiDict) - parsed = [canonical(ext.info.to_dict()) for ext in ip6.extension_headers.values()] - held = [canonical(val) for _, val in dict_.items(multi=True) if canonical(val) in parsed] + parsed = {} # type: dict[str, list[object]] + for _, ext in ip6.extension_headers.items(multi=True): + parsed.setdefault(IPv6._exthdr_key(ext.alias), []).append(canonical(ext.info.to_dict())) + held = {key: [canonical(val) for val in dict_.getlist(key)] for key in parsed} self.assertEqual(held, parsed) # Indexing sees the first of each.