From 0ae6a546006eda0f8bb377215c05edc66d02e824 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 21 Oct 2026 00:44:29 -0400 Subject: [PATCH 1/6] fix(protocols): find the payload by name and the IPv6 chain by next (#2003) The order of an Info's keys is not part of its contract, and on PyPy it follows the attribute layout of an earlier instance of the same class. Two rebuilds from to_dict() read structure off that order. - Protocol._lookup_payload took the last nested key as the payload. It now takes the nested key that is the info_name of a protocol in __proto__ (or its fallback), wherever it sits. - IPv6._lookup_exthdr walked the nested keys in order. It now follows the next fields from data.next, and each step takes the first header not yet taken that is held under the key the step's parser writes (new helper IPv6._exthdr_parser; HIP's version-dependent alias is read off the info). - The repeated-exthdr and MultiInfo tests no longer pin the order across keys; the values of one key keep their order. - New test reorders an instance's __dict__ as PyPy does, so CPython covers it. On CPython both lookups answer every call over the sample captures exactly as before. PyPy 8.0.1: the 1453, 1610 and 1484 tests now pass. --- pcapkit/protocols/internet/ipv6.py | 114 ++++++++++---- pcapkit/protocols/protocol.py | 31 ++-- tests/corekit/test_multiinfo_1484_unit.py | 11 +- .../test_lookup_by_name_2003_unit.py | 144 ++++++++++++++++++ .../test_to_dict_repeated_exthdr_1453_unit.py | 16 +- 5 files changed, 266 insertions(+), 50 deletions(-) create mode 100644 tests/protocols/test_lookup_by_name_2003_unit.py diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index 75122f5a90..8835b83d2a 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -26,7 +26,10 @@ .. [*] https://en.wikipedia.org/wiki/IPv6_packet """ +import collections.abc +import inspect import ipaddress +import types from typing import TYPE_CHECKING from pcapkit.const.ipv6.extension_header import ExtensionHeader as Enum_ExtensionHeader @@ -462,9 +465,19 @@ def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[str, ...], tuple[tupl """Recover the extension header chain of ``data`` without ``__exthdr__``. :meth:`_decode_next_layer` writes each extension header's info under a - key of its own, in wire order, ahead of the upper layer. Walking the - ``next`` field from :attr:`data.next ` names the parser - of each in turn, exactly as the walk that parsed them did. + key of its own. Walking the ``next`` field from :attr:`data.next + ` names the parser of each in turn, exactly as the walk + that parsed them did, and each step takes the first header not yet + taken that is held under the key that parser writes. + + So the chain comes from the ``next`` fields, and never from the order + of the keys of ``data``: that order is not part of the contract of an + :class:`~pcapkit.corekit.infoclass.Info`, and on PyPy it follows the + attribute layout of an earlier instance of the same class, which put + the Hop-by-Hop Options header after the upper layer (:issue:`2003`). + A header repeated in the chain is held under the same key as its + predecessor, and the headers held under one key are taken in the order + ``items(multi=True)`` yields them (:issue:`1453`). Args: data: protocol data, without ``__exthdr__`` @@ -473,24 +486,20 @@ def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[str, ...], tuple[tupl The keys the extension headers are under, and the chain in the shape of ``__exthdr__``, i.e. the parser class and info of each. - A header repeated in the chain is held under the same key as its - predecessor, and ``data.items(multi=True)`` yields each of them in turn. - """ - from pcapkit.protocols.internet.ipv6_ext import \ - IPv6_Ext # isort: skip # pylint: disable=import-outside-toplevel + nested = cls._lookup_nested(data) + pairs = data.items(multi=True) if isinstance(data, (Info, MultiDict)) else data.items() + # NOTE: ``fragment`` is declared only for type checkers, so it reads as + # nested, but it is the packet split at the Fragment header and sits + # between that header and the next one. + pool = [(key, info) for (key, info) in pairs + if key in nested and key != 'fragment'] # type: list[tuple[str, Any]] + taken = [False] * len(pool) names = [] # type: list[str] chain = [] # type: list[tuple[Type[ProtocolBase], Any]] proto = data.get('next') - nested = cls._lookup_nested(data) - pairs = data.items(multi=True) if isinstance(data, (Info, MultiDict)) else data.items() - for (key, info) in pairs: - # NOTE: ``fragment`` is declared only for type checkers, so it reads - # as nested, but it is the packet split at the Fragment header and - # sits between that header and the next one. - if key not in nested or key == 'fragment': - continue + while True: try: ex_proto = Enum_ExtensionHeader(proto) except ValueError: @@ -498,20 +507,19 @@ def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[str, ...], tuple[tupl # NOTE: As in :meth:`_decode_next_layer`, a code with no parser # carrying ``next`` ends the chain, and so does a Fragment header - # other than a first fragment's. - if 'next' not in info: + # other than a first fragment's. Nor does the walk go on past a + # code none of the headers left is held under -- the upper layer + # kept raw at a limit, say. + index, parser = -1, None # type: int, Optional[Type[ProtocolBase]] + for (index, (key, info)) in enumerate(pool): + if not taken[index]: + parser = cls._exthdr_parser(key, info, ex_proto) + if parser is not None: + break + if parser is None: break - - # NOTE: A header whose dedicated parser raised was parsed by - # :class:`~pcapkit.protocols.internet.ipv6_ext.IPv6_Ext` instead, - # see :meth:`_import_next_layer`, and written under that class's - # key rather than the dedicated one's. Rebuilding it with the - # dedicated class would look for fields only that class parses - # (#1473), so the key decides the parser. - if key == cls._exthdr_key(IPv6_Ext.alias.fget(None)): # type: ignore[attr-defined] - parser = IPv6_Ext # type: Type[ProtocolBase] - else: - parser = cls._lookup_next_layer(cls.__proto__, int(ex_proto)) + key, info = pool[index] + taken[index] = True names.append(key) chain.append((parser, info)) @@ -520,6 +528,54 @@ def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[str, ...], tuple[tupl break return tuple(names), tuple(chain) + @classmethod + def _exthdr_parser(cls, key: 'str', info: 'Any', code: 'Enum_ExtensionHeader') -> 'Optional[Type[ProtocolBase]]': # pylint: disable=line-too-long + """Parser of the extension header ``code`` names, if ``key`` holds it as ``info``. + + Args: + key: key of ``info`` in :class:`Data_IPv6` + info: info of an extension header, as parsed or from + :meth:`Info.to_dict ` + code: extension header code the walk dispatches on + + Returns: + The parser to rebuild ``info`` with, or :data:`None` if ``info`` + under ``key`` is not the header ``code`` names. + + """ + from pcapkit.protocols.internet.ipv6_ext import \ + IPv6_Ext # isort: skip # pylint: disable=import-outside-toplevel + + if not isinstance(info, collections.abc.Mapping) or 'next' not in info: + return None + + # NOTE: A header whose dedicated parser raised was parsed by + # :class:`~pcapkit.protocols.internet.ipv6_ext.IPv6_Ext` instead, see + # :meth:`_import_next_layer`, and written under that class's key rather + # than the dedicated one's, with the code it stands in for as its + # ``protocol``. Rebuilding it with the dedicated class would look for + # fields only that class parses (#1473), so the key decides the parser. + if key == cls._exthdr_key(IPv6_Ext.alias.fget(None)): # type: ignore[attr-defined] + stands_for = info.get('protocol') + return IPv6_Ext if stands_for is None or stands_for == code else None + + parser = cls._lookup_next_layer(cls.__proto__, int(code)) + # NOTE: The key is the parser's alias, which HIP reads off the info it + # parsed -- ``HIPv2`` -- so the alias is read off a stand-in for an + # instance holding ``info``. + alias = inspect.getattr_static(parser, 'alias', None) + try: + if isinstance(alias, property): + view = info # type: Any + if not isinstance(info, Info): + view = types.SimpleNamespace(**dict(info.items())) + alias = alias.fget(types.SimpleNamespace(_info=view)) # type: ignore[misc] + except Exception: # pylint: disable=broad-except + return None + if not isinstance(alias, str) or cls._exthdr_key(alias) != key: + return None + return parser + @staticmethod def _is_first_fragment(info: 'Mapping[str, Any]') -> 'bool': """Whether a Fragment header's info is that of a first fragment. diff --git a/pcapkit/protocols/protocol.py b/pcapkit/protocols/protocol.py index 7fb13784ed..8b74fd1166 100644 --- a/pcapkit/protocols/protocol.py +++ b/pcapkit/protocols/protocol.py @@ -2168,13 +2168,20 @@ def _lookup_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'tupl """Find the payload of ``data`` that carries no ``__next_type__``. :meth:`_decode_next_layer` writes the payload's info under the payload's - :attr:`info_name`, after every field of the data model, so the payload is - the *last* key that the data model does not declare and whose value is a - mapping. Its protocol is the one of :attr:`__proto__` -- or its fallback - -- whose :attr:`info_name` the key is. That is the lowercased class name - by default, but a protocol may override it: every version of + :attr:`info_name`, so the payload is the key, among those the data + model does not declare and whose value is a mapping, that is the + :attr:`info_name` of a protocol of :attr:`__proto__` -- or of its + fallback. That is the lowercased class name by default, but a protocol + may override it: every version of :class:`~pcapkit.protocols.link.l2tp.L2TP` is ``l2tp``. + The payload is found by that name, and never by its place among the + keys: the order of the keys of an :class:`~pcapkit.corekit.infoclass.Info` + is not part of its contract, and on PyPy it follows the attribute + layout of an earlier instance of the same class (:issue:`2003`). Should + ``data`` hold the names of several such protocols, the first of them in + :attr:`__proto__`, its fallback last, is the payload. + Args: data: protocol data exclude: keys of ``data`` to pass over, such as the IPv6 extension @@ -2186,10 +2193,9 @@ def _lookup_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'tupl dispatch to. """ - keys = [key for key in cls._lookup_nested(data) if key not in exclude] + keys = {key for key in cls._lookup_nested(data) if key not in exclude} if not keys: return None, None - name = keys[-1] registry = cls.__proto__ candidates = list(registry.values()) @@ -2207,22 +2213,23 @@ def _lookup_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'tupl label = proto.__new__(proto).info_name except TypeError: continue - if label == name: - return proto, name + if label in keys: + return proto, label return None, None @staticmethod def _lookup_nested(data: 'Data') -> 'list[str]': """Keys of ``data`` that its data model does not declare and that hold a mapping. - These are where the next layer protocols wrote their info, in the order - they wrote it -- the payload, and the IPv6 extension headers. + These are where the next layer protocols wrote their info -- the + payload, and the IPv6 extension headers. Args: data: protocol data Returns: - The keys, in the order of ``data``. + The keys, in the order of ``data``, which no caller relies on + (:issue:`2003`). """ declared = set() # type: set[str] diff --git a/tests/corekit/test_multiinfo_1484_unit.py b/tests/corekit/test_multiinfo_1484_unit.py index 736f775677..ef8efb2fdf 100644 --- a/tests/corekit/test_multiinfo_1484_unit.py +++ b/tests/corekit/test_multiinfo_1484_unit.py @@ -408,11 +408,16 @@ def test_to_dict_returns_a_plain_ordered_multi_dict_at_every_level(self) -> None exported = info.to_dict() self.assertIs(type(exported), _OrderedMultiDict) self.assertIsInstance(exported, OrderedMultiDict) - self.assertEqual([key for key, _ in exported.items(multi=True)], ['nested', 'ordered', 'grouped', 'nested']) - self.assertEqual([type(value) for _, value in exported.items(multi=True)], - [_OrderedMultiDict, _OrderedMultiDict, MultiDict, _OrderedMultiDict]) + # NOTE: The order across keys is that of the Info's own keys, which is + # not part of its contract and differs on PyPy (#2003); the values of + # one key keep theirs. + self.assertEqual(sorted(key for key, _ in exported.items(multi=True)), ['grouped', 'nested', 'nested', 'ordered']) + self.assertEqual({key: [type(value) for value in exported.getlist(key)] for key in exported}, + {'nested': [_OrderedMultiDict, _OrderedMultiDict], 'ordered': [_OrderedMultiDict], + 'grouped': [MultiDict]}) self.assertEqual([value['x'] for value in exported.getlist('nested')], [1, 2]) self.assertEqual(list(exported['ordered'].items(multi=True)), PAIRS) + self.assertEqual(list(exported['grouped'].items(multi=True)), list(MultiDict(PAIRS).items(multi=True))) # The export is a copy the caller may edit. exported['ordered'].add('z', 9) diff --git a/tests/protocols/test_lookup_by_name_2003_unit.py b/tests/protocols/test_lookup_by_name_2003_unit.py new file mode 100644 index 0000000000..ffefd4a0f0 --- /dev/null +++ b/tests/protocols/test_lookup_by_name_2003_unit.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +"""A dict rebuild finds the payload and the extension headers whatever the key order. + +GitHub issue #2003: on PyPy, an :class:`~pcapkit.corekit.infoclass.Info` +lays its keys out as an earlier instance of the same class did, so its key +order is not CPython's. Two lookups of a rebuild from ``to_dict()`` read +structure off that order: the payload was taken to be the *last* nested key, +and the IPv6 extension header chain was walked in key order. A cut frame +rebuilt short, and so did a Hop-by-Hop, Destination, Destination chain once +``hopopt`` had moved past the upper layer. + +The order of an :class:`~pcapkit.corekit.infoclass.Info` is not part of its +contract, so the payload is now found by the name of a protocol it dispatches +to, and the chain by its ``next`` fields. These tests reorder an instance's +:attr:`~object.__dict__` as PyPy does, so CPython checks the same thing. + +Every case builds its own octets and reads no capture. Classes are imported +inside each test, after :func:`~tests._support.reimport_once_per_class`. + +""" + +import copy +import struct +import unittest +import warnings + +from tests._support import reimport_once_per_class + +#: Test tier (#1958): a hand-picked round trip, pinning that a rebuild does not +#: depend on key order, which differs by interpreter. +__tier__ = 'ALL' + +HOP, DST, FRAG, UDP = 0, 60, 44, 17 +SRC6 = bytes.fromhex('20010db8' + '00' * 11 + '01') +DST6 = bytes.fromhex('20010db8' + '00' * 11 + '02') +DATAGRAM = struct.pack('!HHHH', 40000, 40000, 12, 0) + b'ping' + + +def options(nxt: int, mark: int) -> bytes: + """An 8-octet options header carrying a PadN; ``mark`` tells repeats apart.""" + return bytes([nxt, 0, 1, 4, 0, 0, 0, mark]) + + +def ipv6(first: int, body: bytes) -> bytes: + """An IPv6 header whose Next Header is ``first``, followed by ``body``.""" + return struct.pack('!IHBB16s16s', 6 << 28, len(body), first, 64, SRC6, DST6) + body + + +#: Hop-by-Hop, Destination, Destination, then UDP. +HOP_DST_DST = ipv6(HOP, options(DST, 0) + options(DST, 1) + options(UDP, 2) + DATAGRAM) + +#: An atomic Fragment header -- offset zero, no more fragments -- then UDP, so +#: the walk goes on to the upper layer and ``fragment`` is held beside it. +ATOMIC = ipv6(FRAG, struct.pack('!BBHI', UDP, 0, 0, 0x2003) + DATAGRAM) + + +def moved(info, key: str, before=None): # type: ignore[no-untyped-def] + """A copy of ``info`` with ``key`` moved before ``before``, or last, in its ``__dict__``. + + That is the reordering PyPy makes when an earlier instance of the class + laid its attributes out differently. PyPy may lay the copy out its own way + again, so the order is asserted of nothing but the rebuild. + + """ + clone = copy.copy(info) + layout = clone.__dict__ + value = layout.pop(key) + items = list(layout.items()) + index = len(items) if before is None else [name for name, _ in items].index(before) + items.insert(index, (key, value)) + layout.clear() + layout.update(items) + return clone + + +class LookupByNameTests(unittest.TestCase): + """Pin that the dict rebuild reads no structure off the key order.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + quiet = warnings.catch_warnings() + quiet.__enter__() # pylint: disable=unnecessary-dunder-call + self.addCleanup(quiet.__exit__, None, None, None) + warnings.simplefilter('ignore') + + def test_extension_headers_follow_next_not_key_order(self) -> None: + from pcapkit.protocols.internet.hopopt import HOPOPT + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts + + ip6 = IPv6(HOP_DST_DST) + self.assertEqual(len(ip6.data), len(HOP_DST_DST)) + + # As PyPy laid it out: ``hopopt`` after every other key. + exported = moved(ip6.info, 'hopopt').to_dict() + names, chain = IPv6._lookup_exthdr(exported) + self.assertEqual(names, ('hopopt', 'opts', 'opts')) + self.assertEqual([parser for parser, _ in chain], [HOPOPT, IPv6_Opts, IPv6_Opts]) + self.assertEqual([parser.from_data(hdr).data for parser, hdr in chain], + [ext.data for _, ext in ip6.extension_headers.items(multi=True)]) + self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) + + def test_payload_is_found_by_name_not_as_the_last_key(self) -> None: + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.transport.udp import UDP as UDP_ + + ip6 = IPv6(ATOMIC) + self.assertIn('fragment', ip6.info) + self.assertEqual(len(ip6.data), len(ATOMIC)) + + # ``fragment`` is a nested key too, and not the payload; laid out + # after the payload, it is the last nested key. + info = moved(ip6.info, 'udp', before='fragment') + exported = info.to_dict() + names, _ = IPv6._lookup_exthdr(exported) + self.assertEqual(names, ('frag',)) + self.assertEqual(IPv6._lookup_payload(exported, exclude=names), (UDP_, 'udp')) + self.assertEqual(IPv6.from_data(exported).data, ATOMIC) + + def test_cut_upper_layer_is_found_ahead_of_the_headers(self) -> None: + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.misc.raw import Raw + from pcapkit.utilities import config + + limit = config.EXTENSION_HEADER_LIMIT + self.addCleanup(setattr, config, 'EXTENSION_HEADER_LIMIT', limit) + config.EXTENSION_HEADER_LIMIT = 2 + + # The third header is cut, and kept raw as the upper layer. + ip6 = IPv6(HOP_DST_DST) + self.assertIn('raw', ip6.info) + self.assertEqual(len(ip6.data), len(HOP_DST_DST)) + + # As on PyPy for #1610: ``opts`` after the upper layer. + info = moved(moved(ip6.info, 'raw', before='opts'), 'hopopt', before='raw') + exported = info.to_dict() + names, _ = IPv6._lookup_exthdr(exported) + self.assertEqual(sorted(names), ['hopopt', 'opts']) + self.assertEqual(IPv6._lookup_payload(exported, exclude=names), (Raw, 'raw')) + self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py index 6a1b89d082..24fb5b6b8b 100644 --- a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py +++ b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py @@ -7,9 +7,11 @@ Destination Options, Destination Options, UDP rebuilt 56 of 64 octets. ``to_dict`` returns an :class:`~pcapkit.corekit.multidict.OrderedMultiDict` -(#1484): indexing sees the first value of each key, and ``items(multi=True)`` -yields every header in wire order for the rebuild. The dumpers write every -header too. +(#1484): indexing sees the first value of each key, and ``getlist`` gives every +header of a type in wire order. The order across types is that of the keys of +the :class:`~pcapkit.corekit.infoclass.Info`, which is not part of its contract +and differs on PyPy, so the rebuild walks the ``next`` fields instead (#2003). +The dumpers write every header too. Every case builds its own octets and reads no capture. Classes are imported inside each test, after :func:`~tests._support.reimport_once_per_class`. @@ -133,7 +135,7 @@ def test_rebuilt_chain_has_the_parsed_headers(self) -> None: self.assertEqual([ext.data for ext in rebuilt.extension_headers.values()], [ext.data for ext in ip6.extension_headers.values()]) - def test_to_dict_keeps_every_header_in_wire_order(self) -> None: + def test_to_dict_keeps_every_header_of_a_type_in_wire_order(self) -> None: from pcapkit.corekit.multidict import OrderedMultiDict from pcapkit.protocols.internet.ipv6 import IPv6 @@ -143,8 +145,10 @@ def test_to_dict_keeps_every_header_in_wire_order(self) -> None: dict_ = ip6.info.to_dict() self.assertIsInstance(dict_, OrderedMultiDict) - parsed = [canonical(ext.info.to_dict()) for ext in ip6.extension_headers.values()] - held = [canonical(val) for _, val in dict_.items(multi=True) if canonical(val) in parsed] + parsed = {} # type: dict[str, list[object]] + for _, ext in ip6.extension_headers.items(multi=True): + parsed.setdefault(IPv6._exthdr_key(ext.alias), []).append(canonical(ext.info.to_dict())) + held = {key: [canonical(val) for val in dict_.getlist(key)] for key in parsed} self.assertEqual(held, parsed) # Indexing sees the first of each. From cf6931e8fa3a4f366d62be418dab792a635b4cee Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 21 Oct 2026 11:43:22 -0400 Subject: [PATCH 2/6] fix(protocols): hold the IPv6 extension headers in extension_headers (#2003) The extension headers of an IPv6 info are now held, in wire order, in one field, ``extension_headers``: an OrderedMultiInfo keyed as before (``hopopt``, ``opts``, ...), so a repeated header keeps every info (#1453) and the order is the headers' own rather than that of the Info's keys, which differs on PyPy. ``_lookup_exthdr`` reads the chain from it, the code of each header being the ``next`` of the one before, with the #1473 rule that a header under IPv6_Ext's key is rebuilt with IPv6_Ext. The ``next``-field walk and ``_exthdr_parser`` are gone, and so is ``__exthdr__``: the parse and the dict rebuild now take the chain from the same place. Tests reading the per-header top-level keys read them from ``extension_headers``; the dump tests pin its shape, one single-key mapping per header. --- pcapkit/protocols/data/internet/ipv6.py | 14 +- pcapkit/protocols/internet/ipv6.py | 173 ++++++------------ .../test_repeated_field_dump_1484_unit.py | 20 +- tests/protocols/internet/test_esp_unit.py | 2 +- .../internet/test_ipv6_core_roundtrip_unit.py | 2 +- .../test_ipv6_exthdr_cut_rebuild_unit.py | 2 +- .../internet/test_ipv6_from_data_ext_unit.py | 19 +- tests/protocols/internet/test_ipv6_unit.py | 2 +- .../internet/test_mpl_seed_rebuild_unit.py | 2 +- .../test_declared_length_overrun_unit.py | 2 +- .../test_lookup_by_name_2003_unit.py | 37 ++-- .../protocols/test_option_coverage_runtime.py | 2 +- tests/protocols/test_protocol_base_unit.py | 2 +- .../test_to_dict_repeated_exthdr_1453_unit.py | 41 +++-- 14 files changed, 137 insertions(+), 183 deletions(-) diff --git a/pcapkit/protocols/data/internet/ipv6.py b/pcapkit/protocols/data/internet/ipv6.py index 4181e9e13b..b4f435b7cf 100644 --- a/pcapkit/protocols/data/internet/ipv6.py +++ b/pcapkit/protocols/data/internet/ipv6.py @@ -8,14 +8,14 @@ if TYPE_CHECKING: from ipaddress import IPv6Address - from typing import Any, Type + from typing import Any from typing_extensions import Literal from pcapkit.const.reg.transtype import TransType + from pcapkit.corekit.infoclass import OrderedMultiInfo from pcapkit.protocols.data.data import Data from pcapkit.protocols.data.protocol import Packet - from pcapkit.protocols.protocol import ProtocolBase __all__ = [ 'IPv6', @@ -33,8 +33,6 @@ class IPv6(Protocol): """ - __excluded__ = ['__exthdr__'] - #: Version. version: 'Literal[6]' #: Traffic class. @@ -64,10 +62,10 @@ class IPv6(Protocol): #: Octets captured past the Payload Length -- or, for a jumbogram, #: past the Jumbo Payload Length -- present only when there are any. trailer: 'bytes' - #: Extension header chain, in wire order, as pairs of the parser - #: class and its info. Kept apart from the per-header keys above, - #: which a repeated header overwrites. - __exthdr__: 'tuple[tuple[Type[ProtocolBase], Data], ...]' + #: Extension headers, in wire order, each under the key its parser + #: writes, such as ``hopopt`` or ``opts``; a repeated header keeps + #: every one of its infos (:issue:`1453`, :issue:`2003`). + extension_headers: 'OrderedMultiInfo[str, Data]' def __new__(cls, *args: 'Any', **kwargs: 'Any') -> 'IPv6': self = super().__new__(cls, *args, **kwargs) diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index 8835b83d2a..51be402d10 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -27,15 +27,12 @@ """ import collections.abc -import inspect import ipaddress -import types from typing import TYPE_CHECKING from pcapkit.const.ipv6.extension_header import ExtensionHeader as Enum_ExtensionHeader from pcapkit.const.reg.transtype import TransType as Enum_TransType from pcapkit.corekit.decorators import beholder -from pcapkit.corekit.infoclass import Info from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict from pcapkit.corekit.protochain import ProtoChain from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6 @@ -413,14 +410,8 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over Key-value pairs for protocol construction. """ - exthdr = data.get('__exthdr__') # type: Optional[tuple[tuple[Type[ProtocolBase], Any], ...]] # pylint: disable=line-too-long - exclude = () # type: tuple[str, ...] - if exthdr is None: - # NOTE: ``data`` was rebuilt from :meth:`Info.to_dict`, which leaves - # ``__exthdr__`` out, so the chain is recovered from the keys the - # extension headers were written under; those are no payload. - exclude, exthdr = cls._lookup_exthdr(data) - upper = cls._make_payload(data, exclude=exclude) + exthdr = cls._lookup_exthdr(data) + upper = cls._make_payload(data) payload = upper # type: bytes | ProtocolBase # NOTE: The extension headers sit between this header and the upper @@ -447,7 +438,7 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over @staticmethod def _exthdr_key(alias: 'str') -> 'str': - """Key an extension header's info is written under in :class:`Data_IPv6`. + """Key an extension header's info is written under in :attr:`Data_IPv6.extension_headers`. Args: alias: :attr:`~pcapkit.protocols.protocol.ProtocolBase.alias` of @@ -461,45 +452,41 @@ def _exthdr_key(alias: 'str') -> 'str': return alias.lstrip('IPv6-').lower() @classmethod - def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[str, ...], tuple[tuple[Type[ProtocolBase], Any], ...]]': # pylint: disable=line-too-long - """Recover the extension header chain of ``data`` without ``__exthdr__``. - - :meth:`_decode_next_layer` writes each extension header's info under a - key of its own. Walking the ``next`` field from :attr:`data.next - ` names the parser of each in turn, exactly as the walk - that parsed them did, and each step takes the first header not yet - taken that is held under the key that parser writes. - - So the chain comes from the ``next`` fields, and never from the order - of the keys of ``data``: that order is not part of the contract of an - :class:`~pcapkit.corekit.infoclass.Info`, and on PyPy it follows the - attribute layout of an earlier instance of the same class, which put - the Hop-by-Hop Options header after the upper layer (:issue:`2003`). - A header repeated in the chain is held under the same key as its - predecessor, and the headers held under one key are taken in the order - ``items(multi=True)`` yields them (:issue:`1453`). + def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[Type[ProtocolBase], Any], ...]': + """Recover the extension header chain of ``data``, with the parser of each header. + + :meth:`_decode_next_layer` holds the extension headers in + :attr:`data.extension_headers `, in wire + order, each under the key its parser writes. The code of each is the + ``next`` field of the one before it, or :attr:`data.next + ` for the first, exactly as the walk that parsed them + dispatched on it. + + The order is that of ``extension_headers``, which keeps it across keys + and so across a repeated header (:issue:`1453`), and never that of the + keys of ``data``: those are an :class:`~pcapkit.corekit.infoclass.Info`'s, + whose order is not part of its contract and on PyPy follows the + attribute layout of an earlier instance of the same class + (:issue:`2003`). Args: - data: protocol data, without ``__exthdr__`` + data: protocol data Returns: - The keys the extension headers are under, and the chain in the - shape of ``__exthdr__``, i.e. the parser class and info of each. + The parser class and info of each extension header, in wire order. """ - nested = cls._lookup_nested(data) - pairs = data.items(multi=True) if isinstance(data, (Info, MultiDict)) else data.items() - # NOTE: ``fragment`` is declared only for type checkers, so it reads as - # nested, but it is the packet split at the Fragment header and sits - # between that header and the next one. - pool = [(key, info) for (key, info) in pairs - if key in nested and key != 'fragment'] # type: list[tuple[str, Any]] - taken = [False] * len(pool) - - names = [] # type: list[str] + from pcapkit.protocols.internet.ipv6_ext import \ + IPv6_Ext # isort: skip # pylint: disable=import-outside-toplevel + + headers = data.get('extension_headers') + if not isinstance(headers, collections.abc.Mapping): + return () + pairs = headers.items(multi=True) if isinstance(headers, MultiDict) else headers.items() + chain = [] # type: list[tuple[Type[ProtocolBase], Any]] proto = data.get('next') - while True: + for (key, info) in pairs: try: ex_proto = Enum_ExtensionHeader(proto) except ValueError: @@ -507,74 +494,26 @@ def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[str, ...], tuple[tupl # NOTE: As in :meth:`_decode_next_layer`, a code with no parser # carrying ``next`` ends the chain, and so does a Fragment header - # other than a first fragment's. Nor does the walk go on past a - # code none of the headers left is held under -- the upper layer - # kept raw at a limit, say. - index, parser = -1, None # type: int, Optional[Type[ProtocolBase]] - for (index, (key, info)) in enumerate(pool): - if not taken[index]: - parser = cls._exthdr_parser(key, info, ex_proto) - if parser is not None: - break - if parser is None: + # other than a first fragment's. + if 'next' not in info: break - key, info = pool[index] - taken[index] = True - names.append(key) + + # NOTE: A header whose dedicated parser raised was parsed by + # :class:`~pcapkit.protocols.internet.ipv6_ext.IPv6_Ext` instead, + # see :meth:`_import_next_layer`, and written under that class's + # key rather than the dedicated one's. Rebuilding it with the + # dedicated class would look for fields only that class parses + # (#1473), so the key decides the parser. + if key == cls._exthdr_key(IPv6_Ext.alias.fget(None)): # type: ignore[attr-defined] + parser = IPv6_Ext # type: Type[ProtocolBase] + else: + parser = cls._lookup_next_layer(cls.__proto__, int(ex_proto)) chain.append((parser, info)) proto = info['next'] if ex_proto == Enum_ExtensionHeader.IPv6_Frag and not cls._is_first_fragment(info): break - return tuple(names), tuple(chain) - - @classmethod - def _exthdr_parser(cls, key: 'str', info: 'Any', code: 'Enum_ExtensionHeader') -> 'Optional[Type[ProtocolBase]]': # pylint: disable=line-too-long - """Parser of the extension header ``code`` names, if ``key`` holds it as ``info``. - - Args: - key: key of ``info`` in :class:`Data_IPv6` - info: info of an extension header, as parsed or from - :meth:`Info.to_dict ` - code: extension header code the walk dispatches on - - Returns: - The parser to rebuild ``info`` with, or :data:`None` if ``info`` - under ``key`` is not the header ``code`` names. - - """ - from pcapkit.protocols.internet.ipv6_ext import \ - IPv6_Ext # isort: skip # pylint: disable=import-outside-toplevel - - if not isinstance(info, collections.abc.Mapping) or 'next' not in info: - return None - - # NOTE: A header whose dedicated parser raised was parsed by - # :class:`~pcapkit.protocols.internet.ipv6_ext.IPv6_Ext` instead, see - # :meth:`_import_next_layer`, and written under that class's key rather - # than the dedicated one's, with the code it stands in for as its - # ``protocol``. Rebuilding it with the dedicated class would look for - # fields only that class parses (#1473), so the key decides the parser. - if key == cls._exthdr_key(IPv6_Ext.alias.fget(None)): # type: ignore[attr-defined] - stands_for = info.get('protocol') - return IPv6_Ext if stands_for is None or stands_for == code else None - - parser = cls._lookup_next_layer(cls.__proto__, int(code)) - # NOTE: The key is the parser's alias, which HIP reads off the info it - # parsed -- ``HIPv2`` -- so the alias is read off a stand-in for an - # instance holding ``info``. - alias = inspect.getattr_static(parser, 'alias', None) - try: - if isinstance(alias, property): - view = info # type: Any - if not isinstance(info, Info): - view = types.SimpleNamespace(**dict(info.items())) - alias = alias.fget(types.SimpleNamespace(_info=view)) # type: ignore[misc] - except Exception: # pylint: disable=broad-except - return None - if not isinstance(alias, str) or cls._exthdr_key(alias) != key: - return None - return parser + return tuple(chain) @staticmethod def _is_first_fragment(info: 'Mapping[str, Any]') -> 'bool': @@ -642,7 +581,7 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, hdr_len = self.length # header length raw_len = ipv6.payload if length is None else length # payload length _protos = [] # type: list[ProtocolBase] # ProtoChain buffer - _exthdr = [] # (parser class, info) per extension header + _headers = OrderedMultiDict() # type: OrderedMultiDict[str, Any] # info per extension header split = None # type: Optional[tuple[int, int]] # at first Fragment header opaque = False # walk ended at a non-zero Fragment Offset depth = _frame_depth.get() # layers of the frame up to this one @@ -703,24 +642,23 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, # # Such a layer is not recorded as an extension header: it consumed # nothing (``Raw.length`` is 0), so recording it would emit its - # octets twice on rebuild, once from ``__exthdr__`` and once more as - # the upper layer. Stopping here with ``proto`` unchanged instead + # octets twice on rebuild, once from ``extension_headers`` and once + # more as the upper layer. Stopping here with ``proto`` unchanged instead # hands the remaining octets to ``super()._decode_next_layer`` # below, which decodes them once, as the upper layer. if not hasattr(info, 'next'): break - # NOTE: Added through a :class:`MultiDict`, so a repeated header - # keeps its predecessor's info rather than overwriting it, and - # :meth:`Info.to_dict ` - # writes every one of them, in wire order (#1453). - name = self._exthdr_key(next_.alias) - ipv6.__update__(OrderedMultiDict([(name, info)])) + # NOTE: Held in an :class:`OrderedMultiDict` of their own, so a + # repeated header keeps its predecessor's info rather than + # overwriting it (#1453), and the wire order is that of the + # headers, not of the keys of ``ipv6``, which differs on PyPy + # (#2003). + _headers.add(self._exthdr_key(next_.alias), info) # record protocol name # self._protos = ProtoChain(name, chain, alias) _protos.append(next_) - _exthdr.append((type(next_), info)) # update header & payload length hdr_len += next_.length # type: ignore[assignment] @@ -773,9 +711,8 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, # update next header 'protocol': proto, - # extension header chain in wire order, with the parser of each, - # for ``from_data`` - '__exthdr__': tuple(_exthdr), + # extension headers in wire order + 'extension_headers': _headers, }) # NOTE: Nor does a later fragment's slice start with the upper-layer diff --git a/tests/dumpkit/test_repeated_field_dump_1484_unit.py b/tests/dumpkit/test_repeated_field_dump_1484_unit.py index fd5f26d61e..fc1aef50cc 100644 --- a/tests/dumpkit/test_repeated_field_dump_1484_unit.py +++ b/tests/dumpkit/test_repeated_field_dump_1484_unit.py @@ -3,8 +3,10 @@ GitHub issue #1484. An IPv6 packet carrying two Destination Options headers holds both under the one ``opts`` key, and every dump held only the first. -Each dump now writes the field as the list of both headers, in wire order, -while a field held once is written as its value. +Each dump now writes both headers, in wire order, while a field held once is +written as its value. The headers are held in ``extension_headers`` +(#2003), an option list, which a dump writes as one single-key mapping per +entry. Every case builds its input in a temporary directory, and reads no capture. :mod:`pcapkit` is imported inside each test, after @@ -46,6 +48,11 @@ def capture() -> bytes: return GLOBAL + struct.pack(' 'list[dict]': + """The Destination Options headers of a dumped IPv6 header, in wire order.""" + return [entry['opts'] for entry in ipv6['extension_headers']] + + def pads(headers: 'list[dict]') -> 'list[str]': """The PadN octets of each Destination Options header, as the JSON dump spells them.""" return [next(iter(header['options'][0].values()))['pad']['hex'] for header in headers] @@ -77,15 +84,14 @@ def dump(self, fmt: 'str') -> 'bytes': def test_json_holds_both_headers_in_order(self) -> None: ipv6 = json.loads(self.dump('json'))['Frame 1']['ethernet']['ipv6'] - self.assertIsInstance(ipv6['opts'], list) - self.assertEqual(pads(ipv6['opts']), [f'000000{mark:02x}' for mark in MARKS]) + self.assertEqual([list(entry) for entry in ipv6['extension_headers']], [['opts'], ['opts']]) + self.assertEqual(pads(opts(ipv6)), [f'000000{mark:02x}' for mark in MARKS]) # A field held once is still written as its value. self.assertEqual(ipv6['limit'], 64) def test_plist_holds_both_headers_in_order(self) -> None: ipv6 = plistlib.loads(self.dump('plist'))['Frame 1']['ethernet']['ipv6'] - self.assertIsInstance(ipv6['opts'], list) - self.assertEqual([next(iter(header['options'][0].values()))['pad'] for header in ipv6['opts']], + self.assertEqual([next(iter(header['options'][0].values()))['pad'] for header in opts(ipv6)], [b'\x00\x00\x00' + bytes([mark]) for mark in MARKS]) def test_tree_holds_both_headers(self) -> None: @@ -118,7 +124,7 @@ def test_the_writer_takes_the_info_or_its_export(self) -> None: make_dumper(dictdumper.JSON)(path)(value, name='Frame 1') with open(path, encoding='utf-8') as file: tops.append(json.load(file)['Frame 1']) - self.assertEqual(pads(tops[0]['ethernet']['ipv6']['opts']), [f'000000{mark:02x}' for mark in MARKS]) + self.assertEqual(pads(opts(tops[0]['ethernet']['ipv6'])), [f'000000{mark:02x}' for mark in MARKS]) self.assertEqual(list(tops[1]), list(tops[0])) self.assertEqual(list(tops[0]), list(frame.info.to_dict())) diff --git a/tests/protocols/internet/test_esp_unit.py b/tests/protocols/internet/test_esp_unit.py index 08fc53e417..59bac8c1c7 100644 --- a/tests/protocols/internet/test_esp_unit.py +++ b/tests/protocols/internet/test_esp_unit.py @@ -1029,7 +1029,7 @@ def test_ipv6_extension_header_position(self) -> None: packet = header + body ipv6 = IPv6(packet, len(packet), __context__=registry) - esp = ipv6.info.esp + esp = ipv6.info.extension_headers['esp'] self.assertIs(esp.status, ESPStatus.DECRYPTED) self.assertEqual(esp.next, TransType.TCP) self.assertEqual(esp.plaintext, INNER_TCP[20:]) diff --git a/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py b/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py index 6afa2f4ad3..40a6947b01 100644 --- a/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py +++ b/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py @@ -67,7 +67,7 @@ def test_private_codes_are_not_extension_headers(self) -> None: with self.subTest(code=code): frame = ipv6(code, bytes.fromhex('3b00010203040506')) parsed = IPv6(frame) - self.assertEqual(parsed.info.__exthdr__, ()) + self.assertEqual(len(parsed.info.extension_headers), 0) self.assertEqual(len(parsed.protochain), 2) self.assertEqual(IPv6.from_data(parsed.info).data, frame) diff --git a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py index ba3124476d..17cfbad749 100644 --- a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py +++ b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py @@ -112,7 +112,7 @@ def test_dict_rebuild_keeps_the_generic_parser(self) -> None: with self.subTest(cut=cut): ip6 = IPv6(data, len(data)) self.assertIs(type(ip6._exthdr[ExtensionHeader.HOPOPT]), IPv6_Ext) - _, chain = IPv6._lookup_exthdr(ip6.info.to_dict()) + chain = IPv6._lookup_exthdr(ip6.info.to_dict()) self.assertEqual([parser for parser, _ in chain], [IPv6_Ext]) self.assertEqual(IPv6.from_data(ip6.info.to_dict()).data, data) diff --git a/tests/protocols/internet/test_ipv6_from_data_ext_unit.py b/tests/protocols/internet/test_ipv6_from_data_ext_unit.py index 19ffcaa589..a5abd60665 100644 --- a/tests/protocols/internet/test_ipv6_from_data_ext_unit.py +++ b/tests/protocols/internet/test_ipv6_from_data_ext_unit.py @@ -82,13 +82,22 @@ def test_repeated_header_keeps_both_instances(self) -> None: ) self.assertEqual(IPv6.from_data(parsed.info).data, packet) - def test_chain_is_hidden_from_info_views(self) -> None: + def test_chain_is_held_in_extension_headers(self) -> None: + from pcapkit.corekit.infoclass import OrderedMultiInfo + from pcapkit.corekit.multidict import OrderedMultiDict from pcapkit.protocols.internet.ipv6 import IPv6 - packet = PACKETS['routing'] - info = IPv6(packet, len(packet)).info - self.assertNotIn('__exthdr__', list(info)) - self.assertNotIn('__exthdr__', info.to_dict()) + packet = PACKETS['chain-with-repeated-header'] + parsed = IPv6(packet, len(packet)) + headers = parsed.info.extension_headers + self.assertIsInstance(headers, OrderedMultiInfo) + self.assertEqual([info for _, info in headers.items(multi=True)], + [proto.info for _, proto in parsed.extension_headers.items(multi=True)]) + # ``to_dict`` exports it as the option list it is, in wire order. + exported = parsed.info.to_dict()['extension_headers'] + self.assertIsInstance(exported, OrderedMultiDict) + self.assertEqual([key for key, _ in exported.items(multi=True)], ['hopopt', 'opts', 'route', 'opts']) + self.assertEqual(IPv6.from_data(parsed.info.to_dict()).data, packet) if __name__ == '__main__': diff --git a/tests/protocols/internet/test_ipv6_unit.py b/tests/protocols/internet/test_ipv6_unit.py index eaa96fca89..4b10e9f1bf 100644 --- a/tests/protocols/internet/test_ipv6_unit.py +++ b/tests/protocols/internet/test_ipv6_unit.py @@ -178,7 +178,7 @@ def test_ipv6_decode_next_layer_handles_extension_headers_and_fragments(self) -> self.assertTrue(import_next.call_args.kwargs['extension']) self.assertEqual(import_next.call_args.kwargs['payload'], b'fragment-payload') proto._read_packet.assert_called_once_with(header=48, payload=len(b'fragment-payload') - 8) - self.assertEqual(ipv6.frag.next, TransType.UDP) + self.assertEqual(ipv6.extension_headers['frag'].next, TransType.UDP) self.assertEqual(ipv6.fragment, 'fragment-data') self.assertEqual(ipv6.hdr_len, 48) self.assertEqual(ipv6.raw_len, len(b'fragment-payload') - 8) diff --git a/tests/protocols/internet/test_mpl_seed_rebuild_unit.py b/tests/protocols/internet/test_mpl_seed_rebuild_unit.py index e9852942c0..fb5a424bfc 100644 --- a/tests/protocols/internet/test_mpl_seed_rebuild_unit.py +++ b/tests/protocols/internet/test_mpl_seed_rebuild_unit.py @@ -63,7 +63,7 @@ def test_source_address_seed_rebuilds_inside_ipv6(self) -> None: octets = HEADERS[0] packet = (struct.pack('!IHBB', 6 << 28, len(octets), nh, 64) + SRC.packed + DST.packed + octets) - info = IPv6(packet, len(packet)).info[key] + info = IPv6(packet, len(packet)).info.extension_headers[key] opt = list(info.options.values())[0] self.assertEqual(opt.seed_id, SRC) self.assertEqual(cls.from_data(info).data, octets) diff --git a/tests/protocols/test_declared_length_overrun_unit.py b/tests/protocols/test_declared_length_overrun_unit.py index a2503f1c05..624e687616 100644 --- a/tests/protocols/test_declared_length_overrun_unit.py +++ b/tests/protocols/test_declared_length_overrun_unit.py @@ -153,7 +153,7 @@ def test_an_overrunning_layer_is_kept_as_captured_by_its_parent(self) -> None: if name == 'IPv6': # an extension header is kept as raw extension data - self.assertIn('has an option that runs past the end of the data', str(parsed.info.ext.error)) + self.assertIn('has an option that runs past the end of the data', str(parsed.info.extension_headers['ext'].error)) else: # any other layer is kept as a raw payload, as captured self.assertIsInstance(parsed.payload, Raw) diff --git a/tests/protocols/test_lookup_by_name_2003_unit.py b/tests/protocols/test_lookup_by_name_2003_unit.py index ffefd4a0f0..79e77ab835 100644 --- a/tests/protocols/test_lookup_by_name_2003_unit.py +++ b/tests/protocols/test_lookup_by_name_2003_unit.py @@ -11,7 +11,9 @@ The order of an :class:`~pcapkit.corekit.infoclass.Info` is not part of its contract, so the payload is now found by the name of a protocol it dispatches -to, and the chain by its ``next`` fields. These tests reorder an instance's +to, and the chain is held in ``extension_headers``, an +:class:`~pcapkit.corekit.infoclass.OrderedMultiInfo` keeping the order of its +own. These tests reorder an instance's :attr:`~object.__dict__` as PyPy does, so CPython checks the same thing. Every case builds its own octets and reads no capture. Classes are imported @@ -83,7 +85,7 @@ def setUp(self) -> None: self.addCleanup(quiet.__exit__, None, None, None) warnings.simplefilter('ignore') - def test_extension_headers_follow_next_not_key_order(self) -> None: + def test_extension_headers_keep_their_order_whatever_the_key_order(self) -> None: from pcapkit.protocols.internet.hopopt import HOPOPT from pcapkit.protocols.internet.ipv6 import IPv6 from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts @@ -91,14 +93,15 @@ def test_extension_headers_follow_next_not_key_order(self) -> None: ip6 = IPv6(HOP_DST_DST) self.assertEqual(len(ip6.data), len(HOP_DST_DST)) - # As PyPy laid it out: ``hopopt`` after every other key. - exported = moved(ip6.info, 'hopopt').to_dict() - names, chain = IPv6._lookup_exthdr(exported) - self.assertEqual(names, ('hopopt', 'opts', 'opts')) - self.assertEqual([parser for parser, _ in chain], [HOPOPT, IPv6_Opts, IPv6_Opts]) - self.assertEqual([parser.from_data(hdr).data for parser, hdr in chain], - [ext.data for _, ext in ip6.extension_headers.items(multi=True)]) - self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) + # The headers first, then last, of the keys. + for before in ('version', None): + with self.subTest(before=before): + exported = moved(ip6.info, 'extension_headers', before=before).to_dict() + chain = IPv6._lookup_exthdr(exported) + self.assertEqual([parser for parser, _ in chain], [HOPOPT, IPv6_Opts, IPv6_Opts]) + self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) + self.assertEqual([parser.from_data(hdr).data for parser, hdr in chain], + [ext.data for _, ext in ip6.extension_headers.items(multi=True)]) def test_payload_is_found_by_name_not_as_the_last_key(self) -> None: from pcapkit.protocols.internet.ipv6 import IPv6 @@ -112,9 +115,8 @@ def test_payload_is_found_by_name_not_as_the_last_key(self) -> None: # after the payload, it is the last nested key. info = moved(ip6.info, 'udp', before='fragment') exported = info.to_dict() - names, _ = IPv6._lookup_exthdr(exported) - self.assertEqual(names, ('frag',)) - self.assertEqual(IPv6._lookup_payload(exported, exclude=names), (UDP_, 'udp')) + self.assertEqual(list(exported['extension_headers']), ['frag']) + self.assertEqual(IPv6._lookup_payload(exported), (UDP_, 'udp')) self.assertEqual(IPv6.from_data(exported).data, ATOMIC) def test_cut_upper_layer_is_found_ahead_of_the_headers(self) -> None: @@ -131,12 +133,11 @@ def test_cut_upper_layer_is_found_ahead_of_the_headers(self) -> None: self.assertIn('raw', ip6.info) self.assertEqual(len(ip6.data), len(HOP_DST_DST)) - # As on PyPy for #1610: ``opts`` after the upper layer. - info = moved(moved(ip6.info, 'raw', before='opts'), 'hopopt', before='raw') + # As on PyPy for #1610: the headers after the upper layer. + info = moved(ip6.info, 'raw', before='extension_headers') exported = info.to_dict() - names, _ = IPv6._lookup_exthdr(exported) - self.assertEqual(sorted(names), ['hopopt', 'opts']) - self.assertEqual(IPv6._lookup_payload(exported, exclude=names), (Raw, 'raw')) + self.assertEqual(len(IPv6._lookup_exthdr(exported)), 2) + self.assertEqual(IPv6._lookup_payload(exported), (Raw, 'raw')) self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) diff --git a/tests/protocols/test_option_coverage_runtime.py b/tests/protocols/test_option_coverage_runtime.py index 6c511312e3..7e167d51d2 100644 --- a/tests/protocols/test_option_coverage_runtime.py +++ b/tests/protocols/test_option_coverage_runtime.py @@ -186,7 +186,7 @@ def test_the_jumbo_payload_frame_is_a_genuine_jumbogram(self) -> None: self.assertEqual(str(frame.protochain), 'Ethernet:IPv6:HOPOPT:UDP:Raw') ipv6 = frame[IPv6].info - self.assertEqual(ipv6.hopopt.options[Option.Jumbo_Payload].jumbo_len, jumbo) + self.assertEqual(ipv6.extension_headers['hopopt'].options[Option.Jumbo_Payload].jumbo_len, jumbo) self.assertEqual(ipv6.hdr_len + ipv6.raw_len, 40 + jumbo) def test_option_captures_dump_to_tree_and_json(self) -> None: diff --git a/tests/protocols/test_protocol_base_unit.py b/tests/protocols/test_protocol_base_unit.py index 5db35cd3c1..70e00089ca 100644 --- a/tests/protocols/test_protocol_base_unit.py +++ b/tests/protocols/test_protocol_base_unit.py @@ -653,7 +653,7 @@ def test_outer_address_reaches_a_schema_that_needs_it(self) -> None: + hopopt) parsed = IPv6(io.BytesIO(packet), len(packet)) - option = list(parsed.info.hopopt.options.values())[0] + option = list(parsed.info.extension_headers['hopopt'].options.values())[0] self.assertEqual(str(parsed.protochain), 'IPv6:HOPOPT') self.assertEqual(parsed.info.src, source) diff --git a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py index 24fb5b6b8b..b6284bc40f 100644 --- a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py +++ b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py @@ -6,12 +6,13 @@ overwrote the first and the dict rebuild silently came back shorter -- Destination Options, Destination Options, UDP rebuilt 56 of 64 octets. -``to_dict`` returns an :class:`~pcapkit.corekit.multidict.OrderedMultiDict` -(#1484): indexing sees the first value of each key, and ``getlist`` gives every -header of a type in wire order. The order across types is that of the keys of -the :class:`~pcapkit.corekit.infoclass.Info`, which is not part of its contract -and differs on PyPy, so the rebuild walks the ``next`` fields instead (#2003). -The dumpers write every header too. +The extension headers are held in ``extension_headers``, an +:class:`~pcapkit.corekit.infoclass.OrderedMultiInfo` that ``to_dict`` exports +as an :class:`~pcapkit.corekit.multidict.OrderedMultiDict` (#1484): indexing +sees the first header of each type, and ``items(multi=True)`` yields every +header in wire order for the rebuild -- the order of the headers, not of the +keys of the :class:`~pcapkit.corekit.infoclass.Info`, which differs on PyPy +(#2003). The dumpers write every header too. Every case builds its own octets and reads no capture. Classes are imported inside each test, after :func:`~tests._support.reimport_once_per_class`. @@ -135,7 +136,7 @@ def test_rebuilt_chain_has_the_parsed_headers(self) -> None: self.assertEqual([ext.data for ext in rebuilt.extension_headers.values()], [ext.data for ext in ip6.extension_headers.values()]) - def test_to_dict_keeps_every_header_of_a_type_in_wire_order(self) -> None: + def test_to_dict_keeps_every_header_in_wire_order(self) -> None: from pcapkit.corekit.multidict import OrderedMultiDict from pcapkit.protocols.internet.ipv6 import IPv6 @@ -144,18 +145,20 @@ def test_to_dict_keeps_every_header_of_a_type_in_wire_order(self) -> None: ip6 = self.frame(name)[IPv6] dict_ = ip6.info.to_dict() self.assertIsInstance(dict_, OrderedMultiDict) + headers = dict_['extension_headers'] + self.assertIsInstance(headers, OrderedMultiDict) - parsed = {} # type: dict[str, list[object]] - for _, ext in ip6.extension_headers.items(multi=True): - parsed.setdefault(IPv6._exthdr_key(ext.alias), []).append(canonical(ext.info.to_dict())) - held = {key: [canonical(val) for val in dict_.getlist(key)] for key in parsed} - self.assertEqual(held, parsed) + # NOTE: As for every OrderedMultiInfo, the export holds the + # headers' infos as they are. + parsed = [(IPv6._exthdr_key(ext.alias), ext.info) + for _, ext in ip6.extension_headers.items(multi=True)] + self.assertEqual(list(headers.items(multi=True)), parsed) # Indexing sees the first of each. first = {} # type: dict[str, object] - for key, val in dict_.items(multi=True): + for key, val in headers.items(multi=True): first.setdefault(key, val) - self.assertEqual({key: dict_[key] for key in dict_}, first) + self.assertEqual({key: headers[key] for key in headers}, first) def test_info_from_dict_keeps_the_repeats(self) -> None: from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6 @@ -196,12 +199,12 @@ def test_info_copy_keeps_its_own_repeats(self) -> None: info = self.frame('dst-dst')[IPv6].info before = list(info.items(multi=True)) - extra = info['opts'] + extra = info['extension_headers'] for clone in (copy.copy(info), copy.deepcopy(info)): with self.subTest(clone=type(clone).__name__): self.assertIsNot(clone.__multi__, info.__multi__) self.assertIsNot(clone.__map__, info.__map__) - clone.__update__(OrderedMultiDict([('opts', extra)])) + clone.__update__(OrderedMultiDict([('extension_headers', extra)])) self.assertEqual(len(list(clone.items(multi=True))), len(before) + 1) self.assertEqual(list(info.items(multi=True)), before) @@ -218,9 +221,9 @@ def test_dumper_writes_every_header(self) -> None: make_dumper(dictdumper.json.JSON)(path)(value, name='Frame 1') with open(path, encoding='utf-8') as file: outputs.append(json.load(file)) - ipv6 = outputs[0]['Frame 1']['ethernet']['ipv6'] - self.assertEqual(len(ipv6['opts']), 2) - self.assertIsInstance(ipv6['route'], dict) + # One single-key mapping per header, in wire order. + headers = outputs[0]['Frame 1']['ethernet']['ipv6']['extension_headers'] + self.assertEqual([list(header) for header in headers], [['opts'], ['route'], ['opts']]) # Handed the export, the dumper writes the top level the same way. self.assertEqual(list(outputs[1]['Frame 1']), list(outputs[0]['Frame 1'])) From 6a84be9b8154ddc0b0a10243894ab33bb9880ed9 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 21 Oct 2026 12:06:23 -0400 Subject: [PATCH 3/6] fix(protocols): carry the dispatch keys in to_dict, guess no payload (#2003) ``__next_name__`` and ``__next_type__`` are no longer excluded, so ``from_data(info.to_dict())`` dispatches the payload exactly as ``from_data(info)`` does, whatever the key order. - ``ProtocolBase._lookup_payload`` and ``_lookup_nested`` are gone: the payload is dispatched on the two keys only. A dict without them, or without the key ``__next_name__`` names, rebuilds no payload, as for a protocol with no next layer; the ``exclude`` parameter of ``_make_payload`` goes with them. - ``Info.__repr__``/``__str__`` and the dumpers leave every dunder key out, so no dump carries ``__next_type__`` or ``__short_read__``. - ``Data``, ``from_data`` and ``_make_payload`` document that a hand-written dict must carry the metadata, and for IPv6 the headers under ``extension_headers``. The #1447 test now pins that ``to_dict`` carries the keys and a dump does not; the #2003 tests pin the rebuild on reordered keys and that a dict without the keys is not guessed at. --- pcapkit/corekit/infoclass.py | 17 ++- pcapkit/dumpkit/common.py | 17 ++- pcapkit/protocols/data/data.py | 21 ++- pcapkit/protocols/internet/internet.py | 12 +- pcapkit/protocols/misc/pcap/frame.py | 5 +- pcapkit/protocols/misc/pcapng.py | 5 +- pcapkit/protocols/protocol.py | 138 ++++-------------- tests/corekit/test_infoclass.py | 2 - tests/dumpkit/test_capture_dump_runtime.py | 5 +- .../test_repeated_field_dump_1484_unit.py | 4 +- .../test_lookup_by_name_2003_unit.py | 53 ++++--- ...est_to_dict_payload_roundtrip_1447_unit.py | 61 ++++++-- 12 files changed, 177 insertions(+), 163 deletions(-) diff --git a/pcapkit/corekit/infoclass.py b/pcapkit/corekit/infoclass.py index 7964d6a7be..af0be0d0cb 100644 --- a/pcapkit/corekit/infoclass.py +++ b/pcapkit/corekit/infoclass.py @@ -553,7 +553,9 @@ def __update__(self, dict_: 'Optional[Mapping[str, VT] | Iterable[tuple[str, VT] def __str__(self) -> 'str': temp = [] # type: list[str] for (key, value) in self.__dict__.items(): - if key in self.__excluded__: + # NOTE: a dunder key, such as ``__next_type__``, is metadata, and + # left out as the dumpers leave it out (#2003). + if key in self.__excluded__ or _is_dunder(key): continue out_key = self.__map_reverse__.get(key, key) @@ -564,7 +566,7 @@ def __str__(self) -> 'str': def __repr__(self) -> 'str': temp = [] # type: list[str] for (key, value) in self.__dict__.items(): - if key in self.__excluded__: + if key in self.__excluded__ or _is_dunder(key): continue out_key = self.__map_reverse__.get(key, key) @@ -1053,6 +1055,17 @@ def to_dict(self, flat: 'Optional[bool]' = None) -> 'Any': return OrderedMultiDict.to_dict(self, flat) # type: ignore[call-overload] +def _is_dunder(key: 'Any') -> 'bool': + """Whether ``key`` is a dunder key, such as ``__next_type__``. + + Such a key is metadata rather than a field: :meth:`Info.to_dict` carries + it, while ``repr``, ``str`` and the dumpers of :mod:`pcapkit.dumpkit` leave + it out (:issue:`2003`). + + """ + return isinstance(key, str) and len(key) > 4 and key.startswith('__') and key.endswith('__') + + def _freeze(value: 'Any') -> 'Any': """Return ``value`` as stored in a finalised :class:`Info`. diff --git a/pcapkit/dumpkit/common.py b/pcapkit/dumpkit/common.py index d025491560..78d5d47399 100644 --- a/pcapkit/dumpkit/common.py +++ b/pcapkit/dumpkit/common.py @@ -29,7 +29,7 @@ import dictdumper.plist import dictdumper.tree -from pcapkit.corekit.infoclass import Info +from pcapkit.corekit.infoclass import Info, _is_dunder from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict from pcapkit.corekit.sentinels import NoValueType, NullType from pcapkit.protocols.schema.schema import Schema @@ -423,10 +423,13 @@ def fields(info: 'Info | MultiDict[Any, Any]') -> 'dict[Any, Any]': held more than once, such as a repeated IPv6 extension header, maps to the :class:`list` of all its values, in order (GitHub issue :issue:`1484`). The values are left as they are, so each comes back - through :meth:`DictDumper.object_hook` when written. + through :meth:`DictDumper.object_hook` when written. A dunder key, + such as ``__next_type__``, is metadata for :meth:`from_data + ` and is left out + (:issue:`2003`). """ - pairs = list(info.items(multi=True)) + pairs = [(key, val) for key, val in info.items(multi=True) if not _is_dunder(key)] counts = collections.Counter(key for key, _ in pairs) result = {} # type: dict[Any, Any] for key, val in pairs: @@ -540,16 +543,20 @@ def object_hook(self, o: 'Any') -> 'Any': # by name could only gather the repeats of a key under its first # occurrence (GitHub issue #1263). The keys are escaped when the # writer hands each mapping back to this hook, through the - # :class:`dict` branch below. + # :class:`dict` branch below. A dunder key, which an export of + # an :class:`Info` holds, is left out, as :func:`fields` leaves + # it out (#2003). result = [ {render_enum(key) if isinstance(key, (enum.Enum, aenum.Enum)) else key: val} - for key, val in o.items(multi=True) + for key, val in o.items(multi=True) if not _is_dunder(key) ] elif isinstance(o, MultiDict): # NOTE: a :class:`MultiDict` keeps no order across keys, only # among the values of each, so grouping by key loses nothing. temp = collections.defaultdict(list) # type: DefaultDict[str, list[Any]] for key, val in o.items(multi=True): + if _is_dunder(key): + continue if isinstance(key, (enum.Enum, aenum.Enum)): key = render_enum(key) temp[escape_key(key)].append(val) diff --git a/pcapkit/protocols/data/data.py b/pcapkit/protocols/data/data.py index 569c29dbf5..92f83fe2a5 100644 --- a/pcapkit/protocols/data/data.py +++ b/pcapkit/protocols/data/data.py @@ -14,9 +14,20 @@ class Data(Info): - """Base class for data models.""" + """Base class for data models. - __excluded__ = ['__next_name__', '__next_type__'] + The ``__next_name__`` and ``__next_type__`` keys are metadata: they name + the payload, and :meth:`Protocol.from_data + ` dispatches the payload + on them and on nothing else. :meth:`to_dict` carries them, so that + ``from_data(info.to_dict())`` rebuilds as ``from_data(info)`` does, while + the dumpers and ``repr`` leave out every such dunder key. A hand-written + :obj:`dict` must carry them as well -- and, for IPv6, the extension + headers under ``extension_headers`` -- for the payload to be rebuilt: + without them, no payload layer is dispatched, as for a protocol with no + next layer (:issue:`2003`). + + """ if TYPE_CHECKING: #: Next field name, i.e., the name of the payload field. @@ -24,8 +35,8 @@ class Data(Info): #: Next field type, i.e., the type of the payload field. __next_type__: 'Type[ProtocolBase]' #: Name of the header field the data ended inside, and how many of its - #: octets were there; present only then. Not excluded, so that - #: :meth:`to_dict` carries it and ``from_data(info.to_dict())`` cuts - #: the rebuild back as ``from_data(info)`` does. See + #: octets were there; present only then. :meth:`to_dict` carries it, + #: so that ``from_data(info.to_dict())`` cuts the rebuild back as + #: ``from_data(info)`` does. See #: :func:`~pcapkit.protocols.schema.schema.keep_short_read`. __short_read__: 'tuple[str, int]' diff --git a/pcapkit/protocols/internet/internet.py b/pcapkit/protocols/internet/internet.py index 8c4f3863bb..dfc7dee394 100644 --- a/pcapkit/protocols/internet/internet.py +++ b/pcapkit/protocols/internet/internet.py @@ -325,7 +325,7 @@ def _rebuilt_exthdr_depth(cls) -> 'int': return _exthdr_rebuilt.get() if issubclass(cls, IPv6_Ext) else 0 @classmethod - def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'ProtocolBase': + def _make_payload(cls, data: 'Data') -> 'ProtocolBase': """Create payload from ``data`` for protocol construction. As :meth:`ProtocolBase._make_payload @@ -338,9 +338,6 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc Args: data: protocol data - exclude: keys of ``data`` that are not the payload, see - :meth:`ProtocolBase._make_payload - ` Returns: Payload for protocol construction. @@ -352,7 +349,7 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc # where it does not parse), so no rebuilt upper layer reads this place. token = _exthdr_rebuilt.set(cls._rebuilt_exthdr_depth() + 1) try: - return super()._make_payload(data, exclude) + return super()._make_payload(data) finally: _exthdr_rebuilt.reset(token) @@ -430,8 +427,9 @@ def _decode_next_layer(self, dict_: '_PT', proto: 'Optional[int]' = None, # pyl Notes: ``dict_`` gains the keys ``__next_type__`` (next layer protocol - type) and ``__next_name__`` (next layer protocol name). Neither - is included when :meth:`Info.to_dict ` is called. + type) and ``__next_name__`` (next layer protocol name), which + :meth:`Info.to_dict ` + carries and the dumpers leave out. """ next_ = cast('ProtocolBase', # type: ignore[redundant-cast] diff --git a/pcapkit/protocols/misc/pcap/frame.py b/pcapkit/protocols/misc/pcap/frame.py index 11af4bc37d..994da73fa7 100644 --- a/pcapkit/protocols/misc/pcap/frame.py +++ b/pcapkit/protocols/misc/pcap/frame.py @@ -633,8 +633,9 @@ def _decode_next_layer(self, dict_: 'Data_Frame', proto: 'Optional[int]' = None, Notes: We added a new key ``__next_type__`` to ``dict_`` to store the next layer protocol type, and a new key ``__next_name__`` to - store the next layer protocol name. These keys will **NOT** - be included when :meth:`Info.to_dict ` is called. + store the next layer protocol name. :meth:`Info.to_dict + ` carries both, and the + dumpers leave them out. We also added a new key ``protocols`` to ``dict_`` to store the protocol chain of the current packet (frame). diff --git a/pcapkit/protocols/misc/pcapng.py b/pcapkit/protocols/misc/pcapng.py index 40f703f396..3e72a165e8 100644 --- a/pcapkit/protocols/misc/pcapng.py +++ b/pcapkit/protocols/misc/pcapng.py @@ -2142,8 +2142,9 @@ def _decode_next_layer(self, dict_: 'Data_PCAPNG', proto: 'Optional[int]' = None Notes: We added a new key ``__next_type__`` to ``dict_`` to store the next layer protocol type, and a new key ``__next_name__`` to - store the next layer protocol name. These keys will **NOT** - be included when :meth:`Info.to_dict ` is called. + store the next layer protocol name. :meth:`Info.to_dict + ` carries both, and the + dumpers leave them out. We also added a new key ``protocols`` to ``dict_`` to store the protocol chain of the current packet (frame). diff --git a/pcapkit/protocols/protocol.py b/pcapkit/protocols/protocol.py index 8b74fd1166..fcd3f223ef 100644 --- a/pcapkit/protocols/protocol.py +++ b/pcapkit/protocols/protocol.py @@ -59,7 +59,7 @@ from pcapkit.protocols.schema.schema import Schema from pcapkit.utilities import config from pcapkit.utilities.chardet import detect -from pcapkit.utilities.compat import cached_property, get_annotations +from pcapkit.utilities.compat import cached_property from pcapkit.utilities.exceptions import (BaseError, ProtocolError, ProtocolNotFound, ProtocolNotImplemented, RegistryError, StructError, UnsupportedCall, stacklevel) @@ -1129,6 +1129,14 @@ def from_schema(cls, schema: '_ST | dict[str, Any]') -> 'Self': def from_data(cls, data: '_PT | dict[str, Any]', **kwargs: 'Any') -> 'Self': """Create protocol instance from data. + ``data`` may be the :obj:`dict` that :meth:`Info.to_dict + ` returns, which carries the + ``__next_name__`` and ``__next_type__`` metadata the payload is + dispatched on. A hand-written :obj:`dict` must carry them too -- and, + for :class:`~pcapkit.protocols.internet.ipv6.IPv6`, the extension + headers under ``extension_headers`` -- or no payload is rebuilt, see + :meth:`_make_payload`. + Args: data: Protocol data. **kwargs: Construction keywords that ``data`` does not carry, such as @@ -2100,19 +2108,22 @@ def _make_data(cls, data: 'Data') -> 'dict[str, Any]': return data.to_dict() @classmethod - def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'ProtocolBase': + def _make_payload(cls, data: 'Data') -> 'ProtocolBase': """Create payload from ``data`` for protocol construction. This method uses ``__next_type__`` and ``__next_name__`` to - determine the payload type and name. If either of them is - :data:`None`, a :class:`~pcapkit.protocols.misc.null.NoPayload` - instance will be returned. Otherwise, the payload will be - constructed by :meth:`Protocol.from_data `. - - :meth:`Info.to_dict ` leaves - both keys out, so ``data`` rebuilt from such a :obj:`dict` carries - neither. The payload is then found from the structure of ``data`` - instead, by ``_lookup_payload()``. + determine the payload type and name. If either of them is missing or + :data:`None`, or ``data`` holds nothing under ``__next_name__``, a + :class:`~pcapkit.protocols.misc.null.NoPayload` instance will be + returned, exactly as for a protocol with no next layer. Otherwise, the + payload will be constructed by :meth:`Protocol.from_data + `. + + :meth:`Info.to_dict ` carries + both keys, so ``data`` rebuilt from such a :obj:`dict` dispatches as + the info did. A hand-written :obj:`dict` must carry them too: the + payload is never guessed from the other keys of ``data`` + (:issue:`2003`). The payload is built one layer further into the frame than this protocol, as :meth:`_parse_next_layer` counts it, since each layer built @@ -2124,34 +2135,15 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc Args: data: protocol data - exclude: keys of ``data`` that are not the payload, though they may - look like it, passed on to ``_lookup_payload()`` Returns: Payload for protocol construction. """ - if '__next_type__' not in data and '__next_name__' not in data: - proto, name = cls._lookup_payload(data, exclude) - if proto is None or name is None: - from pcapkit.protocols.misc.null import \ - NoPayload # pylint: disable=import-outside-toplevel - return NoPayload() - token, rebuilding = _frame_depth.set(_frame_depth.get() + 1), _rebuilding.set(True) - try: - return proto.from_data(data[name]) - finally: - _rebuilding.reset(rebuilding) - _frame_depth.reset(token) - proto = cast('Optional[Type[Protocol]]', data.get('__next_type__')) - if proto is None or not (isinstance(proto, type) and issubclass(proto, ProtocolBase)): - from pcapkit.protocols.misc.null import \ - NoPayload # pylint: disable=import-outside-toplevel - return NoPayload() - name = cast('Optional[str]', data.get('__next_name__')) - if name is None: + if (proto is None or not (isinstance(proto, type) and issubclass(proto, ProtocolBase)) + or name is None or name not in data): from pcapkit.protocols.misc.null import \ NoPayload # pylint: disable=import-outside-toplevel return NoPayload() @@ -2163,81 +2155,6 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc _rebuilding.reset(rebuilding) _frame_depth.reset(token) - @classmethod - def _lookup_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'tuple[Optional[Type[ProtocolBase]], Optional[str]]': # pylint: disable=line-too-long - """Find the payload of ``data`` that carries no ``__next_type__``. - - :meth:`_decode_next_layer` writes the payload's info under the payload's - :attr:`info_name`, so the payload is the key, among those the data - model does not declare and whose value is a mapping, that is the - :attr:`info_name` of a protocol of :attr:`__proto__` -- or of its - fallback. That is the lowercased class name by default, but a protocol - may override it: every version of - :class:`~pcapkit.protocols.link.l2tp.L2TP` is ``l2tp``. - - The payload is found by that name, and never by its place among the - keys: the order of the keys of an :class:`~pcapkit.corekit.infoclass.Info` - is not part of its contract, and on PyPy it follows the attribute - layout of an earlier instance of the same class (:issue:`2003`). Should - ``data`` hold the names of several such protocols, the first of them in - :attr:`__proto__`, its fallback last, is the payload. - - Args: - data: protocol data - exclude: keys of ``data`` to pass over, such as the IPv6 extension - headers, which are written like a payload but are not one - - Returns: - The payload protocol and the key of its info in ``data``, or - :data:`None` for both if ``data`` has no payload this protocol can - dispatch to. - - """ - keys = {key for key in cls._lookup_nested(data) if key not in exclude} - if not keys: - return None, None - - registry = cls.__proto__ - candidates = list(registry.values()) - if registry.default_factory is not None: - candidates.append(registry.default_factory()) - for entry in candidates: - proto = entry.klass if isinstance(entry, ModuleDescriptor) else entry - if not (isinstance(proto, type) and issubclass(proto, ProtocolBase)): - continue - - # NOTE: :attr:`info_name` is an instance property, so it is read off - # a bare instance; ``__new__`` only sets up the per-instance caches. - # An abstract class cannot be instantiated, and is never a payload. - try: - label = proto.__new__(proto).info_name - except TypeError: - continue - if label in keys: - return proto, label - return None, None - - @staticmethod - def _lookup_nested(data: 'Data') -> 'list[str]': - """Keys of ``data`` that its data model does not declare and that hold a mapping. - - These are where the next layer protocols wrote their info -- the - payload, and the IPv6 extension headers. - - Args: - data: protocol data - - Returns: - The keys, in the order of ``data``, which no caller relies on - (:issue:`2003`). - - """ - declared = set() # type: set[str] - for klass in type(data).mro(): - declared.update(get_annotations(klass)) - return [key for key in data - if key not in declared and isinstance(data[key], collections.abc.Mapping)] - @staticmethod def _lookup_registry(registry: 'DefaultDict[Any, _VT]', code: 'Any') -> '_VT': """Look up a dispatch :term:`registry` entry without recording a miss. @@ -2335,9 +2252,10 @@ def _decode_next_layer(self, dict_: '_PT', proto: 'int', length: 'Optional[int]' Notes: We added a new key ``__next_type__`` to ``dict_`` to store the next layer protocol type, and a new key ``__next_name__`` to - store the next layer protocol name. These keys will **NOT** - be included when :meth:`Info.to_dict ` is called; - ``_lookup_payload()`` finds the payload of such a :obj:`dict` instead. + store the next layer protocol name. :meth:`Info.to_dict + ` carries both, so that + :meth:`_make_payload` dispatches a :obj:`dict` rebuild on them, + while the dumpers and ``repr`` leave them out (:issue:`2003`). """ next_ = cast('ProtocolBase', self._import_next_layer(proto, length, packet=packet)) # type: ignore[misc,call-arg,redundant-cast] diff --git a/tests/corekit/test_infoclass.py b/tests/corekit/test_infoclass.py index 41e8a41b72..2a43e8ee7a 100644 --- a/tests/corekit/test_infoclass.py +++ b/tests/corekit/test_infoclass.py @@ -193,11 +193,9 @@ class BareChild(Bare): def test_annotations_are_read_through_the_compat_shim(self) -> None: """GitHub issue #1709: one version-gated reader, from :mod:`pcapkit.utilities.compat`.""" from pcapkit.corekit import infoclass - from pcapkit.protocols import protocol from pcapkit.utilities import compat self.assertIs(infoclass.get_annotations, compat.get_annotations) - self.assertIs(protocol.get_annotations, compat.get_annotations) def test_the_pre_3_10_reader_gives_the_same_signatures(self) -> None: # NOTE: a stand-in for ``compat.get_annotations`` below 3.10 -- the diff --git a/tests/dumpkit/test_capture_dump_runtime.py b/tests/dumpkit/test_capture_dump_runtime.py index 5d426cbe84..250809fc78 100644 --- a/tests/dumpkit/test_capture_dump_runtime.py +++ b/tests/dumpkit/test_capture_dump_runtime.py @@ -170,7 +170,10 @@ def test_json_frame_sections_carry_every_info_key_and_the_chain(self) -> None: for number, frame in enumerate(extractor.frame, start=1): section = report[f'Frame {number}'] with self.subTest(capture=name, frame=number): - self.assertEqual(list(section), list(frame.info.to_dict())) + # Every key but the dunder metadata, which the dumps + # leave out (#2003). + self.assertEqual(list(section), [key for key in frame.info.to_dict() + if not (key.startswith('__') and key.endswith('__'))]) self.assertEqual(section['protocols'], str(frame.protochain)) def test_plist_dump_reads_back_through_plistlib(self) -> None: diff --git a/tests/dumpkit/test_repeated_field_dump_1484_unit.py b/tests/dumpkit/test_repeated_field_dump_1484_unit.py index fc1aef50cc..453b36f45a 100644 --- a/tests/dumpkit/test_repeated_field_dump_1484_unit.py +++ b/tests/dumpkit/test_repeated_field_dump_1484_unit.py @@ -126,7 +126,9 @@ def test_the_writer_takes_the_info_or_its_export(self) -> None: tops.append(json.load(file)['Frame 1']) self.assertEqual(pads(opts(tops[0]['ethernet']['ipv6'])), [f'000000{mark:02x}' for mark in MARKS]) self.assertEqual(list(tops[1]), list(tops[0])) - self.assertEqual(list(tops[0]), list(frame.info.to_dict())) + # Every key but the dunder metadata, which the dumps leave out (#2003). + self.assertEqual(list(tops[0]), [key for key in frame.info.to_dict() + if not (key.startswith('__') and key.endswith('__'))]) if __name__ == '__main__': diff --git a/tests/protocols/test_lookup_by_name_2003_unit.py b/tests/protocols/test_lookup_by_name_2003_unit.py index 79e77ab835..8d944b80f0 100644 --- a/tests/protocols/test_lookup_by_name_2003_unit.py +++ b/tests/protocols/test_lookup_by_name_2003_unit.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""A dict rebuild finds the payload and the extension headers whatever the key order. +"""A dict rebuild dispatches as the info's, whatever the key order. GitHub issue #2003: on PyPy, an :class:`~pcapkit.corekit.infoclass.Info` lays its keys out as an earlier instance of the same class did, so its key @@ -10,11 +10,13 @@ ``hopopt`` had moved past the upper layer. The order of an :class:`~pcapkit.corekit.infoclass.Info` is not part of its -contract, so the payload is now found by the name of a protocol it dispatches -to, and the chain is held in ``extension_headers``, an +contract, so nothing is read off it any more: ``to_dict()`` carries the +``__next_type__`` and ``__next_name__`` metadata the payload is dispatched on, +and the chain is held in ``extension_headers``, an :class:`~pcapkit.corekit.infoclass.OrderedMultiInfo` keeping the order of its -own. These tests reorder an instance's -:attr:`~object.__dict__` as PyPy does, so CPython checks the same thing. +own. A dict without the metadata is not guessed at. These tests reorder an +instance's :attr:`~object.__dict__` as PyPy does, so CPython checks the same +thing. Every case builds its own octets and reads no capture. Classes are imported inside each test, after :func:`~tests._support.reimport_once_per_class`. @@ -99,11 +101,11 @@ def test_extension_headers_keep_their_order_whatever_the_key_order(self) -> None exported = moved(ip6.info, 'extension_headers', before=before).to_dict() chain = IPv6._lookup_exthdr(exported) self.assertEqual([parser for parser, _ in chain], [HOPOPT, IPv6_Opts, IPv6_Opts]) - self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) self.assertEqual([parser.from_data(hdr).data for parser, hdr in chain], [ext.data for _, ext in ip6.extension_headers.items(multi=True)]) + self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) - def test_payload_is_found_by_name_not_as_the_last_key(self) -> None: + def test_payload_follows_the_metadata_whatever_the_key_order(self) -> None: from pcapkit.protocols.internet.ipv6 import IPv6 from pcapkit.protocols.transport.udp import UDP as UDP_ @@ -111,13 +113,15 @@ def test_payload_is_found_by_name_not_as_the_last_key(self) -> None: self.assertIn('fragment', ip6.info) self.assertEqual(len(ip6.data), len(ATOMIC)) - # ``fragment`` is a nested key too, and not the payload; laid out - # after the payload, it is the last nested key. - info = moved(ip6.info, 'udp', before='fragment') + # ``fragment`` is a mapping too, and not the payload; laid out after + # the payload, it is the last of them, and the metadata first. + info = moved(moved(ip6.info, 'udp', before='fragment'), '__next_name__', before='version') exported = info.to_dict() - self.assertEqual(list(exported['extension_headers']), ['frag']) - self.assertEqual(IPv6._lookup_payload(exported), (UDP_, 'udp')) - self.assertEqual(IPv6.from_data(exported).data, ATOMIC) + self.assertEqual(exported['__next_name__'], 'udp') + self.assertIs(exported['__next_type__'], UDP_) + rebuilt = IPv6.from_data(exported) + self.assertIsInstance(rebuilt.payload, UDP_) + self.assertEqual(rebuilt.data, ATOMIC) def test_cut_upper_layer_is_found_ahead_of_the_headers(self) -> None: from pcapkit.protocols.internet.ipv6 import IPv6 @@ -134,12 +138,27 @@ def test_cut_upper_layer_is_found_ahead_of_the_headers(self) -> None: self.assertEqual(len(ip6.data), len(HOP_DST_DST)) # As on PyPy for #1610: the headers after the upper layer. - info = moved(ip6.info, 'raw', before='extension_headers') - exported = info.to_dict() + exported = moved(ip6.info, 'raw', before='extension_headers').to_dict() self.assertEqual(len(IPv6._lookup_exthdr(exported)), 2) - self.assertEqual(IPv6._lookup_payload(exported), (Raw, 'raw')) - self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) + rebuilt = IPv6.from_data(exported) + self.assertIsInstance(rebuilt.payload, Raw) + self.assertEqual(rebuilt.data, HOP_DST_DST) + def test_dict_without_the_metadata_is_not_guessed_at(self) -> None: + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.misc.null import NoPayload + + ip6 = IPv6(HOP_DST_DST) + exported = ip6.info.to_dict() + for key in ('__next_type__', '__next_name__'): + exported.poplist(key) + self.assertIn('udp', exported) + + # The extension headers are rebuilt from ``extension_headers``, and the + # upper layer, named by nothing, is not. + rebuilt = IPv6.from_data(exported) + self.assertIsInstance(rebuilt.payload, NoPayload) + self.assertEqual(rebuilt.data, HOP_DST_DST[:40 + 3 * 8]) if __name__ == '__main__': unittest.main() diff --git a/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py b/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py index edc9cfd4e0..ce4cc5ee32 100644 --- a/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py +++ b/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py @@ -8,8 +8,9 @@ its 14-octet header. IPv6 lost its extension headers the same way, with ``__exthdr__``, and HTTP raised ``ProtocolError('invalid HTTP data: Raw')``. -The dict itself is left as it was, since the dumps are written from it; the -payload is found from the dict's structure instead. +``to_dict`` now carries both keys, so the dict rebuild dispatches exactly as +the info's, and the dumps leave every dunder key out (#2003). A dict without +them is not guessed at: no payload is rebuilt. Every case builds its own octets and reads no capture. Classes are imported inside each test, after :func:`~tests._support.reimport_once_per_class`. @@ -135,7 +136,6 @@ def test_ipv6_extension_headers_rebuild_from_its_dict(self) -> None: ipv6 = self.frame('ipv6-exthdr-udp')[IPv6] dict_ = ipv6.info.to_dict() - self.assertNotIn('__exthdr__', dict_) rebuilt = IPv6.from_data(dict_) self.assertEqual(rebuilt.data, ipv6.data) @@ -144,7 +144,7 @@ def test_ipv6_extension_headers_rebuild_from_its_dict(self) -> None: ['HOPOPT', 'IPv6_Opts'], ) - def test_payload_is_found_by_info_name_not_class_name(self) -> None: + def test_payload_is_found_under_its_info_name(self) -> None: from pcapkit.protocols.link.l2tpv2 import L2TPv2 from pcapkit.protocols.transport.udp import UDP @@ -178,15 +178,58 @@ def test_dict_without_a_payload_key_rebuilds_the_header_alone(self) -> None: self.assertIsInstance(rebuilt.payload, NoPayload) self.assertEqual(rebuilt.data, FRAMES['ipv4-udp'][:14]) - def test_to_dict_still_leaves_the_dispatch_keys_out(self) -> None: - # The dumps are written from ``to_dict``, so the fix must not add to it. + def test_dict_without_the_dispatch_keys_rebuilds_the_header_alone(self) -> None: + from pcapkit.protocols.link.ethernet import Ethernet + from pcapkit.protocols.misc.null import NoPayload + + # The payload is still there under its name, but nothing names it. + eth = Ethernet(FRAMES['ipv4-udp']) + dict_ = eth.info.to_dict() + for key in ('__next_type__', '__next_name__'): + dict_.poplist(key) + self.assertIn('ipv4', dict_) + + rebuilt = Ethernet.from_data(dict_) + self.assertIsInstance(rebuilt.payload, NoPayload) + self.assertEqual(rebuilt.data, FRAMES['ipv4-udp'][:14]) + + def test_repr_and_str_leave_the_dispatch_keys_out(self) -> None: + for name in FRAMES: + info = self.frame(name).info + self.assertIn('__next_type__', info) + with self.subTest(frame=name): + self.assertNotIn('__next', repr(info)) + self.assertNotIn('__next', str(info)) + + def test_to_dict_carries_the_dispatch_keys_and_a_dump_none(self) -> None: + import json + import os + import tempfile + + import dictdumper.json + + from pcapkit.dumpkit.common import make_dumper + for name in FRAMES: + self.assertIn('__next_type__', self.frame(name).info.to_dict()) for layer in self.layers(name): with self.subTest(frame=name, layer=type(layer).__name__): dict_ = layer.info.to_dict() - for key in ('__next_type__', '__next_name__', '__exthdr__'): - self.assertNotIn(key, dict_) - + for key in ('__next_type__', '__next_name__'): + self.assertEqual(key in dict_, key in layer.info) + if key in dict_: + self.assertIs(dict_[key], layer.info[key]) + + frame = self.frame(name) + with tempfile.TemporaryDirectory() as tmp: + for index, value in enumerate((frame.info, frame.info.to_dict())): + with self.subTest(frame=name, dumped=type(value).__name__): + path = os.path.join(tmp, f'{index}.json') + make_dumper(dictdumper.json.JSON)(path)(value, name='Frame 1') + with open(path, encoding='utf-8') as file: + text = file.read() + json.loads(text) + self.assertNotIn('"__', text) if __name__ == '__main__': unittest.main() From 8db96cc3f1ea46e12b33729f8ed7b26708458c84 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 21 Oct 2026 13:03:49 -0400 Subject: [PATCH 4/6] fix(protocols): rebuild the IPv6 chain from __exthdr__ alone (#2003) Reverts the extension_headers field of cf6931e8f and carries the metadata through to_dict instead. - Data_IPv6 keeps its per-header keys (repeats kept, #1453). Its __exthdr__ is an OrderedMultiInfo keyed by ExtensionHeader, of (parser, info) in wire order, so the #1473 IPv6_Ext fallback rides along. _lookup_exthdr is gone: the chain comes from __exthdr__ only, and a dict without it rebuilds no extension header. - Info.__included__, merged across bases like __excluded__, names the dunder keys iteration, len() and to_dict carry; every other dunder key is left out. __excluded__ now means only "left out of repr, str and the dumps". Data names __next_name__, __next_type__ and __short_read__ in both, Data_IPv6 adds __exthdr__; HTTP/2 Flags includes __value__ and PCAP-NG blocks __truncated_raw__, both still dumped. - to_dict's export records its __excluded__ (kept by copy, deep copy and pickle), and the dumpers skip those keys; _is_dunder is gone. - The hand-written-dict docs name every metadata key. Tests pin the restored shape, order independence on a reordered __dict__, __included__ inheritance, and dumps/repr without the keys. --- pcapkit/corekit/infoclass.py | 131 +++++++++++++----- pcapkit/dumpkit/common.py | 44 ++++-- pcapkit/protocols/data/application/httpv2.py | 4 + pcapkit/protocols/data/data.py | 24 ++-- pcapkit/protocols/data/internet/ipv6.py | 19 ++- pcapkit/protocols/data/misc/pcapng.py | 5 + pcapkit/protocols/internet/ipv6.py | 109 ++++----------- pcapkit/protocols/protocol.py | 18 +-- tests/corekit/test_info_roundtrip_unit.py | 3 + tests/corekit/test_infoclass.py | 44 +++++- tests/dumpkit/test_capture_dump_runtime.py | 5 +- .../test_repeated_field_dump_1484_unit.py | 25 ++-- tests/protocols/internet/test_esp_unit.py | 2 +- .../internet/test_ipv6_core_roundtrip_unit.py | 2 +- .../test_ipv6_exthdr_cut_rebuild_unit.py | 2 +- .../internet/test_ipv6_from_data_ext_unit.py | 27 ++-- tests/protocols/internet/test_ipv6_unit.py | 2 +- .../internet/test_mpl_seed_rebuild_unit.py | 2 +- .../test_declared_length_overrun_unit.py | 2 +- .../test_lookup_by_name_2003_unit.py | 46 ++++-- .../protocols/test_option_coverage_runtime.py | 2 +- tests/protocols/test_protocol_base_unit.py | 2 +- .../test_short_read_roundtrip_unit.py | 28 +++- ...est_to_dict_payload_roundtrip_1447_unit.py | 4 +- .../test_to_dict_repeated_exthdr_1453_unit.py | 41 +++--- 25 files changed, 356 insertions(+), 237 deletions(-) diff --git a/pcapkit/corekit/infoclass.py b/pcapkit/corekit/infoclass.py index af0be0d0cb..e790a2953b 100644 --- a/pcapkit/corekit/infoclass.py +++ b/pcapkit/corekit/infoclass.py @@ -21,7 +21,7 @@ import sys import types import typing -from typing import TYPE_CHECKING, Generic, TypeVar +from typing import TYPE_CHECKING, Generic, TypeVar, cast from pcapkit.corekit.enum import EnumLookup from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict, iter_multi_items @@ -30,7 +30,8 @@ from pcapkit.utilities.warnings import InfoWarning, warn if TYPE_CHECKING: - from typing import Any, ItemsView, Iterable, Iterator, NoReturn, Optional, Type + from typing import (Any, Collection, ItemsView, Iterable, Iterator, NoReturn, Optional, + SupportsIndex, Type) from typing_extensions import Self @@ -255,11 +256,13 @@ class InfoMeta(abc.ABCMeta): :class:`Info` class. It can be useful to reduce runtime generation cost as well as caching already generated attributes. - * :attr:`Info.__additional__` and :attr:`Info.__excluded__` are - lists of additional and excluded field names, which are used to - determine certain names to be included or excluded from the field - dictionary. They will be automatically populated from the class - attributes of the :class:`Info` class and its base classes. + * :attr:`Info.__additional__`, :attr:`Info.__excluded__` and + :attr:`Info.__included__` are lists of additional, excluded and included + field names, which are used to determine certain names to be treated as + built-in names, left out of ``repr`` and the dumps, or carried as + metadata by :meth:`Info.to_dict`. They will be automatically populated + from the class attributes of the :class:`Info` class and its base + classes. .. note:: @@ -286,8 +289,9 @@ class C(B): """ def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]', **kwargs: 'Any') -> 'Type[Info]': - """Create the class, extending its :attr:`Info.__additional__` and - :attr:`Info.__excluded__` with every base's. + """Create the class, extending its :attr:`Info.__additional__`, + :attr:`Info.__excluded__` and :attr:`Info.__included__` with every + base's. Args: name: Class name. @@ -300,6 +304,8 @@ def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]' attrs['__additional__'] = [] if '__excluded__' not in attrs: attrs['__excluded__'] = [] + if '__included__' not in attrs: + attrs['__included__'] = [] for base in bases: if hasattr(base, '__additional__'): @@ -307,6 +313,8 @@ def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]' name for name in base.__additional__ if name not in attrs['__additional__']) if hasattr(base, '__excluded__'): attrs['__excluded__'].extend(name for name in base.__excluded__ if name not in attrs['__excluded__']) + if hasattr(base, '__included__'): + attrs['__included__'].extend(name for name in base.__included__ if name not in attrs['__included__']) return super().__new__(cls, name, bases, attrs, **kwargs) # type: ignore[return-value] @@ -331,6 +339,19 @@ class Info(Mapping[str, VT], Generic[VT], metaclass=InfoMeta): Iterating an instance skips the names listed in :attr:`__excluded__`. + A key whose name is a dunder, such as ``__next_type__``, is metadata. + Iterating an instance, ``len()`` and :meth:`to_dict` leave such a key out + unless :attr:`__included__` names it, and carry every other key; + :attr:`__excluded__` names the keys that ``repr``, ``str`` and the dumps of + :mod:`pcapkit.dumpkit` leave out besides. A subclass extends either list + with its own names, as for :attr:`__additional__`. + + .. versionchanged:: 1.5.0 + + :attr:`__excluded__` names the keys left out of ``repr``, ``str`` and + the dumps only, and :attr:`__included__` the dunder keys that iterating + an instance, ``len()`` and :meth:`to_dict` carry (:issue:`2003`). + """ if TYPE_CHECKING: @@ -352,8 +373,16 @@ class Info(Mapping[str, VT], Generic[VT], metaclass=InfoMeta): #: List of additional built-in names. __additional__: 'list[str]' = [] - #: List of names to be excluded from :obj:`dict` conversion. + #: List of names of keys left out of ``repr``, ``str`` and the dumps of + #: :mod:`pcapkit.dumpkit` -- metadata such as ``__next_type__``, which + #: :meth:`to_dict` may still carry, see :attr:`__included__`. + #: Finalisation adds the names in :attr:`__builtin__`. __excluded__: 'list[str]' = [] + #: List of names of dunder keys that iterating an instance, ``len()`` and + #: :meth:`to_dict` carry -- metadata such as ``__next_type__``, which a + #: rebuild from the :obj:`dict` reads. Every other dunder key is left + #: out; a key that is no dunder is carried regardless. + __included__: 'list[str]' = [] def __init_subclass__(cls, /, *args: 'Any', **kwargs: 'Any') -> 'None': """Refuse to derive from a finalised info class. @@ -446,9 +475,9 @@ class is finalised on its first construction, whatever its # ``info_final`` writes ``BASE`` or ``FINAL`` onto the class it # finalises, and a subclass reading that state through inheritance was # never finalised itself -- it would keep the ``__excluded__`` it copied - # at declaration, without the names ``info_final`` adds, and leak - # ``__map__``, ``__map_reverse__`` and ``__multi__`` from ``to_dict()`` - # (GitHub issue #1490). The inner one: an ancestor hand-marked bare + # at declaration, without the names ``info_final`` adds, which once + # leaked ``__map__``, ``__map_reverse__`` and ``__multi__`` from + # ``to_dict()`` (GitHub issue #1490). The inner one: an ancestor hand-marked bare # ``@final`` passes its marker to every subclass declared before it # landed, and ``getattr`` would blame such a subclass for a mismarking # that is its ancestor's. ``OwnDictRuleTests.test_a_subclass_that_ @@ -553,9 +582,7 @@ def __update__(self, dict_: 'Optional[Mapping[str, VT] | Iterable[tuple[str, VT] def __str__(self) -> 'str': temp = [] # type: list[str] for (key, value) in self.__dict__.items(): - # NOTE: a dunder key, such as ``__next_type__``, is metadata, and - # left out as the dumpers leave it out (#2003). - if key in self.__excluded__ or _is_dunder(key): + if key in self.__excluded__: continue out_key = self.__map_reverse__.get(key, key) @@ -566,7 +593,7 @@ def __str__(self) -> 'str': def __repr__(self) -> 'str': temp = [] # type: list[str] for (key, value) in self.__dict__.items(): - if key in self.__excluded__ or _is_dunder(key): + if key in self.__excluded__: continue out_key = self.__map_reverse__.get(key, key) @@ -581,13 +608,14 @@ def __repr__(self) -> 'str': def __len__(self) -> 'int': # NOTE: count exactly the keys :meth:`__iter__` yields, so the - # bookkeeping attributes in ``__excluded__`` are not counted. - return sum(1 for key in self.__dict__ if key not in self.__excluded__) + # bookkeeping attributes and the dunder keys ``__included__`` does not + # name are not counted, while the keys in ``__excluded__`` are (#2003). + return sum(1 for key in self.__dict__ if not self.__hidden(key)) def __iter__(self) -> 'Iterator[str]': for key in self.__dict__: - if key in self.__excluded__: - continue + if self.__hidden(key): + continue yield self.__map_reverse__.get(key, key) def __getitem__(self, name: 'str') -> 'VT': @@ -611,10 +639,22 @@ def items(self, multi: 'bool' = False) -> 'ItemsView[str, VT] | Iterator[tuple[s def __items_multi(self) -> 'Iterator[tuple[str, VT]]': """Every ``(key, value)`` pair, with the transformed keys, in order.""" for (key, value) in self.__dict__.items(): - if key not in self.__excluded__: + if not self.__hidden(key): yield key, value yield from self.__multi__.get(key, ()) + def __hidden(self, key: 'str') -> 'bool': + """Whether iteration, ``len()`` and :meth:`to_dict` leave ``key`` out. + + That is a bookkeeping attribute in :attr:`__builtin__`, or a dunder key + :attr:`__included__` does not name (:issue:`2003`). + + """ + if key in self.__builtin__: + return True + return (isinstance(key, str) and key.startswith('__') and key.endswith('__') + and key not in self.__included__) + def __setattr__(self, name: 'str', value: 'VT') -> 'NoReturn': raise UnsupportedCall("can't set attribute") @@ -721,10 +761,14 @@ def to_dict(self) -> 'OrderedMultiDict[str, VT]': Returns an :class:`~pcapkit.corekit.multidict.OrderedMultiDict` rather than a :obj:`dict`, so a key held more than once keeps every - value (:issue:`1484`). + value (:issue:`1484`). It leaves out the dunder keys + :attr:`__included__` does not name, carries the keys in + :attr:`__excluded__`, and records those as its own ``__excluded__`` + for the dumpers to leave out (:issue:`2003`). """ - dict_ = _OrderedMultiDict() # type: OrderedMultiDict[str, Any] + dict_ = _OrderedMultiDict() # type: _OrderedMultiDict[str, Any] + dict_.__excluded__ = self.__excluded__ for (key, value) in self.__items_multi(): out_key = self.__map_reverse__.get(key, key) if isinstance(value, Info): @@ -777,6 +821,11 @@ class _OrderedMultiDict(OrderedMultiDict[KT, VT]): __ne__ = _ne + #: Names of the keys the dumpers leave out: the :attr:`Info.__excluded__` + #: of the info exported, kept by copies, deep copies and pickles; none for + #: an export of a multi-mapping (:issue:`2003`). + __excluded__ = () # type: Collection[str] + def __repr__(self) -> 'str': """Write the export as the :class:`~pcapkit.corekit.multidict.OrderedMultiDict` it is, with every pair. @@ -784,6 +833,29 @@ def __repr__(self) -> 'str': """ return f'{OrderedMultiDict.__name__}({list(self.items(multi=True))!r})' + def copy(self) -> 'Self': + """Return a shallow copy, with the same ``__excluded__``.""" + new = super().copy() + new.__excluded__ = self.__excluded__ + return new + + def deepcopy(self, memo: 'Optional[dict]' = None) -> 'Self': + """Return a deep copy, with the same ``__excluded__``.""" + new = cast('Self', super().deepcopy(memo)) + new.__excluded__ = self.__excluded__ + return new + + def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'Any': + return _restore_export, (type(self), list(self.items(multi=True)), list(self.__excluded__)) + + +def _restore_export(cls: 'Type[_OrderedMultiDict]', pairs: 'list[tuple[Any, Any]]', + excluded: 'list[str]') -> '_OrderedMultiDict': + """Unpickle an :class:`_OrderedMultiDict`, with its ``__excluded__``.""" + new = cls(pairs) + new.__excluded__ = excluded + return new + class _MultiInfo(Info[VT], Generic[KT, VT]): """An :class:`Info` that holds its values as a multi-mapping. @@ -1055,17 +1127,6 @@ def to_dict(self, flat: 'Optional[bool]' = None) -> 'Any': return OrderedMultiDict.to_dict(self, flat) # type: ignore[call-overload] -def _is_dunder(key: 'Any') -> 'bool': - """Whether ``key`` is a dunder key, such as ``__next_type__``. - - Such a key is metadata rather than a field: :meth:`Info.to_dict` carries - it, while ``repr``, ``str`` and the dumpers of :mod:`pcapkit.dumpkit` leave - it out (:issue:`2003`). - - """ - return isinstance(key, str) and len(key) > 4 and key.startswith('__') and key.endswith('__') - - def _freeze(value: 'Any') -> 'Any': """Return ``value`` as stored in a finalised :class:`Info`. diff --git a/pcapkit/dumpkit/common.py b/pcapkit/dumpkit/common.py index 78d5d47399..adbf92a705 100644 --- a/pcapkit/dumpkit/common.py +++ b/pcapkit/dumpkit/common.py @@ -29,7 +29,7 @@ import dictdumper.plist import dictdumper.tree -from pcapkit.corekit.infoclass import Info, _is_dunder +from pcapkit.corekit.infoclass import Info from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict from pcapkit.corekit.sentinels import NoValueType, NullType from pcapkit.protocols.schema.schema import Schema @@ -41,7 +41,7 @@ if TYPE_CHECKING: - from typing import Any, DefaultDict, Iterator, Optional, TextIO, Type + from typing import Any, Collection, DefaultDict, Iterator, Optional, TextIO, Type from dictdumper.dumper import Dumper as ABCDumper from typing_extensions import Literal @@ -409,6 +409,26 @@ def escape_key(key: 'Any') -> 'Any': return json.dumps(format(key, ''))[1:-1] return key + def excluded(info: 'Info | MultiDict[Any, Any]') -> 'Collection[Any]': + """Return the keys of ``info`` the writer leaves out. + + Args: + info: An :class:`~pcapkit.corekit.infoclass.Info`, or a multi-mapping + such as the export its :meth:`~pcapkit.corekit.infoclass.Info.to_dict` + returns. + + Returns: + The :attr:`~pcapkit.corekit.infoclass.Info.__excluded__` of a field + model, or of the field model an export was made from -- metadata + such as ``__next_type__``, which only a rebuild reads + (:issue:`2003`). None for an option list, whose keys are its + options. + + """ + if isinstance(info, Info) and isinstance(info, MultiDict): + return () + return getattr(info, '__excluded__', ()) + def fields(info: 'Info | MultiDict[Any, Any]') -> 'dict[Any, Any]': """Return the fields of ``info`` as the :class:`dict` the writer is handed. @@ -423,13 +443,12 @@ def fields(info: 'Info | MultiDict[Any, Any]') -> 'dict[Any, Any]': held more than once, such as a repeated IPv6 extension header, maps to the :class:`list` of all its values, in order (GitHub issue :issue:`1484`). The values are left as they are, so each comes back - through :meth:`DictDumper.object_hook` when written. A dunder key, - such as ``__next_type__``, is metadata for :meth:`from_data - ` and is left out - (:issue:`2003`). + through :meth:`DictDumper.object_hook` when written. The keys + :func:`excluded` names are left out. """ - pairs = [(key, val) for key, val in info.items(multi=True) if not _is_dunder(key)] + skip = excluded(info) + pairs = [(key, val) for key, val in info.items(multi=True) if key not in skip] counts = collections.Counter(key for key, _ in pairs) result = {} # type: dict[Any, Any] for key, val in pairs: @@ -543,20 +562,19 @@ def object_hook(self, o: 'Any') -> 'Any': # by name could only gather the repeats of a key under its first # occurrence (GitHub issue #1263). The keys are escaped when the # writer hands each mapping back to this hook, through the - # :class:`dict` branch below. A dunder key, which an export of - # an :class:`Info` holds, is left out, as :func:`fields` leaves - # it out (#2003). + # :class:`dict` branch below. The keys an export of an + # :class:`Info` excludes are left out, as :func:`fields` leaves + # them out (#2003). + skip = excluded(o) result = [ {render_enum(key) if isinstance(key, (enum.Enum, aenum.Enum)) else key: val} - for key, val in o.items(multi=True) if not _is_dunder(key) + for key, val in o.items(multi=True) if key not in skip ] elif isinstance(o, MultiDict): # NOTE: a :class:`MultiDict` keeps no order across keys, only # among the values of each, so grouping by key loses nothing. temp = collections.defaultdict(list) # type: DefaultDict[str, list[Any]] for key, val in o.items(multi=True): - if _is_dunder(key): - continue if isinstance(key, (enum.Enum, aenum.Enum)): key = render_enum(key) temp[escape_key(key)].append(val) diff --git a/pcapkit/protocols/data/application/httpv2.py b/pcapkit/protocols/data/application/httpv2.py index ec1e8ff360..088fc102fb 100644 --- a/pcapkit/protocols/data/application/httpv2.py +++ b/pcapkit/protocols/data/application/httpv2.py @@ -31,6 +31,10 @@ class Flags(Data): """Data model for HTTP/2 flags.""" + # NOTE: ``__value__`` is a dunder key, which ``to_dict`` carries only so + # named (#2003); the rebuild reads it, and the dumps write it, as before. + __included__ = ['__value__'] + if TYPE_CHECKING: #: Flags as in combination value, bits the frame type leaves #: undefined included. diff --git a/pcapkit/protocols/data/data.py b/pcapkit/protocols/data/data.py index 92f83fe2a5..42a72fbf82 100644 --- a/pcapkit/protocols/data/data.py +++ b/pcapkit/protocols/data/data.py @@ -16,19 +16,25 @@ class Data(Info): """Base class for data models. - The ``__next_name__`` and ``__next_type__`` keys are metadata: they name - the payload, and :meth:`Protocol.from_data - ` dispatches the payload - on them and on nothing else. :meth:`to_dict` carries them, so that + The ``__next_name__``, ``__next_type__`` and ``__short_read__`` keys are + metadata, listed in :attr:`~pcapkit.corekit.infoclass.Info.__included__` + and :attr:`~pcapkit.corekit.infoclass.Info.__excluded__`: + :meth:`Protocol.from_data ` + dispatches the payload on the first two and on nothing else, and cuts a + short read back on the third. :meth:`to_dict` carries them, so that ``from_data(info.to_dict())`` rebuilds as ``from_data(info)`` does, while - the dumpers and ``repr`` leave out every such dunder key. A hand-written - :obj:`dict` must carry them as well -- and, for IPv6, the extension - headers under ``extension_headers`` -- for the payload to be rebuilt: - without them, no payload layer is dispatched, as for a protocol with no - next layer (:issue:`2003`). + the dumpers and ``repr`` leave them out. A hand-written :obj:`dict` must + carry them as well -- and, for IPv6, the extension header chain under + ``__exthdr__`` -- to be rebuilt the same way: without the first two, no + payload layer is dispatched, as for a protocol with no next layer, and + without ``__exthdr__`` no IPv6 extension header is rebuilt + (:issue:`2003`). """ + __included__ = ['__next_name__', '__next_type__', '__short_read__'] + __excluded__ = ['__next_name__', '__next_type__', '__short_read__'] + if TYPE_CHECKING: #: Next field name, i.e., the name of the payload field. __next_name__: 'str' diff --git a/pcapkit/protocols/data/internet/ipv6.py b/pcapkit/protocols/data/internet/ipv6.py index b4f435b7cf..105d5018a9 100644 --- a/pcapkit/protocols/data/internet/ipv6.py +++ b/pcapkit/protocols/data/internet/ipv6.py @@ -8,14 +8,16 @@ if TYPE_CHECKING: from ipaddress import IPv6Address - from typing import Any + from typing import Any, Type from typing_extensions import Literal + from pcapkit.const.ipv6.extension_header import ExtensionHeader from pcapkit.const.reg.transtype import TransType from pcapkit.corekit.infoclass import OrderedMultiInfo from pcapkit.protocols.data.data import Data from pcapkit.protocols.data.protocol import Packet + from pcapkit.protocols.protocol import ProtocolBase __all__ = [ 'IPv6', @@ -33,6 +35,11 @@ class IPv6(Protocol): """ + # NOTE: ``__exthdr__`` is metadata, as the keys ``Data`` names are: + # ``to_dict`` carries it, the dumps and ``repr`` leave it out (#2003). + __included__ = ['__exthdr__'] + __excluded__ = ['__exthdr__'] + #: Version. version: 'Literal[6]' #: Traffic class. @@ -62,10 +69,12 @@ class IPv6(Protocol): #: Octets captured past the Payload Length -- or, for a jumbogram, #: past the Jumbo Payload Length -- present only when there are any. trailer: 'bytes' - #: Extension headers, in wire order, each under the key its parser - #: writes, such as ``hopopt`` or ``opts``; a repeated header keeps - #: every one of its infos (:issue:`1453`, :issue:`2003`). - extension_headers: 'OrderedMultiInfo[str, Data]' + #: Extension header chain, in wire order, keyed by extension header + #: code, each value the pair of the parser class and its info. The + #: per-header keys hold the same infos, but in the order of the keys + #: of this :class:`~pcapkit.corekit.infoclass.Info`, which differs on + #: PyPy, so the chain is rebuilt from this alone (:issue:`2003`). + __exthdr__: 'OrderedMultiInfo[ExtensionHeader, tuple[Type[ProtocolBase], Data]]' def __new__(cls, *args: 'Any', **kwargs: 'Any') -> 'IPv6': self = super().__new__(cls, *args, **kwargs) diff --git a/pcapkit/protocols/data/misc/pcapng.py b/pcapkit/protocols/data/misc/pcapng.py index b5787fc601..0e23025f74 100644 --- a/pcapkit/protocols/data/misc/pcapng.py +++ b/pcapkit/protocols/data/misc/pcapng.py @@ -65,6 +65,11 @@ class PCAPNG(Protocol): """Data model for PCAP-NG file blocks.""" + # NOTE: ``__truncated_raw__`` is a dunder key, which ``to_dict`` carries + # only so named (#2003); the rebuild reads it, and the dumps write it, as + # before. + __included__ = ['__truncated_raw__'] + #: Block type. type: 'Enum_BlockType' #: Block total length. diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index 51be402d10..1c8d8e7620 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -410,14 +410,25 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over Key-value pairs for protocol construction. """ - exthdr = cls._lookup_exthdr(data) + # NOTE: The chain is rebuilt from ``__exthdr__`` alone, which + # :meth:`Info.to_dict ` + # carries; without it, no extension header is rebuilt, and none is + # guessed from the keys of ``data``, whose order is not part of the + # contract of an :class:`~pcapkit.corekit.infoclass.Info` and differs + # on PyPy (#2003). + exthdr = data.get('__exthdr__') # type: Optional[Mapping[Enum_ExtensionHeader, tuple[Type[ProtocolBase], Any]]] # pylint: disable=line-too-long + chain = [] # type: list[tuple[Type[ProtocolBase], Any]] + if isinstance(exthdr, MultiDict): + chain.extend(value for _, value in exthdr.items(multi=True)) + elif isinstance(exthdr, collections.abc.Mapping): + chain.extend(exthdr.values()) upper = cls._make_payload(data) payload = upper # type: bytes | ProtocolBase # NOTE: The extension headers sit between this header and the upper # layer payload, so they are rebuilt in wire order ahead of it. - if exthdr: - payload = b''.join(proto.from_data(info).data for proto, info in exthdr) + upper.data + if chain: + payload = b''.join(proto.from_data(info).data for proto, info in chain) + upper.data return { 'traffic_class': data['class'], @@ -438,7 +449,7 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over @staticmethod def _exthdr_key(alias: 'str') -> 'str': - """Key an extension header's info is written under in :attr:`Data_IPv6.extension_headers`. + """Key an extension header's info is written under in :class:`Data_IPv6`. Args: alias: :attr:`~pcapkit.protocols.protocol.ProtocolBase.alias` of @@ -451,70 +462,6 @@ def _exthdr_key(alias: 'str') -> 'str': """ return alias.lstrip('IPv6-').lower() - @classmethod - def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[Type[ProtocolBase], Any], ...]': - """Recover the extension header chain of ``data``, with the parser of each header. - - :meth:`_decode_next_layer` holds the extension headers in - :attr:`data.extension_headers `, in wire - order, each under the key its parser writes. The code of each is the - ``next`` field of the one before it, or :attr:`data.next - ` for the first, exactly as the walk that parsed them - dispatched on it. - - The order is that of ``extension_headers``, which keeps it across keys - and so across a repeated header (:issue:`1453`), and never that of the - keys of ``data``: those are an :class:`~pcapkit.corekit.infoclass.Info`'s, - whose order is not part of its contract and on PyPy follows the - attribute layout of an earlier instance of the same class - (:issue:`2003`). - - Args: - data: protocol data - - Returns: - The parser class and info of each extension header, in wire order. - - """ - from pcapkit.protocols.internet.ipv6_ext import \ - IPv6_Ext # isort: skip # pylint: disable=import-outside-toplevel - - headers = data.get('extension_headers') - if not isinstance(headers, collections.abc.Mapping): - return () - pairs = headers.items(multi=True) if isinstance(headers, MultiDict) else headers.items() - - chain = [] # type: list[tuple[Type[ProtocolBase], Any]] - proto = data.get('next') - for (key, info) in pairs: - try: - ex_proto = Enum_ExtensionHeader(proto) - except ValueError: - break - - # NOTE: As in :meth:`_decode_next_layer`, a code with no parser - # carrying ``next`` ends the chain, and so does a Fragment header - # other than a first fragment's. - if 'next' not in info: - break - - # NOTE: A header whose dedicated parser raised was parsed by - # :class:`~pcapkit.protocols.internet.ipv6_ext.IPv6_Ext` instead, - # see :meth:`_import_next_layer`, and written under that class's - # key rather than the dedicated one's. Rebuilding it with the - # dedicated class would look for fields only that class parses - # (#1473), so the key decides the parser. - if key == cls._exthdr_key(IPv6_Ext.alias.fget(None)): # type: ignore[attr-defined] - parser = IPv6_Ext # type: Type[ProtocolBase] - else: - parser = cls._lookup_next_layer(cls.__proto__, int(ex_proto)) - chain.append((parser, info)) - - proto = info['next'] - if ex_proto == Enum_ExtensionHeader.IPv6_Frag and not cls._is_first_fragment(info): - break - return tuple(chain) - @staticmethod def _is_first_fragment(info: 'Mapping[str, Any]') -> 'bool': """Whether a Fragment header's info is that of a first fragment. @@ -581,7 +528,8 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, hdr_len = self.length # header length raw_len = ipv6.payload if length is None else length # payload length _protos = [] # type: list[ProtocolBase] # ProtoChain buffer - _headers = OrderedMultiDict() # type: OrderedMultiDict[str, Any] # info per extension header + # (parser class, info) per extension header + _exthdr = OrderedMultiDict() # type: OrderedMultiDict[Enum_ExtensionHeader, tuple[Type[ProtocolBase], Any]] split = None # type: Optional[tuple[int, int]] # at first Fragment header opaque = False # walk ended at a non-zero Fragment Offset depth = _frame_depth.get() # layers of the frame up to this one @@ -642,23 +590,24 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, # # Such a layer is not recorded as an extension header: it consumed # nothing (``Raw.length`` is 0), so recording it would emit its - # octets twice on rebuild, once from ``extension_headers`` and once - # more as the upper layer. Stopping here with ``proto`` unchanged instead + # octets twice on rebuild, once from ``__exthdr__`` and once more as + # the upper layer. Stopping here with ``proto`` unchanged instead # hands the remaining octets to ``super()._decode_next_layer`` # below, which decodes them once, as the upper layer. if not hasattr(info, 'next'): break - # NOTE: Held in an :class:`OrderedMultiDict` of their own, so a - # repeated header keeps its predecessor's info rather than - # overwriting it (#1453), and the wire order is that of the - # headers, not of the keys of ``ipv6``, which differs on PyPy - # (#2003). - _headers.add(self._exthdr_key(next_.alias), info) + # NOTE: Added through a :class:`MultiDict`, so a repeated header + # keeps its predecessor's info rather than overwriting it, and + # :meth:`Info.to_dict ` + # writes every one of them (#1453). + name = self._exthdr_key(next_.alias) + ipv6.__update__(OrderedMultiDict([(name, info)])) # record protocol name # self._protos = ProtoChain(name, chain, alias) _protos.append(next_) + _exthdr.add(ex_proto, (type(next_), info)) # update header & payload length hdr_len += next_.length # type: ignore[assignment] @@ -711,8 +660,10 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, # update next header 'protocol': proto, - # extension headers in wire order - 'extension_headers': _headers, + # extension header chain in wire order, with the parser of each, + # for ``from_data``; the parser keeps the IPv6_Ext a header fell + # back to (#1473) + '__exthdr__': _exthdr, }) # NOTE: Nor does a later fragment's slice start with the upper-layer diff --git a/pcapkit/protocols/protocol.py b/pcapkit/protocols/protocol.py index fcd3f223ef..82f2bcfb21 100644 --- a/pcapkit/protocols/protocol.py +++ b/pcapkit/protocols/protocol.py @@ -1131,11 +1131,13 @@ def from_data(cls, data: '_PT | dict[str, Any]', **kwargs: 'Any') -> 'Self': ``data`` may be the :obj:`dict` that :meth:`Info.to_dict ` returns, which carries the - ``__next_name__`` and ``__next_type__`` metadata the payload is - dispatched on. A hand-written :obj:`dict` must carry them too -- and, - for :class:`~pcapkit.protocols.internet.ipv6.IPv6`, the extension - headers under ``extension_headers`` -- or no payload is rebuilt, see - :meth:`_make_payload`. + metadata the rebuild reads: ``__next_name__`` and ``__next_type__``, + which the payload is dispatched on, ``__short_read__``, and, for + :class:`~pcapkit.protocols.internet.ipv6.IPv6`, the extension header + chain under ``__exthdr__``. A hand-written :obj:`dict` must carry them + too, or no payload (nor IPv6 extension header) is rebuilt and no short + read is cut back, see :meth:`_make_payload`; nothing is guessed from + its other keys (:issue:`2003`). Args: data: Protocol data. @@ -2121,9 +2123,9 @@ def _make_payload(cls, data: 'Data') -> 'ProtocolBase': :meth:`Info.to_dict ` carries both keys, so ``data`` rebuilt from such a :obj:`dict` dispatches as - the info did. A hand-written :obj:`dict` must carry them too: the - payload is never guessed from the other keys of ``data`` - (:issue:`2003`). + the info did. A hand-written :obj:`dict` must carry them too, as it + must ``__short_read__`` and, for IPv6, ``__exthdr__``: the payload is + never guessed from the other keys of ``data`` (:issue:`2003`). The payload is built one layer further into the frame than this protocol, as :meth:`_parse_next_layer` counts it, since each layer built diff --git a/tests/corekit/test_info_roundtrip_unit.py b/tests/corekit/test_info_roundtrip_unit.py index 374e389f79..36ce428db8 100644 --- a/tests/corekit/test_info_roundtrip_unit.py +++ b/tests/corekit/test_info_roundtrip_unit.py @@ -329,6 +329,9 @@ class Leaf(Info): class Node(Info): """Nested, optional, renamed and plain keys together.""" + # NOTE: ``to_dict`` carries a dunder key only if named here (#2003). + __included__ = ['__short_read__', '__truncated_raw__'] + leaf: Leaf maybe: Optional[Leaf] items: int diff --git a/tests/corekit/test_infoclass.py b/tests/corekit/test_infoclass.py index 2a43e8ee7a..60f7e11cc8 100644 --- a/tests/corekit/test_infoclass.py +++ b/tests/corekit/test_infoclass.py @@ -62,20 +62,48 @@ class Bag(Info): bag = Bag({'items': 'conflict', 'visible': 1, 'hidden': 2}, extra=3) self.assertEqual(bag['items'], 'conflict') - self.assertEqual(list(bag), ['items', 'visible', 'extra']) + # NOTE: GitHub issue #2003 -- ``__excluded__`` keys are left out of + # ``repr`` and ``str`` only; iteration and ``to_dict`` carry them. + self.assertEqual(list(bag), ['items', 'visible', 'hidden', 'extra']) # NOTE: GitHub issue #1296 -- ``len`` counts exactly the keys - # iteration yields, so the ``__excluded__`` ``hidden`` key and the - # internal bookkeeping attributes are not counted. - self.assertEqual(len(bag), 3) + # iteration yields, so the internal bookkeeping attributes are not + # counted. + self.assertEqual(len(bag), 4) self.assertEqual(str(bag), 'Bag(items=conflict, visible=1, extra=3)') self.assertIn('items=', repr(bag)) - self.assertEqual(list(bag.to_dict().items(multi=True)), [('items', 'conflict'), ('visible', 1), ('extra', 3)]) + self.assertNotIn('hidden', repr(bag)) + self.assertEqual(list(bag.to_dict().items(multi=True)), + [('items', 'conflict'), ('visible', 1), ('hidden', 2), ('extra', 3)]) with self.assertRaises(UnsupportedCall): bag.visible = 9 with self.assertRaises(UnsupportedCall): del bag.visible + def test_to_dict_carries_only_the_included_dunder_keys(self) -> None: + """GitHub issue #2003: a dunder key is metadata, carried only if ``__included__`` names it.""" + from pcapkit.corekit.infoclass import Info + + class Plain(Info): + pass + + class Meta(Info): + __included__ = ['__meta__'] + + class More(Meta): + __included__ = ['__more__'] + + self.assertEqual(More.__included__, ['__more__', '__meta__']) + pairs = {'a': 1, '__meta__': 2, '__more__': 3} + for cls, kept in ((Plain, ['a']), (Meta, ['a', '__meta__']), (More, ['a', '__meta__', '__more__'])): + with self.subTest(cls=cls.__name__): + info = cls.from_dict(pairs) + self.assertEqual([key for key, _ in info.to_dict().items(multi=True)], kept) + self.assertEqual(list(info), kept) + self.assertEqual(len(info), len(kept)) + # Still held, and read by indexing. + self.assertEqual(info['__meta__'], 2) + def test_from_dict_iterables_nested_info_and_inherited_metadata(self) -> None: from pcapkit.corekit.infoclass import Info @@ -100,8 +128,10 @@ class Child(Base): converted = outer.to_dict() self.assertEqual(list(converted['inner'].items(multi=True)), [('answer', 42)]) self.assertEqual(converted['kept'], 'yes') - self.assertNotIn('base_hidden', converted) - self.assertNotIn('child_hidden', converted) + # Carried by ``to_dict``, left out of ``repr`` (#2003). + self.assertEqual(converted['base_hidden'], 'skip') + self.assertEqual(converted['child_hidden'], 'skip') + self.assertNotIn('hidden', repr(outer)) self.assertEqual(Info.from_dict({'a': 1}, b=2).to_dict().to_dict(), {'a': 1, 'b': 2}) self.assertEqual(Info.from_dict(c=3).to_dict().to_dict(), {'c': 3}) diff --git a/tests/dumpkit/test_capture_dump_runtime.py b/tests/dumpkit/test_capture_dump_runtime.py index 250809fc78..17b4c52887 100644 --- a/tests/dumpkit/test_capture_dump_runtime.py +++ b/tests/dumpkit/test_capture_dump_runtime.py @@ -170,10 +170,9 @@ def test_json_frame_sections_carry_every_info_key_and_the_chain(self) -> None: for number, frame in enumerate(extractor.frame, start=1): section = report[f'Frame {number}'] with self.subTest(capture=name, frame=number): - # Every key but the dunder metadata, which the dumps - # leave out (#2003). + # Every key but those the dumps leave out (#2003). self.assertEqual(list(section), [key for key in frame.info.to_dict() - if not (key.startswith('__') and key.endswith('__'))]) + if key not in frame.info.__excluded__]) self.assertEqual(section['protocols'], str(frame.protochain)) def test_plist_dump_reads_back_through_plistlib(self) -> None: diff --git a/tests/dumpkit/test_repeated_field_dump_1484_unit.py b/tests/dumpkit/test_repeated_field_dump_1484_unit.py index 453b36f45a..f3abbf2fdc 100644 --- a/tests/dumpkit/test_repeated_field_dump_1484_unit.py +++ b/tests/dumpkit/test_repeated_field_dump_1484_unit.py @@ -3,10 +3,8 @@ GitHub issue #1484. An IPv6 packet carrying two Destination Options headers holds both under the one ``opts`` key, and every dump held only the first. -Each dump now writes both headers, in wire order, while a field held once is -written as its value. The headers are held in ``extension_headers`` -(#2003), an option list, which a dump writes as one single-key mapping per -entry. +Each dump now writes the field as the list of both headers, in wire order, +while a field held once is written as its value. Every case builds its input in a temporary directory, and reads no capture. :mod:`pcapkit` is imported inside each test, after @@ -48,11 +46,6 @@ def capture() -> bytes: return GLOBAL + struct.pack(' 'list[dict]': - """The Destination Options headers of a dumped IPv6 header, in wire order.""" - return [entry['opts'] for entry in ipv6['extension_headers']] - - def pads(headers: 'list[dict]') -> 'list[str]': """The PadN octets of each Destination Options header, as the JSON dump spells them.""" return [next(iter(header['options'][0].values()))['pad']['hex'] for header in headers] @@ -84,14 +77,15 @@ def dump(self, fmt: 'str') -> 'bytes': def test_json_holds_both_headers_in_order(self) -> None: ipv6 = json.loads(self.dump('json'))['Frame 1']['ethernet']['ipv6'] - self.assertEqual([list(entry) for entry in ipv6['extension_headers']], [['opts'], ['opts']]) - self.assertEqual(pads(opts(ipv6)), [f'000000{mark:02x}' for mark in MARKS]) + self.assertIsInstance(ipv6['opts'], list) + self.assertEqual(pads(ipv6['opts']), [f'000000{mark:02x}' for mark in MARKS]) # A field held once is still written as its value. self.assertEqual(ipv6['limit'], 64) def test_plist_holds_both_headers_in_order(self) -> None: ipv6 = plistlib.loads(self.dump('plist'))['Frame 1']['ethernet']['ipv6'] - self.assertEqual([next(iter(header['options'][0].values()))['pad'] for header in opts(ipv6)], + self.assertIsInstance(ipv6['opts'], list) + self.assertEqual([next(iter(header['options'][0].values()))['pad'] for header in ipv6['opts']], [b'\x00\x00\x00' + bytes([mark]) for mark in MARKS]) def test_tree_holds_both_headers(self) -> None: @@ -124,11 +118,10 @@ def test_the_writer_takes_the_info_or_its_export(self) -> None: make_dumper(dictdumper.JSON)(path)(value, name='Frame 1') with open(path, encoding='utf-8') as file: tops.append(json.load(file)['Frame 1']) - self.assertEqual(pads(opts(tops[0]['ethernet']['ipv6'])), [f'000000{mark:02x}' for mark in MARKS]) + self.assertEqual(pads(tops[0]['ethernet']['ipv6']['opts']), [f'000000{mark:02x}' for mark in MARKS]) self.assertEqual(list(tops[1]), list(tops[0])) - # Every key but the dunder metadata, which the dumps leave out (#2003). - self.assertEqual(list(tops[0]), [key for key in frame.info.to_dict() - if not (key.startswith('__') and key.endswith('__'))]) + # Every key but those the dumps leave out (#2003). + self.assertEqual(list(tops[0]), [key for key in frame.info.to_dict() if key not in frame.info.__excluded__]) if __name__ == '__main__': diff --git a/tests/protocols/internet/test_esp_unit.py b/tests/protocols/internet/test_esp_unit.py index 59bac8c1c7..08fc53e417 100644 --- a/tests/protocols/internet/test_esp_unit.py +++ b/tests/protocols/internet/test_esp_unit.py @@ -1029,7 +1029,7 @@ def test_ipv6_extension_header_position(self) -> None: packet = header + body ipv6 = IPv6(packet, len(packet), __context__=registry) - esp = ipv6.info.extension_headers['esp'] + esp = ipv6.info.esp self.assertIs(esp.status, ESPStatus.DECRYPTED) self.assertEqual(esp.next, TransType.TCP) self.assertEqual(esp.plaintext, INNER_TCP[20:]) diff --git a/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py b/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py index 40a6947b01..18a4d5cfe3 100644 --- a/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py +++ b/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py @@ -67,7 +67,7 @@ def test_private_codes_are_not_extension_headers(self) -> None: with self.subTest(code=code): frame = ipv6(code, bytes.fromhex('3b00010203040506')) parsed = IPv6(frame) - self.assertEqual(len(parsed.info.extension_headers), 0) + self.assertEqual(len(parsed.info.__exthdr__), 0) self.assertEqual(len(parsed.protochain), 2) self.assertEqual(IPv6.from_data(parsed.info).data, frame) diff --git a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py index 17cfbad749..2d3b045d82 100644 --- a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py +++ b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py @@ -112,7 +112,7 @@ def test_dict_rebuild_keeps_the_generic_parser(self) -> None: with self.subTest(cut=cut): ip6 = IPv6(data, len(data)) self.assertIs(type(ip6._exthdr[ExtensionHeader.HOPOPT]), IPv6_Ext) - chain = IPv6._lookup_exthdr(ip6.info.to_dict()) + chain = [value for _, value in ip6.info.to_dict()['__exthdr__'].items(multi=True)] self.assertEqual([parser for parser, _ in chain], [IPv6_Ext]) self.assertEqual(IPv6.from_data(ip6.info.to_dict()).data, data) diff --git a/tests/protocols/internet/test_ipv6_from_data_ext_unit.py b/tests/protocols/internet/test_ipv6_from_data_ext_unit.py index a5abd60665..9aa6269a54 100644 --- a/tests/protocols/internet/test_ipv6_from_data_ext_unit.py +++ b/tests/protocols/internet/test_ipv6_from_data_ext_unit.py @@ -82,22 +82,19 @@ def test_repeated_header_keeps_both_instances(self) -> None: ) self.assertEqual(IPv6.from_data(parsed.info).data, packet) - def test_chain_is_held_in_extension_headers(self) -> None: - from pcapkit.corekit.infoclass import OrderedMultiInfo - from pcapkit.corekit.multidict import OrderedMultiDict + def test_chain_is_carried_by_to_dict_and_hidden_from_repr(self) -> None: + from pcapkit.const.ipv6.extension_header import ExtensionHeader from pcapkit.protocols.internet.ipv6 import IPv6 - - packet = PACKETS['chain-with-repeated-header'] - parsed = IPv6(packet, len(packet)) - headers = parsed.info.extension_headers - self.assertIsInstance(headers, OrderedMultiInfo) - self.assertEqual([info for _, info in headers.items(multi=True)], - [proto.info for _, proto in parsed.extension_headers.items(multi=True)]) - # ``to_dict`` exports it as the option list it is, in wire order. - exported = parsed.info.to_dict()['extension_headers'] - self.assertIsInstance(exported, OrderedMultiDict) - self.assertEqual([key for key, _ in exported.items(multi=True)], ['hopopt', 'opts', 'route', 'opts']) - self.assertEqual(IPv6.from_data(parsed.info.to_dict()).data, packet) + from pcapkit.protocols.internet.ipv6_route import IPv6_Route + + packet = PACKETS['routing'] + info = IPv6(packet, len(packet)).info + self.assertNotIn('__exthdr__', repr(info)) + self.assertNotIn('__exthdr__', str(info)) + # Keyed by code, each the parser and the info, as ``to_dict`` carries it. + exported = info.to_dict()['__exthdr__'] + self.assertEqual([(code, parser) for code, (parser, _) in exported.items(multi=True)], + [(ExtensionHeader.IPv6_Route, IPv6_Route)]) if __name__ == '__main__': diff --git a/tests/protocols/internet/test_ipv6_unit.py b/tests/protocols/internet/test_ipv6_unit.py index 4b10e9f1bf..eaa96fca89 100644 --- a/tests/protocols/internet/test_ipv6_unit.py +++ b/tests/protocols/internet/test_ipv6_unit.py @@ -178,7 +178,7 @@ def test_ipv6_decode_next_layer_handles_extension_headers_and_fragments(self) -> self.assertTrue(import_next.call_args.kwargs['extension']) self.assertEqual(import_next.call_args.kwargs['payload'], b'fragment-payload') proto._read_packet.assert_called_once_with(header=48, payload=len(b'fragment-payload') - 8) - self.assertEqual(ipv6.extension_headers['frag'].next, TransType.UDP) + self.assertEqual(ipv6.frag.next, TransType.UDP) self.assertEqual(ipv6.fragment, 'fragment-data') self.assertEqual(ipv6.hdr_len, 48) self.assertEqual(ipv6.raw_len, len(b'fragment-payload') - 8) diff --git a/tests/protocols/internet/test_mpl_seed_rebuild_unit.py b/tests/protocols/internet/test_mpl_seed_rebuild_unit.py index fb5a424bfc..e9852942c0 100644 --- a/tests/protocols/internet/test_mpl_seed_rebuild_unit.py +++ b/tests/protocols/internet/test_mpl_seed_rebuild_unit.py @@ -63,7 +63,7 @@ def test_source_address_seed_rebuilds_inside_ipv6(self) -> None: octets = HEADERS[0] packet = (struct.pack('!IHBB', 6 << 28, len(octets), nh, 64) + SRC.packed + DST.packed + octets) - info = IPv6(packet, len(packet)).info.extension_headers[key] + info = IPv6(packet, len(packet)).info[key] opt = list(info.options.values())[0] self.assertEqual(opt.seed_id, SRC) self.assertEqual(cls.from_data(info).data, octets) diff --git a/tests/protocols/test_declared_length_overrun_unit.py b/tests/protocols/test_declared_length_overrun_unit.py index 624e687616..a2503f1c05 100644 --- a/tests/protocols/test_declared_length_overrun_unit.py +++ b/tests/protocols/test_declared_length_overrun_unit.py @@ -153,7 +153,7 @@ def test_an_overrunning_layer_is_kept_as_captured_by_its_parent(self) -> None: if name == 'IPv6': # an extension header is kept as raw extension data - self.assertIn('has an option that runs past the end of the data', str(parsed.info.extension_headers['ext'].error)) + self.assertIn('has an option that runs past the end of the data', str(parsed.info.ext.error)) else: # any other layer is kept as a raw payload, as captured self.assertIsInstance(parsed.payload, Raw) diff --git a/tests/protocols/test_lookup_by_name_2003_unit.py b/tests/protocols/test_lookup_by_name_2003_unit.py index 8d944b80f0..6c45842f63 100644 --- a/tests/protocols/test_lookup_by_name_2003_unit.py +++ b/tests/protocols/test_lookup_by_name_2003_unit.py @@ -12,7 +12,7 @@ The order of an :class:`~pcapkit.corekit.infoclass.Info` is not part of its contract, so nothing is read off it any more: ``to_dict()`` carries the ``__next_type__`` and ``__next_name__`` metadata the payload is dispatched on, -and the chain is held in ``extension_headers``, an +and ``__exthdr__``, the extension header chain keyed by code in an :class:`~pcapkit.corekit.infoclass.OrderedMultiInfo` keeping the order of its own. A dict without the metadata is not guessed at. These tests reorder an instance's :attr:`~object.__dict__` as PyPy does, so CPython checks the same @@ -87,7 +87,8 @@ def setUp(self) -> None: self.addCleanup(quiet.__exit__, None, None, None) warnings.simplefilter('ignore') - def test_extension_headers_keep_their_order_whatever_the_key_order(self) -> None: + def test_extension_headers_follow_exthdr_whatever_the_key_order(self) -> None: + from pcapkit.const.ipv6.extension_header import ExtensionHeader from pcapkit.protocols.internet.hopopt import HOPOPT from pcapkit.protocols.internet.ipv6 import IPv6 from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts @@ -95,14 +96,15 @@ def test_extension_headers_keep_their_order_whatever_the_key_order(self) -> None ip6 = IPv6(HOP_DST_DST) self.assertEqual(len(ip6.data), len(HOP_DST_DST)) - # The headers first, then last, of the keys. - for before in ('version', None): - with self.subTest(before=before): - exported = moved(ip6.info, 'extension_headers', before=before).to_dict() - chain = IPv6._lookup_exthdr(exported) - self.assertEqual([parser for parser, _ in chain], [HOPOPT, IPv6_Opts, IPv6_Opts]) - self.assertEqual([parser.from_data(hdr).data for parser, hdr in chain], - [ext.data for _, ext in ip6.extension_headers.items(multi=True)]) + # As PyPy laid it out: ``hopopt`` after every other key; and the + # chain first. + for key, before in (('hopopt', None), ('__exthdr__', 'version')): + with self.subTest(moved=key): + exported = moved(ip6.info, key, before=before).to_dict() + chain = list(exported['__exthdr__'].items(multi=True)) + self.assertEqual([code for code, _ in chain], + [ExtensionHeader.HOPOPT, ExtensionHeader.IPv6_Opts, ExtensionHeader.IPv6_Opts]) + self.assertEqual([parser for _, (parser, _) in chain], [HOPOPT, IPv6_Opts, IPv6_Opts]) self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) def test_payload_follows_the_metadata_whatever_the_key_order(self) -> None: @@ -137,14 +139,15 @@ def test_cut_upper_layer_is_found_ahead_of_the_headers(self) -> None: self.assertIn('raw', ip6.info) self.assertEqual(len(ip6.data), len(HOP_DST_DST)) - # As on PyPy for #1610: the headers after the upper layer. - exported = moved(ip6.info, 'raw', before='extension_headers').to_dict() - self.assertEqual(len(IPv6._lookup_exthdr(exported)), 2) + # As on PyPy for #1610: ``opts`` after the upper layer. + info = moved(moved(ip6.info, 'raw', before='opts'), 'hopopt', before='raw') + exported = info.to_dict() + self.assertEqual(len(list(exported['__exthdr__'].items(multi=True))), 2) rebuilt = IPv6.from_data(exported) self.assertIsInstance(rebuilt.payload, Raw) self.assertEqual(rebuilt.data, HOP_DST_DST) - def test_dict_without_the_metadata_is_not_guessed_at(self) -> None: + def test_dict_without_the_dispatch_keys_is_not_guessed_at(self) -> None: from pcapkit.protocols.internet.ipv6 import IPv6 from pcapkit.protocols.misc.null import NoPayload @@ -154,11 +157,24 @@ def test_dict_without_the_metadata_is_not_guessed_at(self) -> None: exported.poplist(key) self.assertIn('udp', exported) - # The extension headers are rebuilt from ``extension_headers``, and the + # The extension headers are rebuilt from ``__exthdr__``, and the # upper layer, named by nothing, is not. rebuilt = IPv6.from_data(exported) self.assertIsInstance(rebuilt.payload, NoPayload) self.assertEqual(rebuilt.data, HOP_DST_DST[:40 + 3 * 8]) + def test_dict_without_exthdr_rebuilds_no_extension_header(self) -> None: + from pcapkit.protocols.internet.ipv6 import IPv6 + + ip6 = IPv6(HOP_DST_DST) + exported = ip6.info.to_dict() + exported.poplist('__exthdr__') + self.assertIn('hopopt', exported) + self.assertEqual(len(exported.getlist('opts')), 2) + + # The headers are still there under their keys, but nothing names + # the chain, so only the upper layer follows the IPv6 header. + self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST[:40] + DATAGRAM) + if __name__ == '__main__': unittest.main() diff --git a/tests/protocols/test_option_coverage_runtime.py b/tests/protocols/test_option_coverage_runtime.py index 7e167d51d2..6c511312e3 100644 --- a/tests/protocols/test_option_coverage_runtime.py +++ b/tests/protocols/test_option_coverage_runtime.py @@ -186,7 +186,7 @@ def test_the_jumbo_payload_frame_is_a_genuine_jumbogram(self) -> None: self.assertEqual(str(frame.protochain), 'Ethernet:IPv6:HOPOPT:UDP:Raw') ipv6 = frame[IPv6].info - self.assertEqual(ipv6.extension_headers['hopopt'].options[Option.Jumbo_Payload].jumbo_len, jumbo) + self.assertEqual(ipv6.hopopt.options[Option.Jumbo_Payload].jumbo_len, jumbo) self.assertEqual(ipv6.hdr_len + ipv6.raw_len, 40 + jumbo) def test_option_captures_dump_to_tree_and_json(self) -> None: diff --git a/tests/protocols/test_protocol_base_unit.py b/tests/protocols/test_protocol_base_unit.py index 70e00089ca..5db35cd3c1 100644 --- a/tests/protocols/test_protocol_base_unit.py +++ b/tests/protocols/test_protocol_base_unit.py @@ -653,7 +653,7 @@ def test_outer_address_reaches_a_schema_that_needs_it(self) -> None: + hopopt) parsed = IPv6(io.BytesIO(packet), len(packet)) - option = list(parsed.info.extension_headers['hopopt'].options.values())[0] + option = list(parsed.info.hopopt.options.values())[0] self.assertEqual(str(parsed.protochain), 'IPv6:HOPOPT') self.assertEqual(parsed.info.src, source) diff --git a/tests/protocols/test_short_read_roundtrip_unit.py b/tests/protocols/test_short_read_roundtrip_unit.py index 3ea9ce238a..b5af8cc7c8 100644 --- a/tests/protocols/test_short_read_roundtrip_unit.py +++ b/tests/protocols/test_short_read_roundtrip_unit.py @@ -189,10 +189,36 @@ def test_record_rides_in_to_dict_only_when_cut(self) -> None: again = Ethernet.from_data(cut.info.to_dict()) self.assertEqual(again.info['__short_read__'], ('type', 1)) self.assertEqual(Ethernet.from_data(again.info.to_dict()).data, bytes(13)) - # a dumper writes the record as a list, which replays the same + # a hand-written dict may carry the record as a list, which replays + # the same record = dict(cut.info.to_dict(), __short_read__=['type', 1]) self.assertEqual(Ethernet.from_data(record).data, bytes(13)) + def test_record_is_left_out_of_repr_and_dumps(self) -> None: + import json + import os + import tempfile + + import dictdumper.json + + from pcapkit.dumpkit.common import make_dumper + from pcapkit.protocols.link.ethernet import Ethernet + + # GitHub issue #2003: metadata, which ``__excluded__`` names. + info = Ethernet(bytes(13), 13).info + self.assertIn('__short_read__', info.to_dict()) + self.assertNotIn('__short_read__', repr(info)) + self.assertNotIn('__short_read__', str(info)) + with tempfile.TemporaryDirectory() as tmp: + for index, value in enumerate((info, info.to_dict())): + with self.subTest(dumped=type(value).__name__): + path = os.path.join(tmp, f'{index}.json') + make_dumper(dictdumper.json.JSON)(path)(value, name='Ethernet') + with open(path, encoding='utf-8') as file: + dumped = json.load(file)['Ethernet'] + self.assertNotIn('__short_read__', dumped) + self.assertIn('dst', dumped) + class TestSchemaShortRead(unittest.TestCase): """Pin the schema-level record and the pack that honours it.""" diff --git a/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py b/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py index ce4cc5ee32..afb72a88bd 100644 --- a/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py +++ b/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py @@ -136,6 +136,7 @@ def test_ipv6_extension_headers_rebuild_from_its_dict(self) -> None: ipv6 = self.frame('ipv6-exthdr-udp')[IPv6] dict_ = ipv6.info.to_dict() + self.assertIn('__exthdr__', dict_) rebuilt = IPv6.from_data(dict_) self.assertEqual(rebuilt.data, ipv6.data) @@ -229,7 +230,8 @@ def test_to_dict_carries_the_dispatch_keys_and_a_dump_none(self) -> None: with open(path, encoding='utf-8') as file: text = file.read() json.loads(text) - self.assertNotIn('"__', text) + for key in ('__next_type__', '__next_name__', '__short_read__', '__exthdr__'): + self.assertNotIn(f'"{key}"', text) if __name__ == '__main__': unittest.main() diff --git a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py index b6284bc40f..24fb5b6b8b 100644 --- a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py +++ b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py @@ -6,13 +6,12 @@ overwrote the first and the dict rebuild silently came back shorter -- Destination Options, Destination Options, UDP rebuilt 56 of 64 octets. -The extension headers are held in ``extension_headers``, an -:class:`~pcapkit.corekit.infoclass.OrderedMultiInfo` that ``to_dict`` exports -as an :class:`~pcapkit.corekit.multidict.OrderedMultiDict` (#1484): indexing -sees the first header of each type, and ``items(multi=True)`` yields every -header in wire order for the rebuild -- the order of the headers, not of the -keys of the :class:`~pcapkit.corekit.infoclass.Info`, which differs on PyPy -(#2003). The dumpers write every header too. +``to_dict`` returns an :class:`~pcapkit.corekit.multidict.OrderedMultiDict` +(#1484): indexing sees the first value of each key, and ``getlist`` gives every +header of a type in wire order. The order across types is that of the keys of +the :class:`~pcapkit.corekit.infoclass.Info`, which is not part of its contract +and differs on PyPy, so the rebuild walks the ``next`` fields instead (#2003). +The dumpers write every header too. Every case builds its own octets and reads no capture. Classes are imported inside each test, after :func:`~tests._support.reimport_once_per_class`. @@ -136,7 +135,7 @@ def test_rebuilt_chain_has_the_parsed_headers(self) -> None: self.assertEqual([ext.data for ext in rebuilt.extension_headers.values()], [ext.data for ext in ip6.extension_headers.values()]) - def test_to_dict_keeps_every_header_in_wire_order(self) -> None: + def test_to_dict_keeps_every_header_of_a_type_in_wire_order(self) -> None: from pcapkit.corekit.multidict import OrderedMultiDict from pcapkit.protocols.internet.ipv6 import IPv6 @@ -145,20 +144,18 @@ def test_to_dict_keeps_every_header_in_wire_order(self) -> None: ip6 = self.frame(name)[IPv6] dict_ = ip6.info.to_dict() self.assertIsInstance(dict_, OrderedMultiDict) - headers = dict_['extension_headers'] - self.assertIsInstance(headers, OrderedMultiDict) - # NOTE: As for every OrderedMultiInfo, the export holds the - # headers' infos as they are. - parsed = [(IPv6._exthdr_key(ext.alias), ext.info) - for _, ext in ip6.extension_headers.items(multi=True)] - self.assertEqual(list(headers.items(multi=True)), parsed) + parsed = {} # type: dict[str, list[object]] + for _, ext in ip6.extension_headers.items(multi=True): + parsed.setdefault(IPv6._exthdr_key(ext.alias), []).append(canonical(ext.info.to_dict())) + held = {key: [canonical(val) for val in dict_.getlist(key)] for key in parsed} + self.assertEqual(held, parsed) # Indexing sees the first of each. first = {} # type: dict[str, object] - for key, val in headers.items(multi=True): + for key, val in dict_.items(multi=True): first.setdefault(key, val) - self.assertEqual({key: headers[key] for key in headers}, first) + self.assertEqual({key: dict_[key] for key in dict_}, first) def test_info_from_dict_keeps_the_repeats(self) -> None: from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6 @@ -199,12 +196,12 @@ def test_info_copy_keeps_its_own_repeats(self) -> None: info = self.frame('dst-dst')[IPv6].info before = list(info.items(multi=True)) - extra = info['extension_headers'] + extra = info['opts'] for clone in (copy.copy(info), copy.deepcopy(info)): with self.subTest(clone=type(clone).__name__): self.assertIsNot(clone.__multi__, info.__multi__) self.assertIsNot(clone.__map__, info.__map__) - clone.__update__(OrderedMultiDict([('extension_headers', extra)])) + clone.__update__(OrderedMultiDict([('opts', extra)])) self.assertEqual(len(list(clone.items(multi=True))), len(before) + 1) self.assertEqual(list(info.items(multi=True)), before) @@ -221,9 +218,9 @@ def test_dumper_writes_every_header(self) -> None: make_dumper(dictdumper.json.JSON)(path)(value, name='Frame 1') with open(path, encoding='utf-8') as file: outputs.append(json.load(file)) - # One single-key mapping per header, in wire order. - headers = outputs[0]['Frame 1']['ethernet']['ipv6']['extension_headers'] - self.assertEqual([list(header) for header in headers], [['opts'], ['route'], ['opts']]) + ipv6 = outputs[0]['Frame 1']['ethernet']['ipv6'] + self.assertEqual(len(ipv6['opts']), 2) + self.assertIsInstance(ipv6['route'], dict) # Handed the export, the dumper writes the top level the same way. self.assertEqual(list(outputs[1]['Frame 1']), list(outputs[0]['Frame 1'])) From 2a9539ae94aa16a87e96ba30ec5d35c38d2b583b Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 21 Oct 2026 13:18:19 -0400 Subject: [PATCH 5/6] docs(corekit): document Info.__included__ and Data's metadata lists (#2003) Adds __included__ next to __excluded__ for Info and Data in the API docs, and doc comments stating its contract: dunder keys are dropped from to_dict, iteration and len() unless listed, and a subclass, or a custom Info, lists its own. --- docs/source/pcapkit/corekit/infoclass.rst | 2 ++ docs/source/pcapkit/protocols/index.rst | 3 +++ pcapkit/corekit/infoclass.py | 8 ++++++-- pcapkit/protocols/data/data.py | 2 ++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/source/pcapkit/corekit/infoclass.rst b/docs/source/pcapkit/corekit/infoclass.rst index ffe8563b96..e7e4b1e43a 100644 --- a/docs/source/pcapkit/corekit/infoclass.rst +++ b/docs/source/pcapkit/corekit/infoclass.rst @@ -23,6 +23,8 @@ classes :class:`~pcapkit.corekit.infoclass.MultiInfo` and :no-value: .. autoattribute:: __excluded__ :no-value: + .. autoattribute:: __included__ + :no-value: .. autoattribute:: __map__ .. autoattribute:: __map_reverse__ diff --git a/docs/source/pcapkit/protocols/index.rst b/docs/source/pcapkit/protocols/index.rst index 36918052f3..b076ee2130 100644 --- a/docs/source/pcapkit/protocols/index.rst +++ b/docs/source/pcapkit/protocols/index.rst @@ -268,3 +268,6 @@ Data Model Please refer to :func:`Protocol._decode_next_layer ` for more information with the inserted names to be excluded. + + .. autoattribute:: __included__ + :no-value: diff --git a/pcapkit/corekit/infoclass.py b/pcapkit/corekit/infoclass.py index e790a2953b..d7d80ee2ea 100644 --- a/pcapkit/corekit/infoclass.py +++ b/pcapkit/corekit/infoclass.py @@ -380,8 +380,12 @@ class Info(Mapping[str, VT], Generic[VT], metaclass=InfoMeta): __excluded__: 'list[str]' = [] #: List of names of dunder keys that iterating an instance, ``len()`` and #: :meth:`to_dict` carry -- metadata such as ``__next_type__``, which a - #: rebuild from the :obj:`dict` reads. Every other dunder key is left - #: out; a key that is no dunder is carried regardless. + #: rebuild from the :obj:`dict` reads. Every other dunder key is dropped + #: from all three, though indexing still reads it; a key that is no + #: dunder is carried regardless. A subclass extends the list with its own + #: names, as for :attr:`__excluded__`, so a custom :class:`Info` holding + #: dunder keys of its own must list them here for :meth:`to_dict` to keep + #: them (:issue:`2003`). __included__: 'list[str]' = [] def __init_subclass__(cls, /, *args: 'Any', **kwargs: 'Any') -> 'None': diff --git a/pcapkit/protocols/data/data.py b/pcapkit/protocols/data/data.py index 42a72fbf82..1f6ecebef3 100644 --- a/pcapkit/protocols/data/data.py +++ b/pcapkit/protocols/data/data.py @@ -32,7 +32,9 @@ class Data(Info): """ + #: Metadata keys that :meth:`to_dict` carries, for a rebuild to read. __included__ = ['__next_name__', '__next_type__', '__short_read__'] + #: Metadata keys that ``repr``, ``str`` and the dumps leave out. __excluded__ = ['__next_name__', '__next_type__', '__short_read__'] if TYPE_CHECKING: From 755249666a5c1aa3743f8b5edf95da557df6f013 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 21 Oct 2026 14:08:20 -0400 Subject: [PATCH 6/6] fix(protocols): keep __exthdr__ as a tuple of parser and info pairs (#2003) __exthdr__ is main's tuple of (parser class, info) in wire order again, not an OrderedMultiInfo keyed by ExtensionHeader; the tuple order is the wire order, and the parser class keeps the #1473 IPv6_Ext fallback. to_dict carries the tuple as it is and from_dict restores it, so the chain is still rebuilt from it alone. Tests and the annotation describe the tuple form. --- pcapkit/protocols/data/internet/ipv6.py | 15 +++++++-------- pcapkit/protocols/internet/ipv6.py | 18 ++++++------------ .../internet/test_ipv6_core_roundtrip_unit.py | 2 +- .../test_ipv6_exthdr_cut_rebuild_unit.py | 2 +- .../internet/test_ipv6_from_data_ext_unit.py | 7 +++---- .../protocols/test_lookup_by_name_2003_unit.py | 17 ++++++++--------- 6 files changed, 26 insertions(+), 35 deletions(-) diff --git a/pcapkit/protocols/data/internet/ipv6.py b/pcapkit/protocols/data/internet/ipv6.py index 105d5018a9..73bd0e61d7 100644 --- a/pcapkit/protocols/data/internet/ipv6.py +++ b/pcapkit/protocols/data/internet/ipv6.py @@ -12,9 +12,7 @@ from typing_extensions import Literal - from pcapkit.const.ipv6.extension_header import ExtensionHeader from pcapkit.const.reg.transtype import TransType - from pcapkit.corekit.infoclass import OrderedMultiInfo from pcapkit.protocols.data.data import Data from pcapkit.protocols.data.protocol import Packet from pcapkit.protocols.protocol import ProtocolBase @@ -69,12 +67,13 @@ class IPv6(Protocol): #: Octets captured past the Payload Length -- or, for a jumbogram, #: past the Jumbo Payload Length -- present only when there are any. trailer: 'bytes' - #: Extension header chain, in wire order, keyed by extension header - #: code, each value the pair of the parser class and its info. The - #: per-header keys hold the same infos, but in the order of the keys - #: of this :class:`~pcapkit.corekit.infoclass.Info`, which differs on - #: PyPy, so the chain is rebuilt from this alone (:issue:`2003`). - __exthdr__: 'OrderedMultiInfo[ExtensionHeader, tuple[Type[ProtocolBase], Data]]' + #: Extension header chain, in wire order, as pairs of the parser + #: class and its info. The per-header keys hold the same infos, but in + #: the order of the keys of this :class:`~pcapkit.corekit.infoclass.Info`, + #: which differs on PyPy, so the chain is rebuilt from this alone; the + #: parser class keeps the IPv6_Ext a header fell back to + #: (:issue:`1473`, :issue:`2003`). + __exthdr__: 'tuple[tuple[Type[ProtocolBase], Data], ...]' def __new__(cls, *args: 'Any', **kwargs: 'Any') -> 'IPv6': self = super().__new__(cls, *args, **kwargs) diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index 1c8d8e7620..38dabf7d69 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -26,14 +26,13 @@ .. [*] https://en.wikipedia.org/wiki/IPv6_packet """ -import collections.abc import ipaddress from typing import TYPE_CHECKING from pcapkit.const.ipv6.extension_header import ExtensionHeader as Enum_ExtensionHeader from pcapkit.const.reg.transtype import TransType as Enum_TransType from pcapkit.corekit.decorators import beholder -from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict +from pcapkit.corekit.multidict import OrderedMultiDict from pcapkit.corekit.protochain import ProtoChain from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6 from pcapkit.protocols.internet.ip import IP @@ -416,12 +415,8 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over # guessed from the keys of ``data``, whose order is not part of the # contract of an :class:`~pcapkit.corekit.infoclass.Info` and differs # on PyPy (#2003). - exthdr = data.get('__exthdr__') # type: Optional[Mapping[Enum_ExtensionHeader, tuple[Type[ProtocolBase], Any]]] # pylint: disable=line-too-long - chain = [] # type: list[tuple[Type[ProtocolBase], Any]] - if isinstance(exthdr, MultiDict): - chain.extend(value for _, value in exthdr.items(multi=True)) - elif isinstance(exthdr, collections.abc.Mapping): - chain.extend(exthdr.values()) + exthdr = data.get('__exthdr__') # type: Optional[tuple[tuple[Type[ProtocolBase], Any], ...]] # pylint: disable=line-too-long + chain = tuple(exthdr) if isinstance(exthdr, (tuple, list)) else () upper = cls._make_payload(data) payload = upper # type: bytes | ProtocolBase @@ -528,8 +523,7 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, hdr_len = self.length # header length raw_len = ipv6.payload if length is None else length # payload length _protos = [] # type: list[ProtocolBase] # ProtoChain buffer - # (parser class, info) per extension header - _exthdr = OrderedMultiDict() # type: OrderedMultiDict[Enum_ExtensionHeader, tuple[Type[ProtocolBase], Any]] + _exthdr = [] # type: list[tuple[Type[ProtocolBase], Any]] # (parser class, info) per extension header split = None # type: Optional[tuple[int, int]] # at first Fragment header opaque = False # walk ended at a non-zero Fragment Offset depth = _frame_depth.get() # layers of the frame up to this one @@ -607,7 +601,7 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, # record protocol name # self._protos = ProtoChain(name, chain, alias) _protos.append(next_) - _exthdr.add(ex_proto, (type(next_), info)) + _exthdr.append((type(next_), info)) # update header & payload length hdr_len += next_.length # type: ignore[assignment] @@ -663,7 +657,7 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, # extension header chain in wire order, with the parser of each, # for ``from_data``; the parser keeps the IPv6_Ext a header fell # back to (#1473) - '__exthdr__': _exthdr, + '__exthdr__': tuple(_exthdr), }) # NOTE: Nor does a later fragment's slice start with the upper-layer diff --git a/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py b/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py index 18a4d5cfe3..6afa2f4ad3 100644 --- a/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py +++ b/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py @@ -67,7 +67,7 @@ def test_private_codes_are_not_extension_headers(self) -> None: with self.subTest(code=code): frame = ipv6(code, bytes.fromhex('3b00010203040506')) parsed = IPv6(frame) - self.assertEqual(len(parsed.info.__exthdr__), 0) + self.assertEqual(parsed.info.__exthdr__, ()) self.assertEqual(len(parsed.protochain), 2) self.assertEqual(IPv6.from_data(parsed.info).data, frame) diff --git a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py index 2d3b045d82..6dc492d5d6 100644 --- a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py +++ b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py @@ -112,7 +112,7 @@ def test_dict_rebuild_keeps_the_generic_parser(self) -> None: with self.subTest(cut=cut): ip6 = IPv6(data, len(data)) self.assertIs(type(ip6._exthdr[ExtensionHeader.HOPOPT]), IPv6_Ext) - chain = [value for _, value in ip6.info.to_dict()['__exthdr__'].items(multi=True)] + chain = ip6.info.to_dict()['__exthdr__'] self.assertEqual([parser for parser, _ in chain], [IPv6_Ext]) self.assertEqual(IPv6.from_data(ip6.info.to_dict()).data, data) diff --git a/tests/protocols/internet/test_ipv6_from_data_ext_unit.py b/tests/protocols/internet/test_ipv6_from_data_ext_unit.py index 9aa6269a54..d7b930f5d6 100644 --- a/tests/protocols/internet/test_ipv6_from_data_ext_unit.py +++ b/tests/protocols/internet/test_ipv6_from_data_ext_unit.py @@ -83,7 +83,6 @@ def test_repeated_header_keeps_both_instances(self) -> None: self.assertEqual(IPv6.from_data(parsed.info).data, packet) def test_chain_is_carried_by_to_dict_and_hidden_from_repr(self) -> None: - from pcapkit.const.ipv6.extension_header import ExtensionHeader from pcapkit.protocols.internet.ipv6 import IPv6 from pcapkit.protocols.internet.ipv6_route import IPv6_Route @@ -91,10 +90,10 @@ def test_chain_is_carried_by_to_dict_and_hidden_from_repr(self) -> None: info = IPv6(packet, len(packet)).info self.assertNotIn('__exthdr__', repr(info)) self.assertNotIn('__exthdr__', str(info)) - # Keyed by code, each the parser and the info, as ``to_dict`` carries it. + # The parser and the info of each, as ``to_dict`` carries it. exported = info.to_dict()['__exthdr__'] - self.assertEqual([(code, parser) for code, (parser, _) in exported.items(multi=True)], - [(ExtensionHeader.IPv6_Route, IPv6_Route)]) + self.assertEqual(exported, info.__exthdr__) + self.assertEqual([parser for parser, _ in exported], [IPv6_Route]) if __name__ == '__main__': diff --git a/tests/protocols/test_lookup_by_name_2003_unit.py b/tests/protocols/test_lookup_by_name_2003_unit.py index 6c45842f63..1f3bd7f9d3 100644 --- a/tests/protocols/test_lookup_by_name_2003_unit.py +++ b/tests/protocols/test_lookup_by_name_2003_unit.py @@ -12,9 +12,8 @@ The order of an :class:`~pcapkit.corekit.infoclass.Info` is not part of its contract, so nothing is read off it any more: ``to_dict()`` carries the ``__next_type__`` and ``__next_name__`` metadata the payload is dispatched on, -and ``__exthdr__``, the extension header chain keyed by code in an -:class:`~pcapkit.corekit.infoclass.OrderedMultiInfo` keeping the order of its -own. A dict without the metadata is not guessed at. These tests reorder an +and ``__exthdr__``, the extension header chain as a tuple of parser and info +pairs in wire order. A dict without the metadata is not guessed at. These tests reorder an instance's :attr:`~object.__dict__` as PyPy does, so CPython checks the same thing. @@ -88,7 +87,6 @@ def setUp(self) -> None: warnings.simplefilter('ignore') def test_extension_headers_follow_exthdr_whatever_the_key_order(self) -> None: - from pcapkit.const.ipv6.extension_header import ExtensionHeader from pcapkit.protocols.internet.hopopt import HOPOPT from pcapkit.protocols.internet.ipv6 import IPv6 from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts @@ -101,10 +99,11 @@ def test_extension_headers_follow_exthdr_whatever_the_key_order(self) -> None: for key, before in (('hopopt', None), ('__exthdr__', 'version')): with self.subTest(moved=key): exported = moved(ip6.info, key, before=before).to_dict() - chain = list(exported['__exthdr__'].items(multi=True)) - self.assertEqual([code for code, _ in chain], - [ExtensionHeader.HOPOPT, ExtensionHeader.IPv6_Opts, ExtensionHeader.IPv6_Opts]) - self.assertEqual([parser for _, (parser, _) in chain], [HOPOPT, IPv6_Opts, IPv6_Opts]) + chain = exported['__exthdr__'] + self.assertIsInstance(chain, tuple) + self.assertEqual([parser for parser, _ in chain], [HOPOPT, IPv6_Opts, IPv6_Opts]) + self.assertEqual([info for _, info in chain], + [ext.info for _, ext in ip6.extension_headers.items(multi=True)]) self.assertEqual(IPv6.from_data(exported).data, HOP_DST_DST) def test_payload_follows_the_metadata_whatever_the_key_order(self) -> None: @@ -142,7 +141,7 @@ def test_cut_upper_layer_is_found_ahead_of_the_headers(self) -> None: # As on PyPy for #1610: ``opts`` after the upper layer. info = moved(moved(ip6.info, 'raw', before='opts'), 'hopopt', before='raw') exported = info.to_dict() - self.assertEqual(len(list(exported['__exthdr__'].items(multi=True))), 2) + self.assertEqual(len(exported['__exthdr__']), 2) rebuilt = IPv6.from_data(exported) self.assertIsInstance(rebuilt.payload, Raw) self.assertEqual(rebuilt.data, HOP_DST_DST)