fix(protocols): find the payload by protocol name and follow IPv6 next headers, not key order (#2003)

docs/source/pcapkit/corekit/infoclass.rst

@@ -23,6 +23,8 @@ classes :class:`~pcapkit.corekit.infoclass.MultiInfo` and
       :no-value:
    .. autoattribute:: __excluded__
       :no-value:
+   .. autoattribute:: __included__
+      :no-value:
 
    .. autoattribute:: __map__
    .. autoattribute:: __map_reverse__

docs/source/pcapkit/protocols/index.rst

@@ -268,3 +268,6 @@ Data Model
 
          Please refer to :func:`Protocol._decode_next_layer <pcapkit.protocols.protocol.Protocol._decode_next_layer>`
          for more information with the inserted names to be excluded.
+
+   .. autoattribute:: __included__
+      :no-value:

pcapkit/corekit/infoclass.py

@@ -21,7 +21,7 @@
 import sys
 import types
 import typing
-from typing import TYPE_CHECKING, Generic, TypeVar
+from typing import TYPE_CHECKING, Generic, TypeVar, cast
 
 from pcapkit.corekit.enum import EnumLookup
 from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict, iter_multi_items
@@ -30,7 +30,8 @@
 from pcapkit.utilities.warnings import InfoWarning, warn
 
 if TYPE_CHECKING:
-    from typing import Any, ItemsView, Iterable, Iterator, NoReturn, Optional, Type
+    from typing import (Any, Collection, ItemsView, Iterable, Iterator, NoReturn, Optional,
+                        SupportsIndex, Type)
 
     from typing_extensions import Self
 
@@ -255,11 +256,13 @@ class InfoMeta(abc.ABCMeta):
     :class:`Info` class. It can be useful to reduce runtime generation
     cost as well as caching already generated attributes.
 
-    * :attr:`Info.__additional__` and :attr:`Info.__excluded__` are
-      lists of additional and excluded field names, which are used to
-      determine certain names to be included or excluded from the field
-      dictionary. They will be automatically populated from the class
-      attributes of the :class:`Info` class and its base classes.
+    * :attr:`Info.__additional__`, :attr:`Info.__excluded__` and
+      :attr:`Info.__included__` are lists of additional, excluded and included
+      field names, which are used to determine certain names to be treated as
+      built-in names, left out of ``repr`` and the dumps, or carried as
+      metadata by :meth:`Info.to_dict`. They will be automatically populated
+      from the class attributes of the :class:`Info` class and its base
+      classes.
 
       .. note::
 
@@ -286,8 +289,9 @@ class C(B):
     """
 
     def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]', **kwargs: 'Any') -> 'Type[Info]':
-        """Create the class, extending its :attr:`Info.__additional__` and
-        :attr:`Info.__excluded__` with every base's.
+        """Create the class, extending its :attr:`Info.__additional__`,
+        :attr:`Info.__excluded__` and :attr:`Info.__included__` with every
+        base's.
 
         Args:
             name: Class name.
@@ -300,13 +304,17 @@ def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]'
             attrs['__additional__'] = []
         if '__excluded__' not in attrs:
             attrs['__excluded__'] = []
+        if '__included__' not in attrs:
+            attrs['__included__'] = []
 
         for base in bases:
             if hasattr(base, '__additional__'):
                 attrs['__additional__'].extend(
                     name for name in base.__additional__ if name not in attrs['__additional__'])
             if hasattr(base, '__excluded__'):
                 attrs['__excluded__'].extend(name for name in base.__excluded__ if name not in attrs['__excluded__'])
+            if hasattr(base, '__included__'):
+                attrs['__included__'].extend(name for name in base.__included__ if name not in attrs['__included__'])
         return super().__new__(cls, name, bases, attrs, **kwargs)  # type: ignore[return-value]
 
 
@@ -331,6 +339,19 @@ class Info(Mapping[str, VT], Generic[VT], metaclass=InfoMeta):
 
        Iterating an instance skips the names listed in :attr:`__excluded__`.
 
+    A key whose name is a dunder, such as ``__next_type__``, is metadata.
+    Iterating an instance, ``len()`` and :meth:`to_dict` leave such a key out
+    unless :attr:`__included__` names it, and carry every other key;
+    :attr:`__excluded__` names the keys that ``repr``, ``str`` and the dumps of
+    :mod:`pcapkit.dumpkit` leave out besides. A subclass extends either list
+    with its own names, as for :attr:`__additional__`.
+
+    .. versionchanged:: 1.5.0
+
+       :attr:`__excluded__` names the keys left out of ``repr``, ``str`` and
+       the dumps only, and :attr:`__included__` the dunder keys that iterating
+       an instance, ``len()`` and :meth:`to_dict` carry (:issue:`2003`).
+
     """
 
     if TYPE_CHECKING:
@@ -352,8 +373,20 @@ class Info(Mapping[str, VT], Generic[VT], metaclass=InfoMeta):
 
     #: List of additional built-in names.
     __additional__: 'list[str]' = []
-    #: List of names to be excluded from :obj:`dict` conversion.
+    #: List of names of keys left out of ``repr``, ``str`` and the dumps of
+    #: :mod:`pcapkit.dumpkit` -- metadata such as ``__next_type__``, which
+    #: :meth:`to_dict` may still carry, see :attr:`__included__`.
+    #: Finalisation adds the names in :attr:`__builtin__`.
     __excluded__: 'list[str]' = []
+    #: List of names of dunder keys that iterating an instance, ``len()`` and
+    #: :meth:`to_dict` carry -- metadata such as ``__next_type__``, which a
+    #: rebuild from the :obj:`dict` reads. Every other dunder key is dropped
+    #: from all three, though indexing still reads it; a key that is no
+    #: dunder is carried regardless. A subclass extends the list with its own
+    #: names, as for :attr:`__excluded__`, so a custom :class:`Info` holding
+    #: dunder keys of its own must list them here for :meth:`to_dict` to keep
+    #: them (:issue:`2003`).
+    __included__: 'list[str]' = []
 
     def __init_subclass__(cls, /, *args: 'Any', **kwargs: 'Any') -> 'None':
         """Refuse to derive from a finalised info class.
@@ -446,9 +479,9 @@ class is finalised on its first construction, whatever its
         # ``info_final`` writes ``BASE`` or ``FINAL`` onto the class it
         # finalises, and a subclass reading that state through inheritance was
         # never finalised itself -- it would keep the ``__excluded__`` it copied
-        # at declaration, without the names ``info_final`` adds, and leak
-        # ``__map__``, ``__map_reverse__`` and ``__multi__`` from ``to_dict()``
-        # (GitHub issue #1490). The inner one: an ancestor hand-marked bare
+        # at declaration, without the names ``info_final`` adds, which once
+        # leaked ``__map__``, ``__map_reverse__`` and ``__multi__`` from
+        # ``to_dict()`` (GitHub issue #1490). The inner one: an ancestor hand-marked bare
         # ``@final`` passes its marker to every subclass declared before it
         # landed, and ``getattr`` would blame such a subclass for a mismarking
         # that is its ancestor's. ``OwnDictRuleTests.test_a_subclass_that_
@@ -579,13 +612,14 @@ def __repr__(self) -> 'str':
 
     def __len__(self) -> 'int':
         # NOTE: count exactly the keys :meth:`__iter__` yields, so the
-        # bookkeeping attributes in ``__excluded__`` are not counted.
-        return sum(1 for key in self.__dict__ if key not in self.__excluded__)
+        # bookkeeping attributes and the dunder keys ``__included__`` does not
+        # name are not counted, while the keys in ``__excluded__`` are (#2003).
+        return sum(1 for key in self.__dict__ if not self.__hidden(key))
 
     def __iter__(self) -> 'Iterator[str]':
         for key in self.__dict__:
-            if key in self.__excluded__:
-               continue
+            if self.__hidden(key):
+                continue
             yield self.__map_reverse__.get(key, key)
 
     def __getitem__(self, name: 'str') -> 'VT':
@@ -609,10 +643,22 @@ def items(self, multi: 'bool' = False) -> 'ItemsView[str, VT] | Iterator[tuple[s
     def __items_multi(self) -> 'Iterator[tuple[str, VT]]':
         """Every ``(key, value)`` pair, with the transformed keys, in order."""
         for (key, value) in self.__dict__.items():
-            if key not in self.__excluded__:
+            if not self.__hidden(key):
                 yield key, value
             yield from self.__multi__.get(key, ())
 
+    def __hidden(self, key: 'str') -> 'bool':
+        """Whether iteration, ``len()`` and :meth:`to_dict` leave ``key`` out.
+
+        That is a bookkeeping attribute in :attr:`__builtin__`, or a dunder key
+        :attr:`__included__` does not name (:issue:`2003`).
+
+        """
+        if key in self.__builtin__:
+            return True
+        return (isinstance(key, str) and key.startswith('__') and key.endswith('__')
+                and key not in self.__included__)
+
     def __setattr__(self, name: 'str', value: 'VT') -> 'NoReturn':
         raise UnsupportedCall("can't set attribute")
 
@@ -719,10 +765,14 @@ def to_dict(self) -> 'OrderedMultiDict[str, VT]':
 
            Returns an :class:`~pcapkit.corekit.multidict.OrderedMultiDict`
            rather than a :obj:`dict`, so a key held more than once keeps every
-           value (:issue:`1484`).
+           value (:issue:`1484`). It leaves out the dunder keys
+           :attr:`__included__` does not name, carries the keys in
+           :attr:`__excluded__`, and records those as its own ``__excluded__``
+           for the dumpers to leave out (:issue:`2003`).
 
         """
-        dict_ = _OrderedMultiDict()  # type: OrderedMultiDict[str, Any]
+        dict_ = _OrderedMultiDict()  # type: _OrderedMultiDict[str, Any]
+        dict_.__excluded__ = self.__excluded__
         for (key, value) in self.__items_multi():
             out_key = self.__map_reverse__.get(key, key)
             if isinstance(value, Info):
@@ -775,13 +825,41 @@ class _OrderedMultiDict(OrderedMultiDict[KT, VT]):
 
     __ne__ = _ne
 
+    #: Names of the keys the dumpers leave out: the :attr:`Info.__excluded__`
+    #: of the info exported, kept by copies, deep copies and pickles; none for
+    #: an export of a multi-mapping (:issue:`2003`).
+    __excluded__ = ()  # type: Collection[str]
+
     def __repr__(self) -> 'str':
         """Write the export as the :class:`~pcapkit.corekit.multidict.OrderedMultiDict`
         it is, with every pair.
 
         """
         return f'{OrderedMultiDict.__name__}({list(self.items(multi=True))!r})'
 
+    def copy(self) -> 'Self':
+        """Return a shallow copy, with the same ``__excluded__``."""
+        new = super().copy()
+        new.__excluded__ = self.__excluded__
+        return new
+
+    def deepcopy(self, memo: 'Optional[dict]' = None) -> 'Self':
+        """Return a deep copy, with the same ``__excluded__``."""
+        new = cast('Self', super().deepcopy(memo))
+        new.__excluded__ = self.__excluded__
+        return new
+
+    def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'Any':
+        return _restore_export, (type(self), list(self.items(multi=True)), list(self.__excluded__))
+
+
+def _restore_export(cls: 'Type[_OrderedMultiDict]', pairs: 'list[tuple[Any, Any]]',
+                    excluded: 'list[str]') -> '_OrderedMultiDict':
+    """Unpickle an :class:`_OrderedMultiDict`, with its ``__excluded__``."""
+    new = cls(pairs)
+    new.__excluded__ = excluded
+    return new
+
 
 class _MultiInfo(Info[VT], Generic[KT, VT]):
     """An :class:`Info` that holds its values as a multi-mapping.

pcapkit/dumpkit/common.py

@@ -41,7 +41,7 @@
 
 
 if TYPE_CHECKING:
-    from typing import Any, DefaultDict, Iterator, Optional, TextIO, Type
+    from typing import Any, Collection, DefaultDict, Iterator, Optional, TextIO, Type
 
     from dictdumper.dumper import Dumper as ABCDumper
     from typing_extensions import Literal
@@ -409,6 +409,26 @@ def escape_key(key: 'Any') -> 'Any':
             return json.dumps(format(key, ''))[1:-1]
         return key
 
+    def excluded(info: 'Info | MultiDict[Any, Any]') -> 'Collection[Any]':
+        """Return the keys of ``info`` the writer leaves out.
+
+        Args:
+            info: An :class:`~pcapkit.corekit.infoclass.Info`, or a multi-mapping
+                such as the export its :meth:`~pcapkit.corekit.infoclass.Info.to_dict`
+                returns.
+
+        Returns:
+            The :attr:`~pcapkit.corekit.infoclass.Info.__excluded__` of a field
+            model, or of the field model an export was made from -- metadata
+            such as ``__next_type__``, which only a rebuild reads
+            (:issue:`2003`). None for an option list, whose keys are its
+            options.
+
+        """
+        if isinstance(info, Info) and isinstance(info, MultiDict):
+            return ()
+        return getattr(info, '__excluded__', ())
+
     def fields(info: 'Info | MultiDict[Any, Any]') -> 'dict[Any, Any]':
         """Return the fields of ``info`` as the :class:`dict` the writer is handed.
 
@@ -423,10 +443,12 @@ def fields(info: 'Info | MultiDict[Any, Any]') -> 'dict[Any, Any]':
             held more than once, such as a repeated IPv6 extension header, maps
             to the :class:`list` of all its values, in order (GitHub issue
             :issue:`1484`). The values are left as they are, so each comes back
-            through :meth:`DictDumper.object_hook` when written.
+            through :meth:`DictDumper.object_hook` when written. The keys
+            :func:`excluded` names are left out.
 
         """
-        pairs = list(info.items(multi=True))
+        skip = excluded(info)
+        pairs = [(key, val) for key, val in info.items(multi=True) if key not in skip]
         counts = collections.Counter(key for key, _ in pairs)
         result = {}  # type: dict[Any, Any]
         for key, val in pairs:
@@ -540,10 +562,13 @@ def object_hook(self, o: 'Any') -> 'Any':
                 # by name could only gather the repeats of a key under its first
                 # occurrence (GitHub issue #1263). The keys are escaped when the
                 # writer hands each mapping back to this hook, through the
-                # :class:`dict` branch below.
+                # :class:`dict` branch below. The keys an export of an
+                # :class:`Info` excludes are left out, as :func:`fields` leaves
+                # them out (#2003).
+                skip = excluded(o)
                 result = [
                     {render_enum(key) if isinstance(key, (enum.Enum, aenum.Enum)) else key: val}
-                    for key, val in o.items(multi=True)
+                    for key, val in o.items(multi=True) if key not in skip
                 ]
             elif isinstance(o, MultiDict):
                 # NOTE: a :class:`MultiDict` keeps no order across keys, only

pcapkit/protocols/data/application/httpv2.py

@@ -31,6 +31,10 @@
 class Flags(Data):
     """Data model for HTTP/2 flags."""
 
+    # NOTE: ``__value__`` is a dunder key, which ``to_dict`` carries only so
+    # named (#2003); the rebuild reads it, and the dumps write it, as before.
+    __included__ = ['__value__']
+
     if TYPE_CHECKING:
         #: Flags as in combination value, bits the frame type leaves
         #: undefined included.

pcapkit/protocols/data/data.py

@@ -14,18 +14,37 @@
 
 
 class Data(Info):
-    """Base class for data models."""
-
-    __excluded__ = ['__next_name__', '__next_type__']
+    """Base class for data models.
+
+    The ``__next_name__``, ``__next_type__`` and ``__short_read__`` keys are
+    metadata, listed in :attr:`~pcapkit.corekit.infoclass.Info.__included__`
+    and :attr:`~pcapkit.corekit.infoclass.Info.__excluded__`:
+    :meth:`Protocol.from_data <pcapkit.protocols.protocol.ProtocolBase.from_data>`
+    dispatches the payload on the first two and on nothing else, and cuts a
+    short read back on the third. :meth:`to_dict` carries them, so that
+    ``from_data(info.to_dict())`` rebuilds as ``from_data(info)`` does, while
+    the dumpers and ``repr`` leave them out. A hand-written :obj:`dict` must
+    carry them as well -- and, for IPv6, the extension header chain under
+    ``__exthdr__`` -- to be rebuilt the same way: without the first two, no
+    payload layer is dispatched, as for a protocol with no next layer, and
+    without ``__exthdr__`` no IPv6 extension header is rebuilt
+    (:issue:`2003`).
+
+    """
+
+    #: Metadata keys that :meth:`to_dict` carries, for a rebuild to read.
+    __included__ = ['__next_name__', '__next_type__', '__short_read__']
+    #: Metadata keys that ``repr``, ``str`` and the dumps leave out.
+    __excluded__ = ['__next_name__', '__next_type__', '__short_read__']
 
     if TYPE_CHECKING:
         #: Next field name, i.e., the name of the payload field.
         __next_name__: 'str'
         #: Next field type, i.e., the type of the payload field.
         __next_type__: 'Type[ProtocolBase]'
         #: Name of the header field the data ended inside, and how many of its
-        #: octets were there; present only then. Not excluded, so that
-        #: :meth:`to_dict` carries it and ``from_data(info.to_dict())`` cuts
-        #: the rebuild back as ``from_data(info)`` does. See
+        #: octets were there; present only then. :meth:`to_dict` carries it,
+        #: so that ``from_data(info.to_dict())`` cuts the rebuild back as
+        #: ``from_data(info)`` does. See
         #: :func:`~pcapkit.protocols.schema.schema.keep_short_read`.
         __short_read__: 'tuple[str, int]'

pcapkit/protocols/data/internet/ipv6.py

@@ -33,6 +33,9 @@ class IPv6(Protocol):
 
     """
 
+    # NOTE: ``__exthdr__`` is metadata, as the keys ``Data`` names are:
+    # ``to_dict`` carries it, the dumps and ``repr`` leave it out (#2003).
+    __included__ = ['__exthdr__']
     __excluded__ = ['__exthdr__']
 
     #: Version.
@@ -65,8 +68,11 @@ class IPv6(Protocol):
         #: past the Jumbo Payload Length -- present only when there are any.
         trailer: 'bytes'
         #: Extension header chain, in wire order, as pairs of the parser
-        #: class and its info. Kept apart from the per-header keys above,
-        #: which a repeated header overwrites.
+        #: class and its info. The per-header keys hold the same infos, but in
+        #: the order of the keys of this :class:`~pcapkit.corekit.infoclass.Info`,
+        #: which differs on PyPy, so the chain is rebuilt from this alone; the
+        #: parser class keeps the IPv6_Ext a header fell back to
+        #: (:issue:`1473`, :issue:`2003`).
         __exthdr__: 'tuple[tuple[Type[ProtocolBase], Data], ...]'
 
     def __new__(cls, *args: 'Any', **kwargs: 'Any') -> 'IPv6':

pcapkit/protocols/data/misc/pcapng.py

@@ -65,6 +65,11 @@
 class PCAPNG(Protocol):
     """Data model for PCAP-NG file blocks."""
 
+    # NOTE: ``__truncated_raw__`` is a dunder key, which ``to_dict`` carries
+    # only so named (#2003); the rebuild reads it, and the dumps write it, as
+    # before.
+    __included__ = ['__truncated_raw__']
+
     #: Block type.
     type: 'Enum_BlockType'
     #: Block total length.