From eab4a4a1019794d36b58b17c1825e99c6bdcdad1 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Fri, 10 Jul 2026 00:56:58 +0000 Subject: [PATCH 1/8] feat(meshing): add mesh_meta module as mesh metadata source MeshMeta wraps a ChunkedGraph and exposes mesh dir locations and per-dataset params from custom_data["mesh"] (the single source of truth), shared branch-agnostically across the pcgv2/pcgv3 lines. Resolves absolute initial/dynamic dirs so meshes can live in a different bucket than the watershed, and flags graphs old clients can't reach. Co-Authored-By: Claude --- pychunkedgraph/meshing/mesh_meta/README.md | 69 +++++++++++++++++++ pychunkedgraph/meshing/mesh_meta/__init__.py | 5 ++ pychunkedgraph/meshing/mesh_meta/core.py | 71 ++++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 pychunkedgraph/meshing/mesh_meta/README.md create mode 100644 pychunkedgraph/meshing/mesh_meta/__init__.py create mode 100644 pychunkedgraph/meshing/mesh_meta/core.py diff --git a/pychunkedgraph/meshing/mesh_meta/README.md b/pychunkedgraph/meshing/mesh_meta/README.md new file mode 100644 index 000000000..6451f5847 --- /dev/null +++ b/pychunkedgraph/meshing/mesh_meta/README.md @@ -0,0 +1,69 @@ +# mesh_meta + +Single, branch-agnostic home for **graphene mesh metadata**. Every value comes +from **`custom_data["mesh"]`, the single source of truth**. `MeshMeta(cg)` wraps a +ChunkedGraph and exposes, as properties, the mesh dir/bucket locations and the +per-dataset mesh params used by the v2 manifest and by remeshing; multi-resolution +(LOD) mesh meta will live here too as it lands, so this is the module to grow +rather than scattering mesh-meta reads across the code. + +## Why it exists + +Supervoxels (watershed) can move to a cheap, egress-waived bucket while meshes +stay put. The v2 graphene manifest therefore carries **absolute mesh bucket +paths** rather than paths composed relative to the watershed `data_dir`, so a +client fetches meshes straight from wherever they live. This module is the one +place that decides where meshes live, and what the mesh params are, for a given +ChunkedGraph. + +## Two mesh locations + +- **initial** — the sharded meshes produced at ingest. **Shared / graph-independent**: + several ChunkedGraphs (e.g. copies) point at the *same* initial meshes, so this + location is never namespaced by graph id. +- **dynamic** — the unsharded meshes produced by proofreading edits. + **Per-graph**: each graph keeps its own, so on the pcgv3 line it defaults to a + graph-id-derived dir (`dynamic_`) and stays isolated even while + sharing initial meshes; on the pcgv2 line it is a plain subdir. + +## Configuration + +All read from `custom_data["mesh"]`: + +- `dir` — the mesh root under the watershed (default `graphene_meshes`). +- `initial_mesh_dir` — the initial location (default `initial`). +- `dynamic_mesh_dir` — the dynamic location (default `dynamic`; pcgv3 setup fills + `dynamic_`). +- `max_layer`, `mip`, `max_error` — mesh params (start layer for manifests, and + the mip / max-error used by remeshing). +- `initial_ts` — the ingest timestamp boundary that splits initial (sharded) node + ids from proofread (dynamic) ones. + +Resolution rule for the two location values: a value containing a cloud scheme +(`gs://`, `s3://`, …) is treated as an **absolute** bucket and used as-is; +otherwise it is joined under `/`. This lets a dataset move +initial and/or dynamic meshes to different buckets without touching anything else. + +Branch-agnostic on purpose: this module reads only `custom_data["mesh"]` and the +watershed path — accessors identical on the pcgv2 and pcgv3 lines — so it +cherry-picks clean between them. How that config gets *populated* (e.g. pcgv3's +setup-time `MeshConfig` from the dataset yaml) is branch-specific and lives +upstream; this module only reads the result. + +## API + +`MeshMeta(cg)` exposes these properties: + +- `initial_path` → absolute dir of the initial (sharded) meshes. +- `dynamic_path` → absolute dir of the dynamic (unsharded) meshes. +- `dir` → the sharded mesh dir name under the watershed. +- `max_layer`, `mip`, `max_error`, `initial_ts` → the mesh params. +- `needs_v2` → true when either location falls **outside** the watershed dir, + i.e. meshes are not co-located with the watershed. Old clients compose paths + relative to the watershed, so they cannot reach such meshes: the v2 manifest + serves them correctly, and the v1 path must fail loudly rather than return + unreachable paths. +- `reader_anchor` → the `(data_dir, mesh)` pair to hand a CloudVolume mesh reader. + The reader appends `initial/` internally, so it must be anchored at the + **parent** of the initial dir; this yields shard byte-ranges from the right + bucket even for a migrated graph. diff --git a/pychunkedgraph/meshing/mesh_meta/__init__.py b/pychunkedgraph/meshing/mesh_meta/__init__.py new file mode 100644 index 000000000..33b6c72fd --- /dev/null +++ b/pychunkedgraph/meshing/mesh_meta/__init__.py @@ -0,0 +1,5 @@ +"""Branch-agnostic graphene mesh metadata. See README.md.""" + +from .core import MeshMeta + +__all__ = ["MeshMeta"] diff --git a/pychunkedgraph/meshing/mesh_meta/core.py b/pychunkedgraph/meshing/mesh_meta/core.py new file mode 100644 index 000000000..ae396ace9 --- /dev/null +++ b/pychunkedgraph/meshing/mesh_meta/core.py @@ -0,0 +1,71 @@ +"""MeshMeta — mesh dir locations and per-dataset params for a ChunkedGraph. + +``custom_data["mesh"]`` is the single source of truth. See README.md. +""" + +import posixpath + +_DEFAULT_MESH_DIR = "graphene_meshes" + + +def _resolve(root: str, value: str) -> str: + """Absolute ``value`` (a cloud URL) as-is, else joined under ``root``.""" + return value.rstrip("/") if "://" in value else posixpath.join(root, value) + + +class MeshMeta: + """Mesh metadata for a ChunkedGraph, read from ``custom_data["mesh"]``.""" + + def __init__(self, cg): + self._mesh = cg.meta.custom_data.get("mesh", {}) + self._watershed = cg.meta.data_source.WATERSHED + + @property + def dir(self) -> str: + """Sharded mesh dir name under the watershed.""" + return self._mesh.get("dir", _DEFAULT_MESH_DIR) + + @property + def _root(self) -> str: + return posixpath.join(self._watershed, self.dir) + + @property + def initial_path(self) -> str: + """Absolute dir of the shared initial (sharded) meshes.""" + return _resolve(self._root, self._mesh.get("initial_mesh_dir", "initial")) + + @property + def dynamic_path(self) -> str: + """Absolute dir of the per-graph dynamic (unsharded) meshes.""" + return _resolve(self._root, self._mesh.get("dynamic_mesh_dir", "dynamic")) + + @property + def needs_v2(self) -> bool: + """True when meshes live outside the watershed dir (old clients can't reach).""" + root = self._root + return not ( + self.initial_path.startswith(root) + and self.dynamic_path.startswith(root) + ) + + @property + def reader_anchor(self): + """``(data_dir, mesh)`` so a CloudVolume that appends ``initial/`` lands at + ``initial_path`` — points the shard reader at a migrated bucket.""" + return posixpath.split(posixpath.dirname(self.initial_path)) + + @property + def max_layer(self) -> int: + return self._mesh.get("max_layer", 2) + + @property + def mip(self) -> int: + return self._mesh["mip"] + + @property + def max_error(self): + return self._mesh["max_error"] + + @property + def initial_ts(self): + return self._mesh["initial_ts"] From 00ebda85d2dc423f73f1419e1ae146d09e25c5b0 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Fri, 10 Jul 2026 02:35:04 +0000 Subject: [PATCH 2/8] feat(meshing): add v2 manifest format + Accept negotiation Branch-agnostic manifest/v2.py: parse the client's advertised manifest version from the Accept header, and assemble the v2 format that groups fragments by absolute mesh bucket path so mesh location travels in the manifest itself. Co-Authored-By: Claude --- pychunkedgraph/meshing/manifest/v2.py | 65 +++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 pychunkedgraph/meshing/manifest/v2.py diff --git a/pychunkedgraph/meshing/manifest/v2.py b/pychunkedgraph/meshing/manifest/v2.py new file mode 100644 index 000000000..71207a0c8 --- /dev/null +++ b/pychunkedgraph/meshing/manifest/v2.py @@ -0,0 +1,65 @@ +"""v2 graphene manifest: Accept-header negotiation + format assembly. + +Branch-agnostic, pure-stdlib. The v2 format keys ``fragments`` by absolute mesh +bucket path so mesh location travels in the manifest itself; a client advertises +support via ``Accept: application/x.cave;manifest_version=2``. +""" + +ACCEPT_MEDIA_TYPE = "application/x.cave" +MANIFEST_VERSION = 2 # highest manifest version this server produces + + +def requested_manifest_version(accept_header, default: int = 1) -> int: + """Manifest version the client advertised via + ``Accept: application/x.cave;manifest_version=N``. + + Parsed with stdlib — werkzeug's ``MIMEAccept`` mangles media-type params. + """ + if not accept_header: + return default + for part in accept_header.split(","): + params = [p.strip() for p in part.split(";")] + if params[0].lower() != ACCEPT_MEDIA_TYPE: + continue + for param in params[1:]: + key, _, value = param.partition("=") + if key.strip().lower() == "manifest_version": + try: + return int(value.strip()) + except ValueError: + return default + return default + + +def to_v2_groups(node_ids, fragments, seg_id_in_fragment: bool): + """Split the v1 ``(node_ids, fragments)`` output into + ``(initial_frags, dynamic_frags)`` for the v2 format. + + Initial (sharded) fragments are marked by a leading ``~``; the marker is + dropped and the seg id ensured as the leading field. Dynamic fragments carry + no marker and already lead with the seg id. ``seg_id_in_fragment`` is True + when the initial fragments already embed the seg id (speculative), False when + it must be prepended (verified). + """ + initial, dynamic = [], [] + for node_id, frag in zip(node_ids, fragments): + if frag.startswith("~"): + body = frag[1:] + initial.append(body if seg_id_in_fragment else f"{node_id}:{body}") + else: + dynamic.append(frag) + return initial, dynamic + + +def assemble(initial_path: str, dynamic_path: str, initial_frags, dynamic_frags) -> dict: + """v2 manifest dict: fragment lists grouped by absolute bucket path. + + Empty groups are omitted; each bucket value is a sub-object so per-bucket + metadata can be added later without breaking the shape. + """ + fragments = {} + if len(initial_frags): + fragments[initial_path] = {"fragments": list(initial_frags)} + if len(dynamic_frags): + fragments[dynamic_path] = {"fragments": list(dynamic_frags)} + return {"manifest_version": MANIFEST_VERSION, "fragments": fragments} From 1293035f01cd95670cdeb5c01c705a21408b60eb Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Fri, 10 Jul 2026 02:52:43 +0000 Subject: [PATCH 3/8] refactor(meshing): route manifest reads through MeshMeta Replace custom_data["mesh"] reads (mesh dir, max_layer, initial_ts, dynamic path) with MeshMeta, making the meshing manifest byte-identical to the main line so v2 changes cherry-pick clean both ways. Co-Authored-By: Claude --- pychunkedgraph/meshing/manifest/sharded.py | 5 +++-- pychunkedgraph/meshing/manifest/utils.py | 10 ++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pychunkedgraph/meshing/manifest/sharded.py b/pychunkedgraph/meshing/manifest/sharded.py index 8b122b235..afb41189a 100644 --- a/pychunkedgraph/meshing/manifest/sharded.py +++ b/pychunkedgraph/meshing/manifest/sharded.py @@ -10,6 +10,7 @@ from ...graph.types import empty_1d from ...graph.basetypes import NODE_ID from ...graph.chunks import utils as chunk_utils +from ..mesh_meta import MeshMeta def verified_manifest( @@ -61,7 +62,7 @@ def speculative_manifest( from ..meshgen_utils import get_json_info if start_layer is None: - start_layer = cg.meta.custom_data.get("mesh", {}).get("max_layer", 2) + start_layer = MeshMeta(cg).max_layer start = time() bounding_box = chunk_utils.normalize_bounding_box( @@ -89,7 +90,7 @@ def speculative_manifest( readers = CloudVolume( # pylint: disable=no-member "graphene://https://localhost/segmentation/table/dummy", - mesh_dir=cg.meta.custom_data.get("mesh", {}).get("dir", "graphene_meshes"), + mesh_dir=MeshMeta(cg).dir, info=get_json_info(cg), ).mesh.readers diff --git a/pychunkedgraph/meshing/manifest/utils.py b/pychunkedgraph/meshing/manifest/utils.py index 3c4538797..17f29e2a4 100644 --- a/pychunkedgraph/meshing/manifest/utils.py +++ b/pychunkedgraph/meshing/manifest/utils.py @@ -14,6 +14,7 @@ from .cache import ManifestCache from ..meshgen_utils import get_mesh_name from ..meshgen_utils import get_json_info +from ..mesh_meta import MeshMeta from ...graph import ChunkedGraph from ...graph.types import empty_1d from ...graph.basetypes import NODE_ID @@ -105,10 +106,7 @@ def _get_dynamic_meshes(cg, node_ids: Sequence[np.uint64]) -> Tuple[Dict, List]: if len(node_ids) == 0: return result, not_existing - mesh_meta = cg.meta.custom_data.get("mesh", {}) - mesh_dir = mesh_meta.get("dir", "graphene_meshes") - dynamic_dir = mesh_meta.get("dynamic_mesh_dir", "dynamic") - mesh_path = f"{cg.meta.data_source.WATERSHED}/{mesh_dir}/{dynamic_dir}" + mesh_path = MeshMeta(cg).dynamic_path cf = CloudFiles(mesh_path) manifest_cache = ManifestCache(cg.graph_id, initial=False) @@ -182,7 +180,7 @@ def segregate_node_ids(cg, node_ids): new = created by proofreading edit operations """ - initial_ts = cg.meta.custom_data["mesh"]["initial_ts"] + initial_ts = MeshMeta(cg).initial_ts initial_mesh_dt = np.datetime64(datetime.fromtimestamp(initial_ts)) node_ids_ts = cg.get_node_timestamps(node_ids) initial_mesh_mask = node_ids_ts < initial_mesh_dt @@ -198,7 +196,7 @@ def get_mesh_paths( ) -> Dict: shard_readers = CloudVolume( # pylint: disable=no-member "graphene://https://localhost/segmentation/table/dummy", - mesh_dir=cg.meta.custom_data.get("mesh", {}).get("dir", "graphene_meshes"), + mesh_dir=MeshMeta(cg).dir, info=get_json_info(cg), ).mesh From 251ade32274adf7e372adac19af201f14e0eceb4 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Fri, 10 Jul 2026 02:52:51 +0000 Subject: [PATCH 4/8] feat(meshing): serve v2 manifest via Accept negotiation Serve the v2 grouped-by-bucket manifest to clients that advertise manifest_version=2, else the v1 flat list (byte-identical). Refuse with 406 when a migrated graph is requested without v2, so old clients fail loudly instead of getting unreachable paths. Co-Authored-By: Claude --- pychunkedgraph/app/meshing/common.py | 47 +++++++++++++++++++++------- pychunkedgraph/graph/exceptions.py | 6 ++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/pychunkedgraph/app/meshing/common.py b/pychunkedgraph/app/meshing/common.py index af7f6f7a6..9f9bb79c0 100644 --- a/pychunkedgraph/app/meshing/common.py +++ b/pychunkedgraph/app/meshing/common.py @@ -9,6 +9,7 @@ from pychunkedgraph import __version__ from pychunkedgraph.app import app_utils from pychunkedgraph.graph import chunkedgraph +from pychunkedgraph.graph import exceptions as cg_exceptions __meshing_url_prefix__ = os.environ.get("MESHING_URL_PREFIX", "meshing") @@ -62,14 +63,25 @@ def handle_get_manifest(table_id, node_id): bounds = request.args["bounds"] bounding_box = np.array([b.split("-") for b in bounds.split("_")], dtype=int).T + from pychunkedgraph.meshing.mesh_meta import MeshMeta + from pychunkedgraph.meshing.manifest import v2 + cg = app_utils.get_cg(table_id) + mm = MeshMeta(cg) + manifest_version = v2.requested_manifest_version(request.headers.get("Accept")) + if manifest_version < 2 and mm.needs_v2: + raise cg_exceptions.NotAcceptable( + "This dataset serves meshes from an absolute path; upgrade your " + "client to one that requests " + "'Accept: application/x.cave;manifest_version=2'." + ) verify = request.args.get("verify", False) verify = verify in ["True", "true", "1", True] return_seg_ids = request.args.get("return_seg_ids", False) prepend_seg_ids = request.args.get("prepend_seg_ids", False) return_seg_ids = return_seg_ids in ["True", "true", "1", True] prepend_seg_ids = prepend_seg_ids in ["True", "true", "1", True] - start_layer = cg.meta.custom_data.get("mesh", {}).get("max_layer", 2) + start_layer = mm.max_layer start_layer = int(request.args.get("start_layer", start_layer)) if "start_layer" in data: start_layer = int(data["start_layer"]) @@ -86,13 +98,19 @@ def handle_get_manifest(table_id, node_id): flexible_start_layer, bounding_box, data, + manifest_version, ) - return manifest_response(cg, args) + response = make_response(jsonify(manifest_response(cg, args))) + response.headers["X-Manifest-Version"] = str(manifest_version) + response.headers["Vary"] = "Accept" + return response def manifest_response(cg, args): from pychunkedgraph.meshing.manifest import speculative_manifest_sharded from pychunkedgraph.meshing.manifest import get_highest_child_nodes_with_meshes + from pychunkedgraph.meshing.manifest import v2 + from pychunkedgraph.meshing.mesh_meta import MeshMeta ( node_id, @@ -103,25 +121,32 @@ def manifest_response(cg, args): flexible_start_layer, bounding_box, data, + manifest_version, ) = args - resp = {} - seg_ids = [] if not verify: - seg_ids, resp["fragments"] = speculative_manifest_sharded( + seg_ids, fragments = speculative_manifest_sharded( cg, node_id, start_layer=start_layer, bounding_box=bounding_box ) - else: - seg_ids, resp["fragments"] = get_highest_child_nodes_with_meshes( + seg_ids, fragments = get_highest_child_nodes_with_meshes( cg, np.uint64(node_id), start_layer=start_layer, bounding_box=bounding_box, ) - if prepend_seg_ids: - resp["fragments"] = [f"~{i}:{f}" for i, f in zip(seg_ids, resp["fragments"])] - if return_seg_ids: - resp["seg_ids"] = seg_ids + + if manifest_version >= 2: + mm = MeshMeta(cg) + initial, dynamic = v2.to_v2_groups( + seg_ids, fragments, seg_id_in_fragment=not verify + ) + resp = v2.assemble(mm.initial_path, mm.dynamic_path, initial, dynamic) + else: + resp = {"fragments": fragments} + if prepend_seg_ids: + resp["fragments"] = [f"~{i}:{f}" for i, f in zip(seg_ids, fragments)] + if return_seg_ids: + resp["seg_ids"] = seg_ids return _check_post_options(cg, resp, data, seg_ids) diff --git a/pychunkedgraph/graph/exceptions.py b/pychunkedgraph/graph/exceptions.py index 496f55e4f..cee7348b9 100644 --- a/pychunkedgraph/graph/exceptions.py +++ b/pychunkedgraph/graph/exceptions.py @@ -63,6 +63,12 @@ class Forbidden(ClientError): status_code = http_client.FORBIDDEN +class NotAcceptable(ClientError): + """Exception mapping a ``406 Not Acceptable`` response.""" + + status_code = http_client.NOT_ACCEPTABLE + + class Conflict(ClientError): """Exception mapping a ``409 Conflict`` response.""" From 8df9d0659f8cedbf0845f8efc8fba498a58d35b8 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Mon, 13 Jul 2026 16:04:40 +0000 Subject: [PATCH 5/8] fix(meshing): serve migrated meshes and preserve Vary: Accept Anchor the verify=1 shard reader at initial_path (via reader_anchor) so migrated datasets read initial meshes from the real bucket, and append Accept-Encoding to Vary so manifests keep advertising Accept. Co-Authored-By: Claude --- pychunkedgraph/app/common.py | 2 +- pychunkedgraph/meshing/manifest/utils.py | 11 ++++-- .../tests/meshing/test_mesh_meta.py | 35 +++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 pychunkedgraph/tests/meshing/test_mesh_meta.py diff --git a/pychunkedgraph/app/common.py b/pychunkedgraph/app/common.py index f29b482ac..e3b821077 100644 --- a/pychunkedgraph/app/common.py +++ b/pychunkedgraph/app/common.py @@ -85,7 +85,7 @@ def after_request(response): response.data = compression.gzip_compress(response.data) response.headers["Content-Encoding"] = "gzip" - response.headers["Vary"] = "Accept-Encoding" + response.vary.add("Accept-Encoding") response.headers["Content-Length"] = len(response.data) return response diff --git a/pychunkedgraph/meshing/manifest/utils.py b/pychunkedgraph/meshing/manifest/utils.py index 17f29e2a4..af5c58f38 100644 --- a/pychunkedgraph/meshing/manifest/utils.py +++ b/pychunkedgraph/meshing/manifest/utils.py @@ -194,10 +194,17 @@ def get_mesh_paths( node_ids: Sequence[np.uint64], stop_layer: int = 2, ) -> Dict: + # Point the shard reader at initial_path: cloud-volume resolves shards at + # join(info["data_dir"], info["mesh"], "initial"), so reader_anchor splits + # initial_path into that (data_dir, mesh) pair — repoints a migrated bucket. + info = get_json_info(cg) + data_dir, mesh_dir = MeshMeta(cg).reader_anchor + info["data_dir"] = data_dir + info["mesh"] = mesh_dir shard_readers = CloudVolume( # pylint: disable=no-member "graphene://https://localhost/segmentation/table/dummy", - mesh_dir=MeshMeta(cg).dir, - info=get_json_info(cg), + mesh_dir=mesh_dir, + info=info, ).mesh result = {} diff --git a/pychunkedgraph/tests/meshing/test_mesh_meta.py b/pychunkedgraph/tests/meshing/test_mesh_meta.py new file mode 100644 index 000000000..46a76c762 --- /dev/null +++ b/pychunkedgraph/tests/meshing/test_mesh_meta.py @@ -0,0 +1,35 @@ +"""Tests for pychunkedgraph.meshing.mesh_meta.MeshMeta.""" + +import posixpath +from types import SimpleNamespace + +from pychunkedgraph.meshing.mesh_meta import MeshMeta + + +def _cg(watershed, mesh): + meta = SimpleNamespace( + custom_data={"mesh": mesh}, + data_source=SimpleNamespace(WATERSHED=watershed), + ) + return SimpleNamespace(meta=meta) + + +class TestReaderAnchor: + """reader_anchor must recompose to initial_path via cloud-volume's hardcoded + ``initial`` subdir, so the verified shard reader reads the real bucket.""" + + def test_colocated_anchor_is_watershed_dir(self): + mm = MeshMeta(_cg("gs://ws/seg", {"dir": "graphene_meshes"})) + assert mm.reader_anchor == ("gs://ws/seg", "graphene_meshes") + assert posixpath.join(*mm.reader_anchor, "initial") == mm.initial_path + assert mm.needs_v2 is False + + def test_migrated_initial_anchor_recomposes(self): + mesh = { + "dir": "graphene_meshes", + "initial_mesh_dir": "gs://old/graphene_meshes/initial", + } + mm = MeshMeta(_cg("gs://cheap/seg", mesh)) + assert mm.initial_path == "gs://old/graphene_meshes/initial" + assert posixpath.join(*mm.reader_anchor, "initial") == mm.initial_path + assert mm.needs_v2 is True From b12a0af540be748b32a5397888c57fa6eb62d8ba Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Mon, 13 Jul 2026 16:26:38 +0000 Subject: [PATCH 6/8] fix(meshing): write remeshed dynamic meshes to dynamic_path _remeshing wrote unsharded meshes to the dataset_info-derived path, not the dynamic_path the v2 manifest advertises, so clients 404 on freshly remeshed meshes. Source mesh paths from MeshMeta. Co-Authored-By: Claude --- pychunkedgraph/app/meshing/common.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/pychunkedgraph/app/meshing/common.py b/pychunkedgraph/app/meshing/common.py index 9f9bb79c0..26de32227 100644 --- a/pychunkedgraph/app/meshing/common.py +++ b/pychunkedgraph/app/meshing/common.py @@ -180,24 +180,18 @@ def handle_remesh(table_id): def _remeshing(serialized_cg_info, lvl2_nodes): # nested: pulls meshing/cloudvolume, only needed at call time from pychunkedgraph.meshing import meshgen + from pychunkedgraph.meshing.mesh_meta import MeshMeta cg = chunkedgraph.ChunkedGraph(**serialized_cg_info) - cv_mesh_dir = cg.meta.dataset_info["mesh"] - cv_unsharded_mesh_dir = cg.meta.dataset_info["mesh_metadata"]["unsharded_mesh_dir"] - cv_unsharded_mesh_path = os.path.join( - cg.meta.data_source.WATERSHED, cv_mesh_dir, cv_unsharded_mesh_dir - ) - mesh_data = cg.meta.custom_data["mesh"] - - # TODO: stop_layer and mip should be configurable by dataset + mm = MeshMeta(cg) meshgen.remeshing( cg, lvl2_nodes, - stop_layer=mesh_data["max_layer"], - mip=mesh_data["mip"], - max_err=mesh_data["max_error"], - cv_sharded_mesh_dir=cv_mesh_dir, - cv_unsharded_mesh_path=cv_unsharded_mesh_path, + stop_layer=mm.max_layer, + mip=mm.mip, + max_err=mm.max_error, + cv_sharded_mesh_dir=mm.dir, + cv_unsharded_mesh_path=mm.dynamic_path, ) return Response(status=200) From d1266123fe49e6a831c71fcdbe88031269085c02 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Mon, 13 Jul 2026 20:07:37 +0000 Subject: [PATCH 7/8] fix(meshing): mirror the v1 manifest response in v2 v2 emits each fragment identical to v1 (seg id prepended when asked) and honors return_seg_ids, differing only by grouping fragments under their bucket key, so the client reuses its v1 handling unchanged. Co-Authored-By: Claude --- pychunkedgraph/app/meshing/common.py | 6 ++-- pychunkedgraph/meshing/manifest/v2.py | 20 +++++-------- .../tests/meshing/test_manifest_v2.py | 28 +++++++++++++++++++ 3 files changed, 38 insertions(+), 16 deletions(-) create mode 100644 pychunkedgraph/tests/meshing/test_manifest_v2.py diff --git a/pychunkedgraph/app/meshing/common.py b/pychunkedgraph/app/meshing/common.py index 26de32227..8d534a2ab 100644 --- a/pychunkedgraph/app/meshing/common.py +++ b/pychunkedgraph/app/meshing/common.py @@ -137,10 +137,10 @@ def manifest_response(cg, args): if manifest_version >= 2: mm = MeshMeta(cg) - initial, dynamic = v2.to_v2_groups( - seg_ids, fragments, seg_id_in_fragment=not verify - ) + initial, dynamic = v2.to_v2_groups(seg_ids, fragments, prepend_seg_ids) resp = v2.assemble(mm.initial_path, mm.dynamic_path, initial, dynamic) + if return_seg_ids: + resp["seg_ids"] = seg_ids else: resp = {"fragments": fragments} if prepend_seg_ids: diff --git a/pychunkedgraph/meshing/manifest/v2.py b/pychunkedgraph/meshing/manifest/v2.py index 71207a0c8..430f01c64 100644 --- a/pychunkedgraph/meshing/manifest/v2.py +++ b/pychunkedgraph/meshing/manifest/v2.py @@ -31,23 +31,17 @@ def requested_manifest_version(accept_header, default: int = 1) -> int: return default -def to_v2_groups(node_ids, fragments, seg_id_in_fragment: bool): - """Split the v1 ``(node_ids, fragments)`` output into - ``(initial_frags, dynamic_frags)`` for the v2 format. +def to_v2_groups(node_ids, fragments, prepend_seg_ids): + """Group v1 fragments into ``(initial, dynamic)`` by the leading ``~`` marker. - Initial (sharded) fragments are marked by a leading ``~``; the marker is - dropped and the seg id ensured as the leading field. Dynamic fragments carry - no marker and already lead with the seg id. ``seg_id_in_fragment`` is True - when the initial fragments already embed the seg id (speculative), False when - it must be prepended (verified). + Each fragment is emitted exactly as the v1 manifest would (seg id prepended + when requested); only the initial-vs-dynamic grouping is added. The raw ``~`` + selects the group, matching v1's ``~:``. """ initial, dynamic = [], [] for node_id, frag in zip(node_ids, fragments): - if frag.startswith("~"): - body = frag[1:] - initial.append(body if seg_id_in_fragment else f"{node_id}:{body}") - else: - dynamic.append(frag) + out = f"~{node_id}:{frag}" if prepend_seg_ids else frag + (initial if frag.startswith("~") else dynamic).append(out) return initial, dynamic diff --git a/pychunkedgraph/tests/meshing/test_manifest_v2.py b/pychunkedgraph/tests/meshing/test_manifest_v2.py new file mode 100644 index 000000000..b3337e267 --- /dev/null +++ b/pychunkedgraph/tests/meshing/test_manifest_v2.py @@ -0,0 +1,28 @@ +"""Tests for pychunkedgraph.meshing.manifest.v2.""" + +from pychunkedgraph.meshing.manifest.v2 import assemble, to_v2_groups + + +class TestToV2Groups: + def test_prepend_matches_v1_grouped_by_type(self): + """With prepend_seg_ids each fragment is v1's ``~:``; grouping + is by the raw ~ (initial/sharded) vs no-~ (dynamic/whole-file).""" + node_ids = [386, 529] + frags = ["~5/774017-0.shard:74809813:4532", "529:0:90112-98304"] + ini, dyn = to_v2_groups(node_ids, frags, prepend_seg_ids=True) + assert ini == ["~386:~5/774017-0.shard:74809813:4532"] + assert dyn == ["~529:529:0:90112-98304"] + + def test_no_prepend_keeps_raw(self): + ini, dyn = to_v2_groups( + [386, 529], ["~a.shard:1:2", "529:0:bbox"], prepend_seg_ids=False + ) + assert ini == ["~a.shard:1:2"] + assert dyn == ["529:0:bbox"] + + def test_assemble_groups_under_bucket(self): + resp = assemble( + "gs://b/initial", "gs://b/dynamic", ["~386:~a.shard:1:2"], ["~529:529:0:y"] + ) + assert resp["fragments"]["gs://b/initial"]["fragments"] == ["~386:~a.shard:1:2"] + assert resp["manifest_version"] == 2 From f9d4ab1a52e7bc870c581522c00c71d60ae50672 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 14 Jul 2026 13:13:35 +0000 Subject: [PATCH 8/8] feat(meshing): surface absolute mesh paths in the graphene info Emit initial/dynamic absolute bucket locations in mesh_metadata on both the served /info endpoint and get_json_info, so external clients and PCG's own manifest-bypass reads resolve meshes from the real buckets. Co-Authored-By: Claude --- pychunkedgraph/app/segmentation/common.py | 6 ++++++ pychunkedgraph/meshing/meshgen_utils.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/pychunkedgraph/app/segmentation/common.py b/pychunkedgraph/app/segmentation/common.py index 4c0f131a3..3b8eb4e87 100644 --- a/pychunkedgraph/app/segmentation/common.py +++ b/pychunkedgraph/app/segmentation/common.py @@ -18,6 +18,7 @@ logger = get_logger(__name__) from pychunkedgraph.app import app_utils +from pychunkedgraph.meshing.mesh_meta import MeshMeta from pychunkedgraph.graph import attributes, cutting, segmenthistory, ChunkedGraph from pychunkedgraph.graph import ( edges as cg_edges, @@ -122,6 +123,11 @@ def handle_info(table_id): # the right dir. Copy the dict so cg.meta.dataset_info is untouched. mesh_metadata = dict(combined_info.get("mesh_metadata", {})) mesh_metadata["unsharded_mesh_dir"] = dynamic_dir + # Absolute mesh bucket locations (== the v2 manifest keys) so clients that + # bypass the manifest resolve initial/dynamic meshes from the real buckets. + mm = MeshMeta(cg) + mesh_metadata["initial_mesh_path"] = mm.initial_path + mesh_metadata["dynamic_mesh_path"] = mm.dynamic_path combined_info["mesh_metadata"] = mesh_metadata return jsonify(combined_info) diff --git a/pychunkedgraph/meshing/meshgen_utils.py b/pychunkedgraph/meshing/meshgen_utils.py index 7b3471be7..eff3601a3 100644 --- a/pychunkedgraph/meshing/meshgen_utils.py +++ b/pychunkedgraph/meshing/meshgen_utils.py @@ -8,6 +8,7 @@ from pychunkedgraph.graph.basetypes import NODE_ID # noqa from ..graph.types import empty_1d from pychunkedgraph.graph.utils import get_local_segmentation +from .mesh_meta import MeshMeta def str_to_slice(slice_str: str): @@ -160,6 +161,11 @@ def get_json_info(cg): # the right dir. Copy the dict so cg.meta.dataset_info is untouched. mesh_metadata = dict(info.get("mesh_metadata", {})) mesh_metadata["unsharded_mesh_dir"] = dynamic_dir + # Absolute initial/dynamic bucket locations (the v2 manifest's bucket keys) + # so cloud-volume's manifest-bypass reads resolve to the real buckets. + mm = MeshMeta(cg) + mesh_metadata["initial_mesh_path"] = mm.initial_path + mesh_metadata["dynamic_mesh_path"] = mm.dynamic_path info["mesh_metadata"] = mesh_metadata info_str = dumps(info) return loads(info_str)