From 4bef5a3c43a2250be570a6421526196c5ec8363c Mon Sep 17 00:00:00 2001 From: klpoland Date: Wed, 8 Jul 2026 15:13:19 -0400 Subject: [PATCH 01/22] opensearch federated indices integration and tests --- common/sds_opensearch_query/pyproject.toml | 16 ++ .../sds_opensearch_query/__init__.py | 21 ++ .../sds_opensearch_query/filters.py | 75 ++++++ .../sds_opensearch_query/mapping.py | 41 +++ .../sds_opensearch_query/query.py | 88 ++++++ federation/.gitignore | 1 + federation/Dockerfile | 3 +- federation/compose.local.yaml | 4 +- federation/compose.production.yaml | 4 +- federation/justfile | 36 +++ federation/pyproject.toml | 4 + federation/scripts/local_e2e/README.md | 89 +++++++ .../scripts/local_e2e/download_crc_dataset.py | 82 ++++++ federation/scripts/local_e2e/env.example | 19 ++ .../local_e2e/upload_capture_to_local.py | 91 +++++++ .../local_e2e/verify_federation_live.py | 79 ++++++ federation/sds_federation/main.py | 8 + federation/sds_federation/models.py | 9 + .../sds_federation/routes/search_index.py | 103 +++++++ federation/sds_federation/routes/webhooks.py | 13 +- .../schemas/opensearch_indices.py | 85 ++++++ federation/sds_federation/schemas/webhooks.py | 6 +- .../sds_federation/services/bootstrap.py | 10 +- .../sds_federation/services/fed_indices.py | 19 ++ .../sds_federation/services/fed_search.py | 148 ++++++++++- .../sds_federation/services/local_events.py | 5 +- .../sds_federation/testing/sample_data.py | 4 +- federation/tests/conftest.py | 3 +- federation/tests/support/federation_mesh.py | 3 + .../tests/test_integration_bootstrap.py | 10 +- federation/tests/test_integration_mesh.py | 26 +- federation/tests/test_integration_pipeline.py | 3 +- federation/tests/test_integration_webhooks.py | 20 +- federation/tests/test_redis_event_pipeline.py | 11 +- .../tests/test_regression_fed_indices.py | 39 +++ .../tests/test_regression_fed_search.py | 37 ++- federation/tests/test_regression_indexer.py | 30 ++- .../test_regression_opensearch_indices.py | 61 +++++ .../tests/test_regression_opensearch_query.py | 49 ++++ federation/tests/test_regression_schemas.py | 11 +- federation/uv.lock | 13 + gateway/config/settings/base.py | 1 + gateway/pyproject.toml | 4 + .../api_methods/helpers/search_captures.py | 251 ++++-------------- .../commands/publish_for_federation.py | 73 +++++ .../tests/test_federation_export.py | 3 +- .../tests/test_federation_export_contract.py | 4 +- gateway/uv.lock | 15 +- 48 files changed, 1441 insertions(+), 289 deletions(-) create mode 100644 common/sds_opensearch_query/pyproject.toml create mode 100644 common/sds_opensearch_query/sds_opensearch_query/__init__.py create mode 100644 common/sds_opensearch_query/sds_opensearch_query/filters.py create mode 100644 common/sds_opensearch_query/sds_opensearch_query/mapping.py create mode 100644 common/sds_opensearch_query/sds_opensearch_query/query.py create mode 100644 federation/scripts/local_e2e/README.md create mode 100644 federation/scripts/local_e2e/download_crc_dataset.py create mode 100644 federation/scripts/local_e2e/env.example create mode 100644 federation/scripts/local_e2e/upload_capture_to_local.py create mode 100644 federation/scripts/local_e2e/verify_federation_live.py create mode 100644 federation/sds_federation/routes/search_index.py create mode 100644 federation/sds_federation/schemas/opensearch_indices.py create mode 100644 federation/sds_federation/services/fed_indices.py create mode 100644 federation/tests/test_regression_fed_indices.py create mode 100644 federation/tests/test_regression_opensearch_indices.py create mode 100644 federation/tests/test_regression_opensearch_query.py create mode 100644 gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py diff --git a/common/sds_opensearch_query/pyproject.toml b/common/sds_opensearch_query/pyproject.toml new file mode 100644 index 000000000..960cdc00a --- /dev/null +++ b/common/sds_opensearch_query/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["uv_build>=0.11.24,<0.12"] +build-backend = "uv_build" + +[tool.uv.build-backend] +module-name = "sds_opensearch_query" +module-root = "" + +[project] +name = "sds-opensearch-query" +version = "0.1.0" +description = "Shared OpenSearch query builders for SDS gateway and federation" +requires-python = ">=3.13,<3.15" +dependencies = [ + "opensearch-py>=2.7.1", +] diff --git a/common/sds_opensearch_query/sds_opensearch_query/__init__.py b/common/sds_opensearch_query/sds_opensearch_query/__init__.py new file mode 100644 index 000000000..b84914d66 --- /dev/null +++ b/common/sds_opensearch_query/sds_opensearch_query/__init__.py @@ -0,0 +1,21 @@ +"""Framework-agnostic OpenSearch query helpers.""" + +from sds_opensearch_query.filters import build_metadata_filter_clauses +from sds_opensearch_query.filters import nested_query_clause +from sds_opensearch_query.mapping import flatten_property_paths +from sds_opensearch_query.query import bool_must_search_body +from sds_opensearch_query.query import federation_not_deleted_clause +from sds_opensearch_query.query import multi_match_clause +from sds_opensearch_query.query import run_search +from sds_opensearch_query.query import term_clause + +__all__ = [ + "bool_must_search_body", + "build_metadata_filter_clauses", + "federation_not_deleted_clause", + "flatten_property_paths", + "multi_match_clause", + "nested_query_clause", + "run_search", + "term_clause", +] diff --git a/common/sds_opensearch_query/sds_opensearch_query/filters.py b/common/sds_opensearch_query/sds_opensearch_query/filters.py new file mode 100644 index 000000000..a7526a211 --- /dev/null +++ b/common/sds_opensearch_query/sds_opensearch_query/filters.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + + +def nested_query_clause( + field_path: str, + query_type: str, + value: Any, + *, + levels_nested: int | None = None, + last_path: str | None = None, +) -> dict[str, Any]: + """Build a nested OpenSearch clause for a dotted field path.""" + if levels_nested is None: + levels_nested = field_path.count(".") + + if levels_nested == 0: + key = f"{last_path}.{field_path}" if last_path else field_path + return {query_type: {key: value}} + + path_parts = field_path.split(".") + current_path = path_parts[0] + if last_path is not None: + current_path = f"{last_path}.{current_path}" + + return { + "nested": { + "path": current_path, + "query": nested_query_clause( + field_path=".".join(path_parts[1:]), + query_type=query_type, + value=value, + levels_nested=levels_nested - 1, + last_path=current_path, + ), + }, + } + + +def build_metadata_filter_clauses( + metadata_filters: list[dict[str, Any]] | None, + *, + known_field_paths: frozenset[str] | None = None, + on_unknown_field: Callable[[str], None] | None = None, +) -> list[dict[str, Any]]: + """Turn API metadata filter dicts into OpenSearch query clauses.""" + if not metadata_filters: + return [] + + clauses: list[dict[str, Any]] = [] + for query in metadata_filters: + field_path: str = query["field_path"] + query_type: str = query["query_type"] + filter_value: Any = query["filter_value"] + + if known_field_paths is not None and field_path not in known_field_paths: + if on_unknown_field is not None: + on_unknown_field(field_path) + + levels_nested = field_path.count(".") + if levels_nested > 0: + clauses.append( + nested_query_clause( + field_path=field_path, + query_type=query_type, + value=filter_value, + levels_nested=levels_nested, + ), + ) + else: + clauses.append({query_type: {field_path: filter_value}}) + + return clauses diff --git a/common/sds_opensearch_query/sds_opensearch_query/mapping.py b/common/sds_opensearch_query/sds_opensearch_query/mapping.py new file mode 100644 index 000000000..13713240e --- /dev/null +++ b/common/sds_opensearch_query/sds_opensearch_query/mapping.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + + +def flatten_property_paths( + properties: Mapping[str, Any], + *, + prefix: str = "", + separator: str = ".", +) -> frozenset[str]: + """Collect dotted field paths from an OpenSearch ``properties`` mapping.""" + paths: set[str] = set() + + for field, spec in properties.items(): + path = f"{prefix}{separator}{field}" if prefix else field + if not isinstance(spec, dict): + paths.add(path) + continue + + if spec.get("type") == "nested": + nested_props = spec.get("properties", {}) + if isinstance(nested_props, dict): + for nested_field in nested_props: + paths.add(f"{path}{separator}{nested_field}") + continue + + if "properties" in spec: + paths.update( + flatten_property_paths( + spec["properties"], + prefix=path, + separator=separator, + ), + ) + continue + + paths.add(path) + + return frozenset(paths) diff --git a/common/sds_opensearch_query/sds_opensearch_query/query.py b/common/sds_opensearch_query/sds_opensearch_query/query.py new file mode 100644 index 000000000..795751d43 --- /dev/null +++ b/common/sds_opensearch_query/sds_opensearch_query/query.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from typing import Any + +from opensearchpy import OpenSearch +from opensearchpy import exceptions as os_exceptions + +DEFAULT_SEARCH_SIZE = 10_000 + + +def term_clause(field: str, value: Any) -> dict[str, Any]: + return {"term": {field: value}} + + +def multi_match_clause( + query: str, + fields: list[str], + *, + match_type: str = "best_fields", +) -> dict[str, Any]: + return { + "multi_match": { + "query": query, + "fields": fields, + "type": match_type, + }, + } + + +def federation_not_deleted_clause() -> dict[str, Any]: + return {"term": {"is_federated_deleted": False}} + + +def bool_must_search_body( + *must_clauses: dict[str, Any], + source_includes: list[str] | None = None, +) -> dict[str, Any]: + body: dict[str, Any] = { + "query": { + "bool": { + "must": list(must_clauses), + }, + }, + } + if source_includes is not None: + body["_source"] = {"includes": source_includes} + return body + + +def _request_error_message(err: os_exceptions.RequestError) -> str: + info = err.info + if isinstance(info, dict): + root_causes: list[dict[str, str]] = info.get("error", {}).get( + "root_cause", + [], + ) + root_cause_reason = root_causes[0].get("reason", "") if root_causes else "" + reason = str(root_cause_reason) if root_cause_reason else str(info) + else: + reason = str(info) + return f"Query error: {reason}" + + +def run_search( + client: OpenSearch, + *, + index: str, + body: dict[str, Any], + size: int = DEFAULT_SEARCH_SIZE, +) -> list[dict[str, Any]]: + """Run a search and return raw hit dicts. Raises ValueError for client errors.""" + try: + response = client.search( + index=index, + body=body, + size=size, # pyright: ignore[reportCallIssue] + ) + except os_exceptions.NotFoundError as err: + msg = f"Index '{index}' not found" + raise ValueError(msg) from err + except os_exceptions.ConnectionError: + raise + except os_exceptions.RequestError as err: + raise ValueError(_request_error_message(err)) from err + except os_exceptions.OpenSearchException: + raise + + return list(response["hits"]["hits"]) diff --git a/federation/.gitignore b/federation/.gitignore index 8c7ad6af1..acbbb2d06 100644 --- a/federation/.gitignore +++ b/federation/.gitignore @@ -6,3 +6,4 @@ federation.toml .envs/* !/.envs/example +scripts/local_e2e/.env diff --git a/federation/Dockerfile b/federation/Dockerfile index 2f44a39c8..f0171082a 100644 --- a/federation/Dockerfile +++ b/federation/Dockerfile @@ -2,10 +2,11 @@ FROM docker.io/python:3.14-slim-trixie COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ -WORKDIR /app +WORKDIR /app/federation ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 +COPY ../common/sds_opensearch_query /app/common/sds_opensearch_query COPY pyproject.toml uv.lock ./ COPY sds_federation ./sds_federation RUN uv sync --frozen --no-dev diff --git a/federation/compose.local.yaml b/federation/compose.local.yaml index 52d07c195..539f100e5 100644 --- a/federation/compose.local.yaml +++ b/federation/compose.local.yaml @@ -9,8 +9,8 @@ networks: services: sds-federation-local-sync: build: - context: . - dockerfile: Dockerfile + context: .. + dockerfile: federation/Dockerfile image: sds-federation-local-sync container_name: sds-federation-local-sync env_file: diff --git a/federation/compose.production.yaml b/federation/compose.production.yaml index 439c85835..7d377c7c7 100644 --- a/federation/compose.production.yaml +++ b/federation/compose.production.yaml @@ -9,8 +9,8 @@ networks: services: sds-federation-prod-sync: build: - context: . - dockerfile: Dockerfile + context: .. + dockerfile: federation/Dockerfile image: sds-federation-prod-sync container_name: sds-federation-prod-sync env_file: diff --git a/federation/justfile b/federation/justfile index 0685386f0..0cbd0c361 100644 --- a/federation/justfile +++ b/federation/justfile @@ -80,6 +80,42 @@ gen-certs-ca: simulate-redis +args='': REDIS_URL="${REDIS_URL:-redis://localhost:6379/0}" uv run python scripts/simulate_redis_event.py {{ args }} +[group('dev')] +verify-federation-live +args='': + uv run python scripts/local_e2e/verify_federation_live.py {{ args }} + +[group('dev')] +local-e2e-env: + @if [ ! -f scripts/local_e2e/.env ]; then cp scripts/local_e2e/env.example scripts/local_e2e/.env; fi + @echo "Edit federation/scripts/local_e2e/.env (SDS_SECRET_TOKEN, LOCAL_SDS_SECRET_TOKEN, …)" + +# Download from CRC (SDK env); run `just local-e2e-env` first +[group('dev')] +download-crc-dataset dataset_uuid to='../federation/data/downloaded_dataset' +args='': + #!/usr/bin/env bash + set -eu -o pipefail + env_file="scripts/local_e2e/.env" + if [[ ! -f "$env_file" ]]; then + echo "Missing $env_file — run: just local-e2e-env" >&2 + exit 1 + fi + set -a && source "$env_file" && set +a + cd ../sdk && uv run python ../federation/scripts/local_e2e/download_crc_dataset.py \ + --dataset-uuid {{ dataset_uuid }} --to {{ to }} {{ args }} + +[group('dev')] +upload-capture-local local_path sds_path='federation-fixture/sample' +args='': + #!/usr/bin/env bash + set -eu -o pipefail + env_file="scripts/local_e2e/.env" + if [[ ! -f "$env_file" ]]; then + echo "Missing $env_file — run: just local-e2e-env" >&2 + exit 1 + fi + set -a && source "$env_file" && set +a + cd ../sdk && uv run python ../federation/scripts/local_e2e/upload_capture_to_local.py \ + --local-path {{ local_path }} --sds-path {{ sds_path }} {{ args }} + [group('docker')] build +args='': {{ docker_compose }} build {{ args }} diff --git a/federation/pyproject.toml b/federation/pyproject.toml index 4fa9fb01e..805f8aca1 100644 --- a/federation/pyproject.toml +++ b/federation/pyproject.toml @@ -16,6 +16,7 @@ module-root = "" "httpx>=0.28.1", "loguru>=0.7.2", "opensearch-py>=2.8.0", + "sds-opensearch-query", "pydantic>=2.11.0", "redis>=5.2.1", "uvicorn[standard]>=0.34.0", @@ -165,3 +166,6 @@ markers = [ "integration: tests that exercise FastAPI routes or multi-step pipeline", "regression: contract and indexer behavior guards", ] + +[tool.uv.sources] +sds-opensearch-query = { path = "../common/sds_opensearch_query" } diff --git a/federation/scripts/local_e2e/README.md b/federation/scripts/local_e2e/README.md new file mode 100644 index 000000000..9f63f7d20 --- /dev/null +++ b/federation/scripts/local_e2e/README.md @@ -0,0 +1,89 @@ +# Local federation live test (gateway + sync + OpenSearch) + +End-to-end flow: pull real RF data from CRC → load into **local** gateway → publish for federation → index in `fed-*` → query sync search. + +**Never commit API tokens.** Use `env.example` → copy to `.env` in this directory. + +## Prerequisites + +1. Gateway stack up (`gateway` compose local): app, Redis, OpenSearch, MinIO. +2. Federation sync up (`federation/compose.local.yaml`) on port **8001**. +3. Gateway env (e.g. `django.env`): + - `FEDERATION_ENABLED=true` + - `FEDERATION_SITE_NAME=crc` (short id) + - `SDS_SITE_FQDN=localhost` (must match `federation.toml` `[site].fqdn` for local) + - `FEDERATION_EXPORT_ALLOWED_CIDRS` includes your sync container / dev machine +4. Federation sync env: `FEDERATION_GATEWAY_API_KEY` from: + ```bash + docker compose exec sds-gateway-local-app python manage.py create_federation_sync_api_key + ``` +5. OpenSearch capture indices: `python manage.py init_indices` (gateway). + +## 1. Download from CRC (read-only) + +```bash +cd sdk +cp ../federation/scripts/local_e2e/env.example ../federation/scripts/local_e2e/.env +# Edit .env: SDS_SECRET_TOKEN=... (your user token; rotate if exposed) + +set -a && source ../federation/scripts/local_e2e/.env && set +a +uv run python ../federation/scripts/local_e2e/download_crc_dataset.py \ + --dataset-uuid 50e979bd-8018-415c-8212-c08c3dc98654 \ + --to ../federation/data/downloaded_dataset +``` + +To mimic a Haystack-style folder (one time slice), discover `top_level_dir` on CRC with the SDK (`list_dataset_captures`) then pass `--top-level-dir`. + +**Size warning:** full datasets can be huge; prefer `--top-level-dir` or `--skip-contents` for metadata-only federation tests. + +## 2. Upload to local gateway + +Use a **local** user token (`LOCAL_SDS_SECRET_TOKEN` in `.env`). + +```bash +uv run python ../federation/scripts/local_e2e/upload_capture_to_local.py \ + --local-path ../federation/data/downloaded_dataset \ + --sds-path federation-fixture/starlink-sample +``` + +Note the printed **capture UUID**. + +## 3. Create / publish dataset + +Create a dataset in the UI or API, attach the capture, then: + +```bash +docker compose exec sds-gateway-local-app python manage.py publish_for_federation \ + --dataset-uuid \ + --capture-uuids +``` + +This sets `FINAL` + `is_public` and marks captures public (fires `federation:events` when enabled). + +## 4. Re-index federation + +Restart sync (bootstrap on start) or trigger an update: + +```bash +cd federation +just simulate-redis --event-type updated --item-type dataset --uuid +``` + +## 5. Verify + +```bash +cd federation +uv run python scripts/local_e2e/verify_federation_live.py --q starlink +``` + +Or: + +```bash +curl -s 'http://localhost:8001/api/v1/search/datasets?site=localhost&q=fixture' | jq . +curl -s 'http://localhost:8000/api/v1/federation/export/datasets/' \ + -H "Authorization: Api-Key $FEDERATION_GATEWAY_API_KEY" | jq . +``` + +## Haystack public HTTP (optional) + +Apollo directory listings (e.g. `rf@*.h5` under `Vpol_11.325GHz/...`) are the same kind of DigitalRF files CRC stores. You can `wget`/`curl` a **small** subset into `federation/data/apollo_sample/` and upload with the same `upload_capture_to_local.py` script instead of using `download_dataset`. diff --git a/federation/scripts/local_e2e/download_crc_dataset.py b/federation/scripts/local_e2e/download_crc_dataset.py new file mode 100644 index 000000000..4f810bf82 --- /dev/null +++ b/federation/scripts/local_e2e/download_crc_dataset.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Download a public dataset from CRC (or SDS_HOST) for local federation fixtures. + +Run from the SDK environment (spectrumx is not a federation dependency): + + cd sdk + cp ../federation/scripts/local_e2e/env.example ../federation/scripts/local_e2e/.env + # Edit .env: SDS_SECRET_TOKEN=... + + set -a && source ../federation/scripts/local_e2e/.env && set +a + uv run python ../federation/scripts/local_e2e/download_crc_dataset.py \\ + --dataset-uuid 50e979bd-8018-415c-8212-c08c3dc98654 \\ + --to ./../federation/data/downloaded_dataset + +Or from ``federation/``: ``just local-e2e-env`` then ``just download-crc-dataset ``. + +Use ``--top-level-dir`` to limit to one capture tree (Haystack-style DRF folders). +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path +from uuid import UUID + +from spectrumx import Client +from spectrumx.errors import SDSError + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-uuid", required=True) + parser.add_argument( + "--to", + type=Path, + default=Path("downloaded_dataset"), + help="Local download root", + ) + parser.add_argument( + "--top-level-dir", + default="", + help="Optional SDS top_level_dir filter (one capture)", + ) + parser.add_argument("--overwrite", action="store_true") + parser.add_argument("--skip-contents", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + host = os.environ.get("SDS_HOST", "sds.crc.nd.edu") + client = Client(host=host) + client.dry_run = False + client.authenticate() + + dataset_uuid = UUID(args.dataset_uuid) + top_level_dirs = [args.top_level_dir] if args.top_level_dir.strip() else None + + print(f"Downloading {dataset_uuid} from {host} -> {args.to.resolve()}") + try: + results = client.download_dataset( + dataset_uuid=dataset_uuid, + to_local_path=args.to, + top_level_dirs=top_level_dirs, + skip_contents=args.skip_contents, + overwrite=args.overwrite, + verbose=True, + ) + except SDSError as exc: + print(f"Download failed: {exc}", file=sys.stderr) + return 1 + + ok = sum(1 for r in results if r) + fail = len(results) - ok + print(f"Done: {ok} ok, {fail} failed") + return 0 if fail == 0 else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/federation/scripts/local_e2e/env.example b/federation/scripts/local_e2e/env.example new file mode 100644 index 000000000..769b64bed --- /dev/null +++ b/federation/scripts/local_e2e/env.example @@ -0,0 +1,19 @@ +# Copy to federation/scripts/local_e2e/.env (never commit secrets). +# CRC download (production read-only): +SDS_HOST=sds.crc.nd.edu +SDS_SECRET_TOKEN= + +# Optional: filter download to one capture directory (from list_dataset_captures) +# DATASET_UUID=50e979bd-8018-415c-8212-c08c3dc98654 +# TOP_LEVEL_DIR=/files/you@example.com/federation-fixture/... + +# Local gateway upload (after download): +LOCAL_SDS_HOST=localhost:8000 +LOCAL_SDS_SECRET_TOKEN= + +# Federation sync (gateway export for bootstrap): +FEDERATION_GATEWAY_API_KEY= + +# Live checks +FEDERATION_SYNC_URL=http://localhost:8001 +LOCAL_SITE_FQDN=localhost diff --git a/federation/scripts/local_e2e/upload_capture_to_local.py b/federation/scripts/local_e2e/upload_capture_to_local.py new file mode 100644 index 000000000..4fc3f9d5d --- /dev/null +++ b/federation/scripts/local_e2e/upload_capture_to_local.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Upload a local DigitalRF directory to your local gateway as a new capture. + + cd sdk + cp ../federation/scripts/local_e2e/env.example ../federation/scripts/local_e2e/.env + # Edit .env: LOCAL_SDS_SECRET_TOKEN=... + + set -a && source ../federation/scripts/local_e2e/.env && set +a + uv run python ../federation/scripts/local_e2e/upload_capture_to_local.py \\ + --local-path ../federation/data/downloaded_dataset \\ + --sds-path federation-fixture/starlink-sample + +Prints the new capture UUID for publish_for_federation / dataset linking. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path +from pathlib import PurePosixPath + +from spectrumx import Client +from spectrumx.errors import SDSError +from spectrumx.models.captures import CaptureType + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--local-path", + type=Path, + required=True, + help="Directory containing DRF/HDF5 files (downloaded tree)", + ) + parser.add_argument( + "--sds-path", + default="federation-fixture/sample", + help="Virtual path under your SDS user root", + ) + parser.add_argument("--channel", default="0") + parser.add_argument("--name", default="Federation local fixture capture") + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + host = os.environ.get("LOCAL_SDS_HOST", "localhost:8000") + host = host.removeprefix("https://").removeprefix("http://") + env_config: dict[str, str] = {} + token = os.environ.get("LOCAL_SDS_SECRET_TOKEN") or os.environ.get( + "SDS_SECRET_TOKEN", + "", + ) + if token: + env_config["SDS_SECRET_TOKEN"] = token + + client = Client(host=host, env_config=env_config or None) + client.dry_run = False + client.authenticate() + + if not args.local_path.is_dir(): + print(f"Not a directory: {args.local_path}", file=sys.stderr) + return 1 + + print(f"Uploading {args.local_path} -> {args.sds_path} on {client.host}") + try: + capture = client.upload_capture( + local_path=args.local_path, + sds_path=PurePosixPath(args.sds_path), + capture_type=CaptureType.DigitalRF, + channel=args.channel, + name=args.name, + verbose=True, + ) + except SDSError as exc: + print(f"Upload failed: {exc}", file=sys.stderr) + return 1 + + if capture is None: + print("Upload returned no capture", file=sys.stderr) + return 2 + + print(f"Capture UUID: {capture.uuid}") + print(f"top_level_dir: {capture.top_level_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/federation/scripts/local_e2e/verify_federation_live.py b/federation/scripts/local_e2e/verify_federation_live.py new file mode 100644 index 000000000..a269ad6ba --- /dev/null +++ b/federation/scripts/local_e2e/verify_federation_live.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Smoke-test federation export + search against a running local sync service.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys + +import httpx + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--sync-base", + default=os.environ.get("FEDERATION_SYNC_URL", "http://localhost:8001"), + ) + parser.add_argument( + "--site-fqdn", + default=os.environ.get("LOCAL_SITE_FQDN", "localhost"), + ) + parser.add_argument("--q", default="") + parser.add_argument( + "--gateway-export", + default=os.environ.get( + "GATEWAY_EXPORT_URL", + "http://localhost:8000/api/v1/federation/export/datasets/", + ), + ) + parser.add_argument( + "--api-key", + default=os.environ.get("FEDERATION_GATEWAY_API_KEY", ""), + ) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + base = args.sync_base.rstrip("/") + + with httpx.Client(timeout=30.0) as client: + health = client.get(f"{base}/health") + print(f"health {health.status_code}: {health.text[:200]}") + if health.status_code != 200: + return 1 + + headers = {} + if args.api_key: + headers["Authorization"] = f"Api-Key: {args.api_key}" + export = client.get(args.gateway_export, headers=headers) + print(f"gateway export {export.status_code}") + if export.status_code == 200: + data = export.json() + print(f" export datasets: {len(data)}") + if data: + print(f" first site_name: {data[0].get('site_name')}") + + params = {"site": args.site_fqdn} + if args.q: + params["q"] = args.q + search = client.get( + f"{base}/api/v1/search/datasets", + params=params, + ) + print(f"search datasets {search.status_code}") + if search.status_code == 200: + body = search.json() + print(json.dumps(body, indent=2)[:1500]) + else: + print(search.text, file=sys.stderr) + return 2 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/federation/sds_federation/main.py b/federation/sds_federation/main.py index f8a19dc9e..ed44fc7cd 100644 --- a/federation/sds_federation/main.py +++ b/federation/sds_federation/main.py @@ -11,9 +11,11 @@ from sds_federation.models import load_federation_config from sds_federation.routes.health import health_router +from sds_federation.routes.search_index import search_index_router from sds_federation.routes.webhooks import webhooks_router from sds_federation.services.bootstrap import run_bootstrap from sds_federation.services.fed_index import FederatedAssetIndexer +from sds_federation.services.fed_indices import ensure_fed_indices from sds_federation.services.local_events import build_gateway_http_client from sds_federation.services.local_events import run_federation_subscriber from sds_federation.services.peer_registry import PeerRegistry @@ -32,6 +34,7 @@ def _bootstrap_enabled() -> bool: sync_app = FastAPI(title="SDS Federation Sync") sync_app.include_router(health_router) +sync_app.include_router(search_index_router, prefix=API_PREFIX) sync_app.include_router(webhooks_router, prefix=API_PREFIX) @@ -44,6 +47,11 @@ async def lifespan(app: FastAPI): os_host = os.environ.get("OPENSEARCH_HOST", "opensearch") os_port = os.environ.get("OPENSEARCH_PORT", "9200") os_client = OpenSearch(hosts=[{"host": os_host, "port": int(os_port)}]) + try: + ensure_fed_indices(os_client) + except Exception as exc: # noqa: BLE001 + logger.error("Failed to ensure fed-* OpenSearch indices: {}", exc) + peer_registry = PeerRegistry() fed_indexer = FederatedAssetIndexer(os_client) diff --git a/federation/sds_federation/models.py b/federation/sds_federation/models.py index 8d4165ab0..94c1f09b8 100644 --- a/federation/sds_federation/models.py +++ b/federation/sds_federation/models.py @@ -14,6 +14,11 @@ class SiteInfo(BaseModel): sync_service_url: AnyHttpUrl | None = None +def site_name_for_federation(site: SiteInfo) -> str: + """FQDN stored in export docs and OpenSearch ``site_name`` (RFC §6).""" + return site.fqdn + + class PeerInfo(BaseModel): name: str fqdn: str @@ -31,6 +36,10 @@ class FederationConfig(BaseModel): sync_service_url: AnyHttpUrl +def allowed_federated_origin_fqdns(config: FederationConfig) -> set[str]: + return {peer.fqdn for peer in config.peers} | {config.site.fqdn} + + def load_federation_config() -> FederationConfig: path = Path(os.environ.get("FEDERATION_CONFIG_PATH", "federation.toml")) data = tomllib.loads(path.read_text(encoding="utf-8")) diff --git a/federation/sds_federation/routes/search_index.py b/federation/sds_federation/routes/search_index.py new file mode 100644 index 000000000..cf5a0f989 --- /dev/null +++ b/federation/sds_federation/routes/search_index.py @@ -0,0 +1,103 @@ +"""Federated metadata search (RFC GET /search/datasets, /search/captures).""" + +from __future__ import annotations + +import json +from typing import Any + +from fastapi import APIRouter +from fastapi import HTTPException +from fastapi import Query +from fastapi import Request +from opensearchpy import OpenSearch + +from sds_federation.services.fed_search import search_federated_captures +from sds_federation.services.fed_search import search_federated_datasets + +search_index_router = APIRouter(tags=["search"]) + + +def _opensearch_client(request: Request) -> OpenSearch: + client = getattr( + request.app.state, + "opensearch_client", + None, + ) + if client is None: + raise HTTPException( + status_code=503, + detail="OpenSearch client not ready", + ) + return client + + +def _parse_metadata_filters(raw: str | None) -> list[dict[str, Any]] | None: + if not raw: + return None + try: + parsed = json.loads(raw) + except json.JSONDecodeError as err: + raise HTTPException( + status_code=400, + detail="'metadata_filters' must be valid JSON", + ) from err + if not isinstance(parsed, list): + raise HTTPException( + status_code=400, + detail="'metadata_filters' must be a JSON list", + ) + return parsed + + +@search_index_router.get("/search/datasets") +async def search_datasets( + request: Request, + q: str | None = Query(default=None, description="Free-text search"), + site: str | None = Query( + default="*", + description="Peer site FQDN (federation.toml fqdn) or *", + ), + metadata_filters: str | None = Query( + default=None, + description="JSON list of metadata filter objects", + ), +) -> dict[str, Any]: + client = _opensearch_client(request) + filters = _parse_metadata_filters(metadata_filters) + try: + return search_federated_datasets( + client, + q=q, + site=site, + metadata_filters=filters, + ) + except ValueError as err: + raise HTTPException(status_code=400, detail=str(err)) from err + + +@search_index_router.get("/search/captures") +async def search_captures( + request: Request, + q: str | None = Query(default=None, description="Free-text search"), + site: str | None = Query( + default="*", + description="Peer site FQDN (federation.toml fqdn) or *", + ), + capture_type: str | None = Query(default=None), + metadata_filters: str | None = Query( + default=None, + description="JSON list of metadata filter objects", + ), +) -> dict[str, Any]: + client = _opensearch_client(request) + filters = _parse_metadata_filters(metadata_filters) + try: + return search_federated_captures( + client, + q=q, + site=site, + capture_type=capture_type, + metadata_filters=filters, + ) + except ValueError as err: + raise HTTPException(status_code=400, detail=str(err)) from err diff --git a/federation/sds_federation/routes/webhooks.py b/federation/sds_federation/routes/webhooks.py index 055bca0b4..b0e1ebc16 100644 --- a/federation/sds_federation/routes/webhooks.py +++ b/federation/sds_federation/routes/webhooks.py @@ -8,6 +8,8 @@ from fastapi import Request from loguru import logger +from sds_federation.models import allowed_federated_origin_fqdns +from sds_federation.models import site_name_for_federation from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import AssetUpdatedWebhook from sds_federation.schemas.webhooks import SiteHelloWebhook @@ -32,7 +34,7 @@ def _local_site_name(request: Request) -> str: config = getattr(request.app.state, "config", None) if config is None: raise HTTPException(status_code=503, detail="Config not ready") - return config.site.name + return site_name_for_federation(config.site) def _indexer(request: Request) -> FederatedAssetIndexer: @@ -55,12 +57,13 @@ def _http_client(request: Request) -> httpx.AsyncClient | None: def _allowed_origin_sites(request: Request, payload: AssetUpdatedWebhook) -> None: config = request.app.state.config - if payload.site_name == config.site.name: + if payload.site_name == site_name_for_federation(config.site): raise HTTPException( status_code=403, detail="Local site metadata is not accepted via peer webhooks", ) - allowed = {peer.name for peer in config.peers} + + allowed = allowed_federated_origin_fqdns(config) if payload.site_name not in allowed: raise HTTPException(status_code=403, detail="Unknown origin site") @@ -144,12 +147,12 @@ async def list_captures(request: Request) -> list[dict]: @webhooks_router.post("/webhook/site-hello") async def site_hello(payload: SiteHelloWebhook, request: Request) -> dict: config = request.app.state.config - if payload.site_name == config.site.name: + if payload.site_name == site_name_for_federation(config.site): raise HTTPException( status_code=422, detail="Cannot register self via site-hello", ) - allowed = {peer.name for peer in config.peers} + allowed = {peer.fqdn for peer in config.peers} if payload.site_name not in allowed: raise HTTPException(status_code=403, detail="Unknown registering site") diff --git a/federation/sds_federation/schemas/opensearch_indices.py b/federation/sds_federation/schemas/opensearch_indices.py new file mode 100644 index 000000000..e359056b8 --- /dev/null +++ b/federation/sds_federation/schemas/opensearch_indices.py @@ -0,0 +1,85 @@ +"""OpenSearch index definitions for federated metadata (RFC fed-datasets / fed-captures). + +Explicit ``properties`` match the RFC search-facing field lists; export-only and +envelope fields (e.g. ``status``, ``capture_props``, ``is_federated_deleted``) rely on +dynamic mapping. +""" + +from __future__ import annotations + +from typing import Any + +from sds_federation.schemas.webhooks import AssetTypeEnum + +FED_INDEX_SETTINGS: dict[str, Any] = { + "index": { + "number_of_shards": 1, + "number_of_replicas": 0, + }, +} + +# RFC §6 — fed-datasets search fields +RFC_FED_DATASET_PROPERTIES: dict[str, dict[str, Any]] = { + "uuid": {"type": "keyword"}, + "site_name": {"type": "keyword"}, # peer FQDN (federation.toml [site].fqdn) + "name": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, + "description": {"type": "text"}, + "abstract": {"type": "text"}, + "keywords": {"type": "keyword"}, + "owner_name": {"type": "keyword"}, + "created_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"}, + "updated_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"}, + "size": {"type": "long"}, + "capture_count": {"type": "integer"}, + "url": {"type": "keyword"}, +} + +# RFC §6 — fed-captures search fields +RFC_FED_CAPTURE_PROPERTIES: dict[str, dict[str, Any]] = { + "uuid": {"type": "keyword"}, + "site_name": {"type": "keyword"}, # peer FQDN (federation.toml [site].fqdn) + "capture_type": {"type": "keyword"}, + "channel": {"type": "keyword"}, + "center_frequency": {"type": "double"}, + "sample_rate": {"type": "double"}, + "start_time": {"type": "long"}, + "end_time": {"type": "long"}, + "dataset_ids": {"type": "keyword"}, + "url": {"type": "keyword"}, +} + + +def _fed_mappings(properties: dict[str, dict[str, Any]]) -> dict[str, Any]: + return { + "dynamic": True, + "properties": properties, + } + + +def fed_dataset_mappings() -> dict[str, Any]: + return _fed_mappings(RFC_FED_DATASET_PROPERTIES) + + +def fed_capture_mappings() -> dict[str, Any]: + return _fed_mappings(RFC_FED_CAPTURE_PROPERTIES) + + +def index_body_for_asset(asset_type: AssetTypeEnum) -> dict[str, Any]: + mappings = ( + fed_dataset_mappings() + if asset_type == AssetTypeEnum.DATASET + else fed_capture_mappings() + ) + return { + "settings": FED_INDEX_SETTINGS, + "mappings": mappings, + } + + +def index_body_for_index_name(index_name: str) -> dict[str, Any]: + if index_name == AssetTypeEnum.DATASET.index_name: + return index_body_for_asset(AssetTypeEnum.DATASET) + if index_name == AssetTypeEnum.CAPTURE.index_name: + return index_body_for_asset(AssetTypeEnum.CAPTURE) + msg = f"unknown federated index: {index_name}" + raise ValueError(msg) diff --git a/federation/sds_federation/schemas/webhooks.py b/federation/sds_federation/schemas/webhooks.py index 24d236642..2b3132bfc 100644 --- a/federation/sds_federation/schemas/webhooks.py +++ b/federation/sds_federation/schemas/webhooks.py @@ -57,7 +57,7 @@ class FederatedDatasetDoc(BaseModel): is_public: bool = False owner_name: str = "" updated_at: str | None = None - site_name: str + site_name: str # peer site FQDN (RFC §6), not federation.toml [site].name size: int = 0 capture_count: int = 0 capture_file_count: int = 0 @@ -79,7 +79,7 @@ class FederatedCaptureDoc(BaseModel): top_level_dir: str = "" created_at: str | None = None updated_at: str | None = None - site_name: str + site_name: str # peer site FQDN (RFC §6), not federation.toml [site].name file_count: int = 0 size: int = 0 capture_props: dict[str, Any] = Field(default_factory=dict) @@ -108,7 +108,7 @@ class SiteHelloWebhook(BaseModel): model_config = ConfigDict(extra="forbid") - site_name: str + site_name: str # registering peer FQDN (same as fqdn) fqdn: str display_name: str = "" sync_service_url: AnyHttpUrl diff --git a/federation/sds_federation/services/bootstrap.py b/federation/sds_federation/services/bootstrap.py index ce0b64010..bd6aad2de 100644 --- a/federation/sds_federation/services/bootstrap.py +++ b/federation/sds_federation/services/bootstrap.py @@ -12,6 +12,7 @@ from sds_federation.models import FederationConfig from sds_federation.models import PeerInfo +from sds_federation.models import site_name_for_federation from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import FederatedCaptureDoc from sds_federation.schemas.webhooks import FederatedDatasetDoc @@ -211,14 +212,15 @@ def _index_export_docs( ) -> int: indexed = 0 for doc in docs: - if doc.site_name != peer.name: + if doc.site_name not in {peer.name, peer.fqdn}: logger.error( "bootstrap export failed for {} {}: site name mismatch " - "(doc.site_name={!r}, peer.name={!r})", + "(doc.site_name={!r}, peer.name={!r}, peer.fqdn={!r})", peer.name, asset_type.value, doc.site_name, peer.name, + peer.fqdn, ) continue @@ -353,7 +355,7 @@ async def bootstrap_all_peers( def peer_by_name(config: FederationConfig, site_name: str) -> PeerInfo | None: for peer in config.peers: - if peer.name == site_name: + if peer.name == site_name or peer.fqdn == site_name: return peer return None @@ -376,7 +378,7 @@ async def backfill_peer_on_hello( def _site_hello_payload(config: FederationConfig) -> SiteHelloWebhook: return SiteHelloWebhook( - site_name=config.site.name, + site_name=site_name_for_federation(config.site), fqdn=config.site.fqdn, display_name=config.site.display_name, sync_service_url=config.sync_service_url, diff --git a/federation/sds_federation/services/fed_indices.py b/federation/sds_federation/services/fed_indices.py new file mode 100644 index 000000000..c60a35550 --- /dev/null +++ b/federation/sds_federation/services/fed_indices.py @@ -0,0 +1,19 @@ +"""Create fed-datasets / fed-captures OpenSearch indices if missing.""" + +from __future__ import annotations + +from opensearchpy import OpenSearch + +from sds_federation.schemas.opensearch_indices import index_body_for_asset +from sds_federation.schemas.webhooks import AssetTypeEnum + + +def ensure_fed_indices(client: OpenSearch) -> None: + for asset_type in AssetTypeEnum: + index_name = asset_type.index_name + if client.indices.exists(index=index_name): + continue + client.indices.create( + index=index_name, + body=index_body_for_asset(asset_type), + ) diff --git a/federation/sds_federation/services/fed_search.py b/federation/sds_federation/services/fed_search.py index 65d6b4951..a8086ac6f 100644 --- a/federation/sds_federation/services/fed_search.py +++ b/federation/sds_federation/services/fed_search.py @@ -1,24 +1,31 @@ -"""Read federated documents from shared fed-* OpenSearch indices.""" +"""Read and search federated documents in shared fed-* OpenSearch indices.""" from __future__ import annotations import asyncio -from typing import TYPE_CHECKING +from collections.abc import Mapping from typing import Any +from uuid import UUID +from opensearchpy import OpenSearch from opensearchpy.exceptions import NotFoundError +from sds_opensearch_query import bool_must_search_body +from sds_opensearch_query import build_metadata_filter_clauses +from sds_opensearch_query import federation_not_deleted_clause +from sds_opensearch_query import flatten_property_paths +from sds_opensearch_query import multi_match_clause +from sds_opensearch_query import run_search +from sds_opensearch_query import term_clause +from sds_opensearch_query.query import DEFAULT_SEARCH_SIZE +from sds_federation.schemas.opensearch_indices import RFC_FED_CAPTURE_PROPERTIES +from sds_federation.schemas.opensearch_indices import RFC_FED_DATASET_PROPERTIES from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import FederatedCaptureDoc from sds_federation.schemas.webhooks import FederatedDatasetDoc from sds_federation.schemas.webhooks import asset_doc_class from sds_federation.services.fed_index import doc_id -if TYPE_CHECKING: - from uuid import UUID - - from opensearchpy import OpenSearch - _FEDERATION_META_KEYS = frozenset({"federation_event_at"}) _LIST_PAGE_SIZE = 1000 @@ -150,3 +157,130 @@ async def alist_federated_assets_for_site( site_name=site_name, asset_type=asset_type, ) + +FED_DATASET_TEXT_FIELDS = [ + "name^2", + "description", + "abstract", + "keywords", + "owner_name", +] + +FED_CAPTURE_TEXT_FIELDS = [ + "name", + "channel", + "capture_type", +] + +_WILDCARD_SITES = frozenset({"", "*", "all"}) + + +def _site_clause(site: str | None) -> dict[str, Any] | None: + if site is None or site.strip().lower() in _WILDCARD_SITES: + return None + return term_clause("site_name", site.strip()) + + +def _text_clause(q: str | None, fields: list[str]) -> dict[str, Any] | None: + if q is None or not q.strip(): + return None + return multi_match_clause(q.strip(), fields) + + +def _hits_to_response(hits: list[dict[str, Any]]) -> dict[str, Any]: + return { + "total": len(hits), + "hits": [ + { + "id": hit["_id"], + "score": hit.get("_score"), + "source": hit.get("_source", {}), + } + for hit in hits + ], + } + + +def _build_fed_must_clauses( + *, + q: str | None, + site: str | None, + metadata_filters: list[dict[str, Any]] | None, + rfc_properties: Mapping[str, dict[str, Any]], + text_fields: list[str], + extra_terms: list[tuple[str, Any]] | None = None, +) -> list[dict[str, Any]]: + known = flatten_property_paths(rfc_properties) + must: list[dict[str, Any]] = [federation_not_deleted_clause()] + + site_filter = _site_clause(site) + if site_filter is not None: + must.append(site_filter) + + for field, value in extra_terms or (): + if value is not None and value != "": + must.append(term_clause(field, value)) + + text = _text_clause(q, text_fields) + if text is not None: + must.append(text) + + must.extend( + build_metadata_filter_clauses( + metadata_filters, + known_field_paths=known, + ), + ) + return must + + +def search_federated_datasets( + client: OpenSearch, + *, + q: str | None = None, + site: str | None = None, + metadata_filters: list[dict[str, Any]] | None = None, + size: int = DEFAULT_SEARCH_SIZE, +) -> dict[str, Any]: + must = _build_fed_must_clauses( + q=q, + site=site, + metadata_filters=metadata_filters, + rfc_properties=RFC_FED_DATASET_PROPERTIES, + text_fields=FED_DATASET_TEXT_FIELDS, + ) + body = bool_must_search_body(*must) + hits = run_search( + client, + index=AssetTypeEnum.DATASET.index_name, + body=body, + size=size, + ) + return _hits_to_response(hits) + + +def search_federated_captures( + client: OpenSearch, + *, + q: str | None = None, + site: str | None = None, + metadata_filters: list[dict[str, Any]] | None = None, + capture_type: str | None = None, + size: int = DEFAULT_SEARCH_SIZE, +) -> dict[str, Any]: + must = _build_fed_must_clauses( + q=q, + site=site, + metadata_filters=metadata_filters, + rfc_properties=RFC_FED_CAPTURE_PROPERTIES, + text_fields=FED_CAPTURE_TEXT_FIELDS, + extra_terms=[("capture_type", capture_type)], + ) + body = bool_must_search_body(*must) + hits = run_search( + client, + index=AssetTypeEnum.CAPTURE.index_name, + body=body, + size=size, + ) + return _hits_to_response(hits) diff --git a/federation/sds_federation/services/local_events.py b/federation/sds_federation/services/local_events.py index 9f9abb94b..5bfa75663 100644 --- a/federation/sds_federation/services/local_events.py +++ b/federation/sds_federation/services/local_events.py @@ -11,6 +11,7 @@ from opensearchpy import OpenSearch from sds_federation.models import FederationConfig +from sds_federation.models import site_name_for_federation from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import AssetUpdatedWebhook from sds_federation.schemas.webhooks import FederatedCaptureDoc @@ -48,7 +49,7 @@ async def _default_load_asset( ) -> FederatedDatasetDoc | FederatedCaptureDoc | None: return await aload_federated_asset( os_client, - site_name=config.site.name, + site_name=site_name_for_federation(config.site), uuid=uuid, asset_type=asset_type, ) @@ -95,7 +96,7 @@ async def handle_redis_asset_event( payload = AssetUpdatedWebhook( timestamp=timestamp, - site_name=config.site.name, + site_name=site_name_for_federation(config.site), asset=asset, asset_type=asset_type, ) diff --git a/federation/sds_federation/testing/sample_data.py b/federation/sds_federation/testing/sample_data.py index d05a515b8..4d51ca792 100644 --- a/federation/sds_federation/testing/sample_data.py +++ b/federation/sds_federation/testing/sample_data.py @@ -31,7 +31,7 @@ def simulated_dataset_redis_payload( def sample_federated_dataset_doc( *, uuid: UUID = TEST_DATASET_UUID, - site_name: str = "testsite", + site_name: str = "localhost", ) -> FederatedDatasetDoc: return FederatedDatasetDoc( uuid=uuid, @@ -47,7 +47,7 @@ def sample_federated_dataset_doc( def sample_federated_capture_doc( *, uuid: UUID = TEST_CAPTURE_UUID, - site_name: str = "testsite", + site_name: str = "localhost", ) -> FederatedCaptureDoc: return FederatedCaptureDoc( uuid=uuid, diff --git a/federation/tests/conftest.py b/federation/tests/conftest.py index 035e82f39..0969b1c1d 100644 --- a/federation/tests/conftest.py +++ b/federation/tests/conftest.py @@ -8,6 +8,7 @@ import pytest from fastapi import FastAPI from sds_federation.models import FederationConfig +from sds_federation.models import site_name_for_federation from sds_federation.routes.webhooks import webhooks_router from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.services.fed_index import FederatedAssetIndexer @@ -97,7 +98,7 @@ def stub_dataset_resolver(test_site_config: FederationConfig): def _seed(recording_opensearch: RecordingOpenSearch) -> None: seed_federated_dataset_in_opensearch( recording_opensearch, - test_site_config.site.name, + site_name_for_federation(test_site_config.site), ) return _seed diff --git a/federation/tests/support/federation_mesh.py b/federation/tests/support/federation_mesh.py index f8da5741e..49b19fe83 100644 --- a/federation/tests/support/federation_mesh.py +++ b/federation/tests/support/federation_mesh.py @@ -20,6 +20,9 @@ API_PREFIX = "/api/v1" +TESTSITE_FQDN = "localhost" +PEER_ONE_FQDN = "peer.test" + TESTSITE_SYNC_ORIGIN = "http://testsite.test" PEER_ONE_SYNC_ORIGIN = "http://peer-one.test" TESTSITE_GATEWAY_HOST = "testsite-gateway.test" diff --git a/federation/tests/test_integration_bootstrap.py b/federation/tests/test_integration_bootstrap.py index de3c5bf0c..bb7df8d45 100644 --- a/federation/tests/test_integration_bootstrap.py +++ b/federation/tests/test_integration_bootstrap.py @@ -106,7 +106,7 @@ async def test_push_site_hello_to_in_process_peer( result = await push_site_hello_to_peer(http, peer, caller_config) assert result["status"] == "registered" - assert app.state.peer_registry.get("testsite") is not None + assert app.state.peer_registry.get("localhost") is not None @pytest.mark.integration @@ -115,8 +115,8 @@ async def test_run_bootstrap_pulls_exports_then_registers( recording_opensearch: RecordingOpenSearch, monkeypatch: pytest.MonkeyPatch, ) -> None: - local_doc = sample_federated_dataset_doc(site_name="testsite") - peer_doc = sample_federated_dataset_doc(site_name="peer-one") + local_doc = sample_federated_dataset_doc(site_name="localhost") + peer_doc = sample_federated_dataset_doc(site_name="peer.test") hello_posts: list[str] = [] def handler(request: httpx.Request) -> httpx.Response: # noqa: PLR0911 @@ -186,7 +186,7 @@ def handler(request: httpx.Request) -> httpx.Response: # noqa: PLR0911 assert len(recording_opensearch.index_calls) == 2 indexed_ids = {call["id"] for call in recording_opensearch.index_calls} assert indexed_ids == { - doc_id("testsite", TEST_DATASET_UUID), - doc_id("peer-one", TEST_DATASET_UUID), + doc_id("localhost", TEST_DATASET_UUID), + doc_id("peer.test", TEST_DATASET_UUID), } assert len(hello_posts) == 1 diff --git a/federation/tests/test_integration_mesh.py b/federation/tests/test_integration_mesh.py index 182f289a8..cf736fece 100644 --- a/federation/tests/test_integration_mesh.py +++ b/federation/tests/test_integration_mesh.py @@ -16,6 +16,8 @@ from sds_federation.services.fed_index import FederatedAssetIndexer from sds_federation.services.fed_index import doc_id from sds_federation.services.local_events import dispatch_federation_redis_payload +from sds_federation.tests.support.federation_mesh import PEER_ONE_FQDN +from sds_federation.tests.support.federation_mesh import TESTSITE_FQDN from sds_federation.testing.sample_data import TEST_DATASET_UUID from sds_federation.testing.sample_data import sample_federated_dataset_doc from sds_federation.testing.sample_data import simulated_dataset_redis_payload @@ -49,13 +51,13 @@ async def test_mesh_dispatches_redis_event_to_peer_opensearch( peer = mesh.site("peer-one") assert len(peer.opensearch.index_calls) == 1 assert peer.opensearch.index_calls[0]["id"] == doc_id( - "testsite", + TESTSITE_FQDN, TEST_DATASET_UUID, ) assert len(mesh.recorded_webhooks) == 1 body = json.loads(mesh.recorded_webhooks[0].content.decode()) webhook = AssetUpdatedWebhook.model_validate(body) - assert webhook.site_name == "testsite" + assert webhook.site_name == TESTSITE_FQDN assert webhook.asset is not None @@ -73,7 +75,7 @@ async def test_mesh_site_hello_registers_on_peer_registry( caller.config, ) assert result["status"] == "registered" - assert peer.registry.get("testsite") is not None + assert peer.registry.get(TESTSITE_FQDN) is not None assert len(peer.registry.registration_events) == 1 @@ -83,12 +85,12 @@ async def test_mesh_site_hello_backfills_caller_exports( two_site_mesh: FederationMesh, ) -> None: mesh = two_site_mesh - doc = sample_federated_dataset_doc(site_name="testsite") + doc = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) caller = mesh.site("testsite") peer = mesh.site("peer-one") FederatedAssetIndexer(caller.opensearch).apply_asset_event( event_at=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), - site_name="testsite", + site_name=TESTSITE_FQDN, asset=doc, asset_type=AssetTypeEnum.DATASET, ) @@ -100,9 +102,9 @@ async def test_mesh_site_hello_backfills_caller_exports( ) assert result["status"] == "registered" - assert peer.registry.get("testsite") is not None + assert peer.registry.get(TESTSITE_FQDN) is not None assert any( - call["id"] == doc_id("testsite", TEST_DATASET_UUID) + call["id"] == doc_id(TESTSITE_FQDN, TEST_DATASET_UUID) for call in peer.opensearch.index_calls ) @@ -115,8 +117,8 @@ async def test_mesh_register_with_peers_both_directions( mesh = two_site_mesh await register_with_peers(mesh.http, mesh.site("testsite").config) await register_with_peers(mesh.http, mesh.site("peer-one").config) - assert mesh.site("peer-one").registry.get("testsite") is not None - assert mesh.site("testsite").registry.get("peer-one") is not None + assert mesh.site("peer-one").registry.get(TESTSITE_FQDN) is not None + assert mesh.site("testsite").registry.get(PEER_ONE_FQDN) is not None @pytest.mark.integration @@ -125,11 +127,11 @@ async def test_mesh_bootstrap_pulls_peer_sync_list( two_site_mesh: FederationMesh, ) -> None: mesh = two_site_mesh - doc = sample_federated_dataset_doc(site_name="testsite") + doc = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) caller = mesh.site("testsite") FederatedAssetIndexer(caller.opensearch).apply_asset_event( event_at=datetime(2026, 6, 11, 11, 0, 0, tzinfo=UTC), - site_name="testsite", + site_name=TESTSITE_FQDN, asset=doc, asset_type=AssetTypeEnum.DATASET, ) @@ -144,6 +146,6 @@ async def test_mesh_bootstrap_pulls_peer_sync_list( ) assert count == 1 assert peer.opensearch.index_calls[-1]["id"] == doc_id( - "testsite", + TESTSITE_FQDN, TEST_DATASET_UUID, ) diff --git a/federation/tests/test_integration_pipeline.py b/federation/tests/test_integration_pipeline.py index c229b4c56..22ab3b317 100644 --- a/federation/tests/test_integration_pipeline.py +++ b/federation/tests/test_integration_pipeline.py @@ -13,6 +13,7 @@ from sds_federation.testing.sample_data import simulated_dataset_redis_payload from tests.conftest import PEER_SYNC_BASE +from tests.support.federation_mesh import TESTSITE_FQDN from tests.support.mock_opensearch import RecordingOpenSearch @@ -45,7 +46,7 @@ async def test_redis_simulation_end_to_end_indexes_on_peer( assert len(local_opensearch.index_calls) == local_calls_before assert len(recording_opensearch.index_calls) == 1 assert recording_opensearch.index_calls[0]["id"] == doc_id( - "testsite", + TESTSITE_FQDN, TEST_DATASET_UUID, ) assert ( diff --git a/federation/tests/test_integration_webhooks.py b/federation/tests/test_integration_webhooks.py index a1599e81a..2fd76a87b 100644 --- a/federation/tests/test_integration_webhooks.py +++ b/federation/tests/test_integration_webhooks.py @@ -22,8 +22,10 @@ if TYPE_CHECKING: from tests.support.mock_opensearch import RecordingOpenSearch +TESTSITE_FQDN = "localhost" -def _dataset_webhook_payload(*, site_name: str = "testsite") -> dict: + +def _dataset_webhook_payload(*, site_name: str = TESTSITE_FQDN) -> dict: asset = sample_federated_dataset_doc(site_name=site_name) webhook = AssetUpdatedWebhook( timestamp=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), @@ -48,7 +50,7 @@ async def test_dataset_webhook_indexes_via_http( ) as client: response = await client.post( f"{SYNC_API_PREFIX}/webhook/dataset-updated", - json=_dataset_webhook_payload(site_name="testsite"), + json=_dataset_webhook_payload(), ) assert response.status_code == 200 @@ -58,7 +60,7 @@ async def test_dataset_webhook_indexes_via_http( recording_opensearch.index_calls[0]["index"] == AssetTypeEnum.DATASET.index_name ) assert recording_opensearch.index_calls[0]["id"] == doc_id( - "testsite", + TESTSITE_FQDN, TEST_DATASET_UUID, ) @@ -112,7 +114,7 @@ async def test_webhook_rejects_site_name_mismatch_on_asset( ) -> None: config = make_peer_config() app = build_webhook_app(config, FederatedAssetIndexer(recording_opensearch)) - body = _dataset_webhook_payload(site_name="testsite") + body = _dataset_webhook_payload() body["asset"]["site_name"] = "mismatch" async with httpx.AsyncClient( @@ -136,8 +138,8 @@ async def test_site_hello_registers_known_peer( config = make_peer_config() app = build_webhook_app(config, FederatedAssetIndexer(recording_opensearch)) body = { - "site_name": "testsite", - "fqdn": "localhost", + "site_name": TESTSITE_FQDN, + "fqdn": TESTSITE_FQDN, "display_name": "Originating test site", "sync_service_url": "http://testsite.test/sync", } @@ -149,8 +151,8 @@ async def test_site_hello_registers_known_peer( response = await client.post(f"{SYNC_API_PREFIX}/webhook/site-hello", json=body) assert response.status_code == 200 - assert response.json() == {"status": "registered", "site_name": "testsite"} - assert app.state.peer_registry.get("testsite") is not None + assert response.json() == {"status": "registered", "site_name": TESTSITE_FQDN} + assert app.state.peer_registry.get(TESTSITE_FQDN) is not None @pytest.mark.integration @@ -240,7 +242,7 @@ async def test_site_hello_rejects_self_registration( config = make_peer_config() app = build_webhook_app(config, FederatedAssetIndexer(recording_opensearch)) body = { - "site_name": "peer-one", + "site_name": "peer.test", "fqdn": "peer.test", "sync_service_url": "http://peer-one.test/sync", } diff --git a/federation/tests/test_redis_event_pipeline.py b/federation/tests/test_redis_event_pipeline.py index 6c0a64e17..7fba7d8a1 100644 --- a/federation/tests/test_redis_event_pipeline.py +++ b/federation/tests/test_redis_event_pipeline.py @@ -20,6 +20,7 @@ from sds_federation.testing.sample_data import simulated_dataset_redis_payload from tests.conftest import seed_federated_dataset_in_opensearch +from tests.support.federation_mesh import TESTSITE_FQDN @pytest.mark.asyncio @@ -57,7 +58,7 @@ async def test_dispatch_reads_opensearch_and_posts_webhook_to_peer( recorded, transport = peer_webhook_recorder seed_federated_dataset_in_opensearch( recording_opensearch, - test_site_config.site.name, + TESTSITE_FQDN, ) calls_before = len(recording_opensearch.index_calls) indexer = FederatedAssetIndexer(recording_opensearch) @@ -82,7 +83,7 @@ async def test_dispatch_reads_opensearch_and_posts_webhook_to_peer( ) assert str(req.url) == expected_url body = json.loads(req.content.decode()) - assert body["site_name"] == "testsite" + assert body["site_name"] == TESTSITE_FQDN assert body["asset_type"] == "dataset" assert body["asset"]["uuid"] == str(TEST_DATASET_UUID) assert body["asset"]["name"] == "Simulated public dataset" @@ -97,11 +98,11 @@ async def test_dispatch_deleted_doc_from_opensearch( ) -> None: recorded, transport = peer_webhook_recorder doc = sample_federated_dataset_doc( - site_name=test_site_config.site.name, + site_name=TESTSITE_FQDN, ).model_copy(update={"is_deleted": True}) FederatedAssetIndexer(recording_opensearch).apply_asset_event( event_at=datetime.now(UTC), - site_name=test_site_config.site.name, + site_name=TESTSITE_FQDN, asset=doc, asset_type=AssetTypeEnum.DATASET, ) @@ -161,7 +162,7 @@ def handler(request: httpx.Request) -> httpx.Response: seed_federated_dataset_in_opensearch( recording_opensearch, - test_site_config.site.name, + TESTSITE_FQDN, ) peer = test_site_config.peers[0] registry = PeerRegistry() diff --git a/federation/tests/test_regression_fed_indices.py b/federation/tests/test_regression_fed_indices.py new file mode 100644 index 000000000..82463468e --- /dev/null +++ b/federation/tests/test_regression_fed_indices.py @@ -0,0 +1,39 @@ +"""Regression: ensure_fed_indices creates both fed-* indices.""" + +from __future__ import annotations + +from typing import Any + +import pytest +from sds_federation.schemas.webhooks import AssetTypeEnum +from sds_federation.services.fed_indices import ensure_fed_indices + + +class _IndicesStub: + def __init__(self) -> None: + self.existing: set[str] = set() + self.create_calls: list[dict[str, Any]] = [] + + def exists(self, *, index: str, **kwargs: Any) -> bool: # noqa: ARG002 + return index in self.existing + + def create(self, *, index: str, body: dict[str, Any], **kwargs: Any) -> dict: # noqa: ARG002 + self.create_calls.append({"index": index, "body": body}) + self.existing.add(index) + return {"acknowledged": True} + + +class _ClientStub: + def __init__(self) -> None: + self.indices = _IndicesStub() + + +@pytest.mark.regression +def test_ensure_fed_indices_creates_missing_indices() -> None: + client = _ClientStub() + ensure_fed_indices(client) # type: ignore[arg-type] + names = {c["index"] for c in client.indices.create_calls} + assert names == { + AssetTypeEnum.DATASET.index_name, + AssetTypeEnum.CAPTURE.index_name, + } diff --git a/federation/tests/test_regression_fed_search.py b/federation/tests/test_regression_fed_search.py index 7ab67330c..167428e22 100644 --- a/federation/tests/test_regression_fed_search.py +++ b/federation/tests/test_regression_fed_search.py @@ -1,22 +1,52 @@ -"""Regression tests for fed_search OpenSearch reads.""" +"""Regression: federated search query assembly and OpenSearch document reads.""" from __future__ import annotations from datetime import UTC from datetime import datetime +import pytest from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.services.fed_index import FederatedAssetIndexer from sds_federation.services.fed_search import load_federated_asset +from sds_federation.services.fed_search import search_federated_datasets from sds_federation.testing.sample_data import TEST_DATASET_UUID from sds_federation.testing.sample_data import sample_federated_dataset_doc from tests.support.mock_opensearch import RecordingOpenSearch +pytest.importorskip("sds_opensearch_query") + +@pytest.mark.regression +def test_search_federated_datasets_builds_bool_query() -> None: + client = RecordingOpenSearch() + result = search_federated_datasets( + client, + q="rf survey", + site="sds.crc.nd.edu", + metadata_filters=[ + { + "field_path": "capture_count", + "query_type": "range", + "filter_value": {"gte": 1}, + }, + ], + ) + assert result["total"] == 0 + assert client.search_calls + call = client.search_calls[0] + assert call["index"] == "fed-datasets" + must = call["body"]["query"]["bool"]["must"] + assert {"term": {"is_federated_deleted": False}} in must + assert {"term": {"site_name": "sds.crc.nd.edu"}} in must + assert any("multi_match" in clause for clause in must) + + +@pytest.mark.regression def test_load_federated_asset_returns_indexed_doc() -> None: opensearch = RecordingOpenSearch() - site = "testsite" + site = "localhost" doc = sample_federated_dataset_doc(site_name=site) FederatedAssetIndexer(opensearch).apply_asset_event( event_at=datetime.now(UTC), @@ -37,12 +67,13 @@ def test_load_federated_asset_returns_indexed_doc() -> None: assert loaded.uuid == TEST_DATASET_UUID +@pytest.mark.regression def test_load_federated_asset_missing_returns_none() -> None: opensearch = RecordingOpenSearch() assert ( load_federated_asset( opensearch, - site_name="testsite", + site_name="localhost", uuid=TEST_DATASET_UUID, asset_type=AssetTypeEnum.DATASET, ) diff --git a/federation/tests/test_regression_indexer.py b/federation/tests/test_regression_indexer.py index 08435ac11..1c6fe9f9b 100644 --- a/federation/tests/test_regression_indexer.py +++ b/federation/tests/test_regression_indexer.py @@ -14,13 +14,17 @@ from sds_federation.testing.sample_data import TEST_DATASET_UUID from sds_federation.testing.sample_data import sample_federated_dataset_doc +from tests.support.federation_mesh import TESTSITE_FQDN + if TYPE_CHECKING: from tests.support.mock_opensearch import RecordingOpenSearch @pytest.mark.regression def test_doc_id_format() -> None: - assert doc_id("crc", TEST_DATASET_UUID) == f"crc:{TEST_DATASET_UUID}" + assert doc_id("sds.crc.nd.edu", TEST_DATASET_UUID) == ( + f"sds.crc.nd.edu:{TEST_DATASET_UUID}" + ) @pytest.mark.regression @@ -29,11 +33,11 @@ def test_indexer_writes_dataset_document( ) -> None: indexer = FederatedAssetIndexer(recording_opensearch) event_at = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) - asset = sample_federated_dataset_doc(site_name="testsite") + asset = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) indexer.apply_asset_event( event_at=event_at, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -41,7 +45,7 @@ def test_indexer_writes_dataset_document( assert len(recording_opensearch.index_calls) == 1 call = recording_opensearch.index_calls[0] assert call["index"] == AssetTypeEnum.DATASET.index_name - assert call["id"] == doc_id("testsite", TEST_DATASET_UUID) + assert call["id"] == doc_id(TESTSITE_FQDN, TEST_DATASET_UUID) assert call["body"]["federation_event_at"] == event_at.isoformat() assert call["body"]["name"] == "Simulated public dataset" assert call["body"]["is_deleted"] is False @@ -53,13 +57,13 @@ def test_indexer_writes_deleted_flag_from_asset_body( ) -> None: indexer = FederatedAssetIndexer(recording_opensearch) event_at = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) - asset = sample_federated_dataset_doc(site_name="testsite").model_copy( + asset = sample_federated_dataset_doc(site_name=TESTSITE_FQDN).model_copy( update={"is_deleted": True}, ) indexer.apply_asset_event( event_at=event_at, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -72,19 +76,19 @@ def test_indexer_writes_deleted_flag_from_asset_body( @pytest.mark.regression def test_indexer_skips_stale_events(recording_opensearch: RecordingOpenSearch) -> None: indexer = FederatedAssetIndexer(recording_opensearch) - asset = sample_federated_dataset_doc(site_name="testsite") + asset = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) t1 = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) t0 = t1 - timedelta(seconds=1) indexer.apply_asset_event( event_at=t1, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) indexer.apply_asset_event( event_at=t0, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -96,20 +100,20 @@ def test_indexer_skips_stale_events(recording_opensearch: RecordingOpenSearch) - def test_indexer_skips_stale_events_after_restart( recording_opensearch: RecordingOpenSearch, ) -> None: - asset = sample_federated_dataset_doc(site_name="testsite") + asset = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) t1 = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) t0 = t1 - timedelta(seconds=1) FederatedAssetIndexer(recording_opensearch).apply_asset_event( event_at=t1, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) # New process: empty in-memory map; persisted federation_event_at must win. FederatedAssetIndexer(recording_opensearch).apply_asset_event( event_at=t0, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -127,7 +131,7 @@ def test_indexer_rejects_site_name_mismatch( with pytest.raises(ValueError, match="site_name must match"): indexer.apply_asset_event( event_at=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) diff --git a/federation/tests/test_regression_opensearch_indices.py b/federation/tests/test_regression_opensearch_indices.py new file mode 100644 index 000000000..40df8785d --- /dev/null +++ b/federation/tests/test_regression_opensearch_indices.py @@ -0,0 +1,61 @@ +"""Regression: OpenSearch fed-* mappings include RFC §6 search fields.""" + +from __future__ import annotations + +import pytest +from sds_federation.schemas.opensearch_indices import RFC_FED_CAPTURE_PROPERTIES +from sds_federation.schemas.opensearch_indices import RFC_FED_DATASET_PROPERTIES +from sds_federation.schemas.opensearch_indices import fed_capture_mappings +from sds_federation.schemas.opensearch_indices import fed_dataset_mappings + +RFC_DATASET_FIELDS = frozenset( + { + "uuid", + "site_name", + "name", + "description", + "abstract", + "keywords", + "owner_name", + "created_at", + "updated_at", + "size", + "capture_count", + "url", + }, +) + +RFC_CAPTURE_FIELDS = frozenset( + { + "uuid", + "site_name", + "capture_type", + "channel", + "center_frequency", + "sample_rate", + "start_time", + "end_time", + "dataset_ids", + "url", + }, +) + + +@pytest.mark.regression +def test_rfc_dataset_fields_explicit_in_mapping() -> None: + assert set(RFC_FED_DATASET_PROPERTIES) == RFC_DATASET_FIELDS + props = fed_dataset_mappings()["properties"] + assert set(props) == RFC_DATASET_FIELDS + + +@pytest.mark.regression +def test_rfc_capture_fields_explicit_in_mapping() -> None: + assert set(RFC_FED_CAPTURE_PROPERTIES) == RFC_CAPTURE_FIELDS + props = fed_capture_mappings()["properties"] + assert set(props) == RFC_CAPTURE_FIELDS + + +@pytest.mark.regression +def test_fed_mappings_allow_dynamic_extra_fields() -> None: + assert fed_dataset_mappings()["dynamic"] is True + assert fed_capture_mappings()["dynamic"] is True diff --git a/federation/tests/test_regression_opensearch_query.py b/federation/tests/test_regression_opensearch_query.py new file mode 100644 index 000000000..50f6aa00b --- /dev/null +++ b/federation/tests/test_regression_opensearch_query.py @@ -0,0 +1,49 @@ +"""Regression tests for shared sds_opensearch_query package.""" + +from __future__ import annotations + +import pytest +from sds_opensearch_query import bool_must_search_body +from sds_opensearch_query import build_metadata_filter_clauses +from sds_opensearch_query import nested_query_clause + +pytest.importorskip("sds_opensearch_query") + + +@pytest.mark.regression +def test_nested_query_clause_single_level() -> None: + clause = nested_query_clause( + field_path="center_frequency", + query_type="term", + value=1.0, + levels_nested=0, + last_path="search_props", + ) + assert clause == {"term": {"search_props.center_frequency": 1.0}} + + +@pytest.mark.regression +def test_build_metadata_filter_clauses_term_and_nested() -> None: + clauses = build_metadata_filter_clauses( + [ + { + "field_path": "capture_type", + "query_type": "term", + "filter_value": "digital-rf", + }, + { + "field_path": "search_props.center_frequency", + "query_type": "range", + "filter_value": {"gte": 100}, + }, + ], + ) + assert len(clauses) == 2 + assert clauses[0] == {"term": {"capture_type": "digital-rf"}} + assert "nested" in clauses[1] + + +@pytest.mark.regression +def test_bool_must_search_body() -> None: + body = bool_must_search_body({"term": {"site_name": "crc"}}) + assert body["query"]["bool"]["must"] == [{"term": {"site_name": "crc"}}] diff --git a/federation/tests/test_regression_schemas.py b/federation/tests/test_regression_schemas.py index 85ab1fef4..5ab81bcfa 100644 --- a/federation/tests/test_regression_schemas.py +++ b/federation/tests/test_regression_schemas.py @@ -25,12 +25,15 @@ def test_federated_dataset_doc_rejects_unknown_fields() -> None: FederatedDatasetDoc.model_validate(base) +CRC_FQDN = "sds.crc.nd.edu" + + @pytest.mark.regression def test_asset_updated_webhook_round_trip_dataset() -> None: - asset = sample_federated_dataset_doc(site_name="crc") + asset = sample_federated_dataset_doc(site_name=CRC_FQDN) payload = AssetUpdatedWebhook( timestamp=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), - site_name="crc", + site_name=CRC_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -41,10 +44,10 @@ def test_asset_updated_webhook_round_trip_dataset() -> None: @pytest.mark.regression def test_asset_updated_webhook_round_trip_capture() -> None: - asset = sample_federated_capture_doc(site_name="crc") + asset = sample_federated_capture_doc(site_name=CRC_FQDN) payload = AssetUpdatedWebhook( timestamp=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), - site_name="crc", + site_name=CRC_FQDN, asset=asset, asset_type=AssetTypeEnum.CAPTURE, ) diff --git a/federation/uv.lock b/federation/uv.lock index 3f2e50b0c..16efa70f2 100644 --- a/federation/uv.lock +++ b/federation/uv.lock @@ -554,6 +554,7 @@ dependencies = [ { name = "opensearch-py" }, { name = "pydantic" }, { name = "redis" }, + { name = "sds-opensearch-query" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -579,10 +580,22 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.25.0" }, { name = "redis", specifier = ">=5.2.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.0" }, + { name = "sds-opensearch-query", directory = "../common/sds_opensearch_query" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, ] provides-extras = ["dev"] +[[package]] +name = "sds-opensearch-query" +version = "0.1.0" +source = { directory = "../common/sds_opensearch_query" } +dependencies = [ + { name = "opensearch-py" }, +] + +[package.metadata] +requires-dist = [{ name = "opensearch-py", specifier = ">=2.7.1" }] + [[package]] name = "six" version = "1.17.0" diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index dd0b98e90..40f8b5eb5 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -754,6 +754,7 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: # Federation peer short name (RFC [site].name, e.g. crc, haystack); # not SDS_PROGRAMMATIC_SITE_NAME. +# Export/OpenSearch ``site_name`` uses SDS_SITE_FQDN (matches federation.toml [site].fqdn). FEDERATION_SITE_NAME: str = env.str("FEDERATION_SITE_NAME", default="").strip() # Master switch: when False, federation export and Redis events are inactive. FEDERATION_ENABLED: bool = env.bool("FEDERATION_ENABLED", default=False) diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml index d35a667c7..e0e0cf1d4 100644 --- a/gateway/pyproject.toml +++ b/gateway/pyproject.toml @@ -56,6 +56,7 @@ "redis>=5.2.0", "requests>=2.32.5", "rich>=13.9.3", + "sds-opensearch-query", "scipy>=1.16.1", "sentry-sdk[django]>=2.25.1", "uvicorn>=0.32.0", @@ -662,3 +663,6 @@ [tool.uv] # https://docs.astral.sh/uv/concepts/resolution/#dependency-cooldowns exclude-newer = "1 week" + +[tool.uv.sources] +sds-opensearch-query = { path = "../common/sds_opensearch_query" } diff --git a/gateway/sds_gateway/api_methods/helpers/search_captures.py b/gateway/sds_gateway/api_methods/helpers/search_captures.py index 6cfa408bd..aa357ad13 100644 --- a/gateway/sds_gateway/api_methods/helpers/search_captures.py +++ b/gateway/sds_gateway/api_methods/helpers/search_captures.py @@ -8,6 +8,11 @@ from opensearchpy import exceptions as os_exceptions from rest_framework.request import Request from rich.pretty import pretty_repr +from sds_opensearch_query import bool_must_search_body +from sds_opensearch_query import build_metadata_filter_clauses +from sds_opensearch_query import flatten_property_paths +from sds_opensearch_query import run_search +from sds_opensearch_query.query import DEFAULT_SEARCH_SIZE from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import CaptureType @@ -20,10 +25,10 @@ from sds_gateway.api_methods.utils.asset_access_control import ( get_accessible_captures_queryset, ) -from sds_gateway.api_methods.utils.metadata_schemas import base_index_fields from sds_gateway.api_methods.utils.metadata_schemas import ( capture_index_mapping_by_type as md_props_by_type, ) +from sds_gateway.api_methods.utils.metadata_schemas import get_mapping_by_capture_type from sds_gateway.api_methods.utils.metadata_schemas import infer_index_name from sds_gateway.api_methods.utils.opensearch_client import get_opensearch_client from sds_gateway.api_methods.utils.relationship_utils import ( @@ -31,148 +36,38 @@ ) from sds_gateway.users.models import User -RangeValue = dict[str, int | float] UNKNOWN_CAPTURE_TYPE = "Unknown capture type" -# maximum size (doc count) of OpenSearch searches -MAX_OS_SIZE = 10_000 +MAX_OS_SIZE = DEFAULT_SEARCH_SIZE -def handle_nested_query( - field_path: str, - query_type: str, - value: Any, - levels_nested: int, - last_path: str | None = None, -) -> dict[str, Any]: - """Build a nested metadata query for a given field path and value. - - Args: - field_path: Full path to the field (e.g.'capture_props.metadata.fmax') - query_type: Type of query (e.g. 'match', 'term') - value: Value to match against - levels_nested: Number of nested levels to traverse - Returns: - Nested query dictionary for OpenSearch - """ - - if levels_nested == 0: - return {query_type: {f"{last_path}.{field_path}": value}} - - path_parts = field_path.split(".") - current_path = path_parts[0] - if last_path is not None: - current_path = f"{last_path}.{current_path}" - - return { - "nested": { - "path": current_path, - "query": handle_nested_query( - field_path=".".join(path_parts[1:]), - query_type=query_type, - value=value, - levels_nested=levels_nested - 1, - last_path=current_path, - ), - }, - } +def _warn_unknown_field(field_path: str) -> None: + msg = ( + f"Field '{field_path}' does not match an indexed field. " + "The filter may not be applied to the query accurately." + ) + log.warning(msg) def _build_os_metadata_query( capture_type: CaptureType | None = None, metadata_filters: list[dict[str, Any]] | None = None, ) -> list[dict[str, Any]]: - """Build OpenSearch query for metadata fields. - - Args: - capture_type: Type of capture (e.g. 'drf') - metadata_filters: list of dicts with 'field', 'type', and 'value' keys - Returns: - List of OpenSearch query clauses for the metadata fields - """ - - metadata_queries: list[dict[str, Any]] = [] - if metadata_filters is None: - log.debug("No metadata filters provided to build the OpenSearch query.") - return metadata_queries - - index_fields = _flatten_index_mapping( - index_mapping=_get_index_mapping(capture_type=capture_type), - index_fields=base_index_fields.copy(), + known_paths = _known_field_paths_for_capture_type(capture_type) + clauses = build_metadata_filter_clauses( + metadata_filters, + known_field_paths=known_paths, + on_unknown_field=_warn_unknown_field, ) - - for query in metadata_filters: - field_path: str = query["field_path"] - query_type: str = query["query_type"] - filter_value: Any = query["filter_value"] - - # warn if the field is not in the index mapping - # but continue to build the query - if field_path not in index_fields: - msg = ( - f"Field '{field_path}' does not match an indexed field." - "The filter may not be applied to the query accurately." - ) - log.warning(msg) - - levels_nested = field_path.count(".") - if levels_nested > 0: - metadata_queries.append( - handle_nested_query( - field_path=field_path, - query_type=query_type, - value=filter_value, - levels_nested=levels_nested, - ), - ) - else: - metadata_queries.append({query_type: {field_path: filter_value}}) - log.debug( - f"Built {len(metadata_queries)} OpenSearch metadata " - f"queries: {metadata_queries}", + f"Built {len(clauses)} OpenSearch metadata queries: {clauses}", ) - return metadata_queries + return clauses -def _flatten_index_mapping( - index_mapping: Mapping[str, Any], - index_fields: list[str], - prefix: str = "capture_props", - separator: str = ".", -) -> list[str]: - """Flatten the index mapping to a list of fields. - Args: - index_mapping: The index mapping to flatten. - index_fields: The list of fields to flatten. - Returns: - A list of flattened fields. - """ - for field, field_type in index_mapping.items(): - if isinstance(field_type, dict) and field_type.get("type") == "nested": - index_fields.extend( - [ - f"{prefix}{separator}{field}{separator}{nested_field}" - for nested_field in field_type.get("properties", {}) - ], - ) - else: - index_fields.append(f"{prefix}{separator}{field}") - return index_fields - - -def _get_index_mapping(capture_type: CaptureType | None) -> dict[str, dict[str, Any]]: - """Retrieves the OpenSearch index mapping for a given capture type or all types. - - Args: - capture_type: The capture type for which the index mapping is requested. - If None, the function merges all capture type properties. - Raises: - ValueError: If the capture type is not None and is not recognized. - Returns: - A dict where the keys are property names and the values are dicts with - metadata for the specified capture type or all capture types combined. - """ +def _known_field_paths_for_capture_type( + capture_type: CaptureType | None, +) -> frozenset[str]: implemented_capture_types = set(md_props_by_type.keys()) assert implemented_capture_types, ( "No capture types are implemented. Please check the metadata properties." @@ -182,13 +77,14 @@ def _get_index_mapping(capture_type: CaptureType | None) -> dict[str, dict[str, if capture_type not in implemented_capture_types: msg = f"{UNKNOWN_CAPTURE_TYPE}: {capture_type}" raise ValueError(msg) - index_mapping = md_props_by_type.get(capture_type, {}) - else: - # merge all capture type properties into a single flat dictionary - index_mapping = {} - for ct in implemented_capture_types: - index_mapping.update(md_props_by_type[ct]) - return index_mapping + properties = get_mapping_by_capture_type(capture_type)["properties"] + return flatten_property_paths(properties) + + paths: set[str] = set() + for ct in implemented_capture_types: + properties = get_mapping_by_capture_type(ct)["properties"] + paths.update(flatten_property_paths(properties)) + return frozenset(paths) def get_capture_queryset( @@ -196,12 +92,9 @@ def get_capture_queryset( capture_type: CaptureType | None, ) -> QuerySet[Capture]: """Get the capture queryset based on the capture type.""" - # Get captures accessible to the user using database-level filtering capture_queryset = get_accessible_captures_queryset(request_user) - # filter by capture type if provided if capture_type: - # verify capture type exists before filtering if not md_props_by_type.get(capture_type): raise ValueError(UNKNOWN_CAPTURE_TYPE) capture_queryset = capture_queryset.filter(capture_type=capture_type) @@ -238,10 +131,14 @@ def search_captures( log.debug("No metadata queries provided. Returning all captures.") return capture_queryset - os_query = _build_os_query_for_captures( - capture_type=capture_type, - metadata_queries=metadata_queries, - ) + must_clauses: list[dict[str, Any]] = [] + if capture_type: + must_clauses.append({"term": {"capture_type": capture_type}}) + must_clauses.extend(metadata_queries) + + os_query = bool_must_search_body(*must_clauses) + log.debug("OpenSearch query:") + log.debug(pretty_repr(os_query, indent_size=4)) client = get_opensearch_client() index_name: str = ( @@ -249,50 +146,29 @@ def search_captures( ) try: - response = client.search( + hits = run_search( + client, index=index_name, body=os_query, - size=MAX_OS_SIZE, # pyright: ignore[reportCallIssue] + size=MAX_OS_SIZE, ) - except os_exceptions.NotFoundError as err: - msg = f"Index '{index_name}' not found" - log.exception(msg) - raise ValueError(msg) from err except os_exceptions.ConnectionError as err: msg = f"Failed to connect to OpenSearch: {err}" log.exception(msg) raise - except os_exceptions.RequestError as err: - # raise ValueError to trigger a 400 HTTP response - # and pass the information to the client about what - # went wrong with the query they provided - context_for_user = "Query error" - info = err.info - if isinstance(info, dict): - root_causes: list[dict[str, str]] = info.get("error", {}).get( - "root_cause", [] - ) - root_cause_reason: str = ( - root_causes[0].get("reason", "") if root_causes else "" - ) - reason = str(root_cause_reason) if root_cause_reason else str(info) - else: - reason = str(info) - msg = f"{context_for_user}: {reason}" - raise ValueError(msg) from err + except ValueError as err: + raise ValueError(str(err)) from err except os_exceptions.OpenSearchException as err: msg = f"OpenSearch generic error: {err}" log.exception(msg) raise - # Extract capture UUIDs from OpenSearch response - capture_uuids: list[str] = [hit["_id"] for hit in response["hits"]["hits"]] + capture_uuids: list[str] = [hit["_id"] for hit in hits] if not capture_uuids: log.debug("No captures found in OpenSearch.") return capture_queryset.none() - # Check if we're approaching the maximum size limit num_hits = len(capture_uuids) if num_hits > 0.9 * MAX_OS_SIZE: log.warning( @@ -301,9 +177,8 @@ def search_captures( ) log.debug(f"Found {len(capture_uuids)} matching captures.") - # Filter the queryset to only include captures found in OpenSearch filtered_queryset = capture_queryset.filter(uuid__in=capture_uuids).order_by( - "-updated_at" + "-updated_at", ) log.debug( @@ -314,43 +189,13 @@ def search_captures( return filtered_queryset -def _build_os_query_for_captures( - capture_type: CaptureType | None, - metadata_queries: list[dict[str, Any]], -) -> dict[str, Any]: - """Build the OpenSearch query for searching captures. - - Args: - capture_type: The type of capture to filter by, or None for all types. - metadata_queries: A list of metadata query clauses to include in the query. - - Returns: - A dictionary representing the OpenSearch query. - """ - must_clauses: list[dict[str, dict[str, Any]]] = [] - if capture_type: - must_clauses.append({"term": {"capture_type": capture_type}}) - must_clauses.extend(metadata_queries) - - query = { - "query": { - "bool": { - "must": must_clauses, - }, - }, - } - log.debug("OpenSearch query:") - log.debug(pretty_repr(query, indent_size=4)) - return query - - # TODO: add pagination before retrieval rather than after # Need to paginate/limit OpenSearch results list before grouping # and then paginate/limit the grouped captures def get_composite_captures( captures: QuerySet[Capture], request: Request | None = None, - bulk_metadata: dict[str, dict[str, Any]] | None = None, + bulk_metadata: dict[str, Mapping[str, Any]] | None = None, ) -> list[dict[str, Any]]: """Get captures as composite objects, grouping multi-channel captures. @@ -374,11 +219,9 @@ def get_composite_captures( for capture_list in grouped_captures.values(): if len(capture_list) > 1: - # Multiple captures with same top_level_dir - create composite composite_data = build_composite_capture_data(capture_list) composite_captures.append(composite_data) else: - # Single capture - serialize normally capture = capture_list[0] capture_data = serialize_capture_or_composite(capture, context=context) composite_captures.append(capture_data) diff --git a/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py b/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py new file mode 100644 index 000000000..27c8280e0 --- /dev/null +++ b/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py @@ -0,0 +1,73 @@ +"""Mark a dataset and its captures as public + FINAL for federation export.""" + +from __future__ import annotations + +from uuid import UUID + +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError +from django.db import transaction +from loguru import logger as log + +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import DatasetStatus + + +class Command(BaseCommand): + help = ( + "Set dataset status to FINAL and is_public=True, and mark linked captures " + "public (triggers federation Redis events when FEDERATION_ENABLED)." + ) + + def add_arguments(self, parser) -> None: + parser.add_argument( + "--dataset-uuid", + required=True, + help="Dataset UUID to publish for federation", + ) + parser.add_argument( + "--capture-uuids", + nargs="*", + default=None, + help="Optional capture UUIDs to attach before publishing", + ) + + def handle(self, *args, **options) -> None: + dataset_uuid = UUID(str(options["dataset_uuid"])) + capture_uuids = options["capture_uuids"] or [] + + with transaction.atomic(): + dataset = Dataset.objects.select_for_update().get(uuid=dataset_uuid) + if capture_uuids: + captures = Capture.objects.filter( + uuid__in=[UUID(str(u)) for u in capture_uuids], + is_deleted=False, + ) + missing = set(capture_uuids) - { + str(c.uuid) for c in captures + } + if missing: + msg = f"Captures not found: {sorted(missing)}" + raise CommandError(msg) + dataset.captures.add(*captures) + + Capture.objects.filter( + datasets=dataset, + is_deleted=False, + ).update(is_public=True) + + dataset.status = DatasetStatus.FINAL + dataset.is_public = True + dataset.save(update_fields=["status", "is_public", "updated_at"]) + + log.info( + "Published dataset {} for federation export (FINAL, public)", + dataset_uuid, + ) + self.stdout.write( + self.style.SUCCESS( + f"Dataset {dataset_uuid} is FINAL and public. " + "Re-run federation bootstrap or save again to re-index if needed.", + ), + ) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export.py b/gateway/sds_gateway/api_methods/tests/test_federation_export.py index 4d0318bf1..39fdc002a 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export.py @@ -23,6 +23,7 @@ @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.crc.nd.edu", FEDERATION_OPERATIONAL_OVERRIDE=True, FEDERATION_EXPORT_ALLOWED_CIDRS=["0.0.0.0/0", "::/0"], ) @@ -96,7 +97,7 @@ def test_sync_key_can_retrieve_public_dataset(self) -> None: ) assert response.status_code == status.HTTP_200_OK assert response.json()["uuid"] == str(self.public_dataset.uuid) - assert response.json()["site_name"] == "crc" + assert response.json()["site_name"] == "sds.crc.nd.edu" def test_sync_key_dataset_detail_404_when_not_indexed(self) -> None: with patch( diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py index 33d8bde47..27e314994 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py @@ -49,7 +49,7 @@ def test_dataset_export_field_names_match_pydantic() -> None: ) serializer = DatasetFederationSerializer( dataset, - context={"site_name": "crc"}, + context={"site_name": "sds.crc.nd.edu"}, ) assert_field_names_match( serializer, @@ -64,7 +64,7 @@ def test_capture_export_field_names_match_pydantic() -> None: capture = CaptureFactory(owner=owner, is_public=True) serializer = CaptureFederationSerializer( capture, - context={"site_name": "crc"}, + context={"site_name": "sds.crc.nd.edu"}, ) assert_field_names_match( serializer, diff --git a/gateway/uv.lock b/gateway/uv.lock index f51a7e6d2..addc5907f 100644 --- a/gateway/uv.lock +++ b/gateway/uv.lock @@ -3,7 +3,7 @@ revision = 3 requires-python = "==3.13.*" [options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer = "2026-06-25T12:23:34.503498Z" exclude-newer-span = "P1W" [[package]] @@ -2135,6 +2135,7 @@ dependencies = [ { name = "requests" }, { name = "rich" }, { name = "scipy" }, + { name = "sds-opensearch-query" }, { name = "sentry-sdk", extra = ["django"] }, { name = "uvicorn" }, { name = "uvicorn-worker" }, @@ -2221,6 +2222,7 @@ requires-dist = [ { name = "rich", marker = "extra == 'local'", specifier = ">=13.9.3" }, { name = "ruff", marker = "extra == 'local'", specifier = ">=0.7" }, { name = "scipy", specifier = ">=1.16.1" }, + { name = "sds-opensearch-query", directory = "../common/sds_opensearch_query" }, { name = "sentry-sdk", extras = ["django"], specifier = ">=2.25.1" }, { name = "uvicorn", specifier = ">=0.32.0" }, { name = "uvicorn-worker", specifier = ">=0.2.0" }, @@ -2230,6 +2232,17 @@ requires-dist = [ ] provides-extras = ["local", "production"] +[[package]] +name = "sds-opensearch-query" +version = "0.1.0" +source = { directory = "../common/sds_opensearch_query" } +dependencies = [ + { name = "opensearch-py" }, +] + +[package.metadata] +requires-dist = [{ name = "opensearch-py", specifier = ">=2.7.1" }] + [[package]] name = "sentry-sdk" version = "2.61.1" From 49f360eb8ec52dfb01272e765a07e5e0c6e120c7 Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 10 Jul 2026 14:01:02 -0400 Subject: [PATCH 02/22] linting, hooks --- gateway/sds_gateway/api_methods/federation/reindex.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gateway/sds_gateway/api_methods/federation/reindex.py b/gateway/sds_gateway/api_methods/federation/reindex.py index a292b50db..8ec1b684e 100644 --- a/gateway/sds_gateway/api_methods/federation/reindex.py +++ b/gateway/sds_gateway/api_methods/federation/reindex.py @@ -32,6 +32,9 @@ if TYPE_CHECKING: from collections.abc import Iterable +if TYPE_CHECKING: + from collections.abc import Iterable + def reindex_federated_asset( instance: Dataset | Capture, From b0bacc3d0cd59e54b5f6ce5f6c07eddcfe7eecc4 Mon Sep 17 00:00:00 2001 From: klpoland Date: Wed, 15 Jul 2026 16:55:06 -0400 Subject: [PATCH 03/22] centralize opensearch client init --- .../sds_opensearch_query/client.py | 39 +++++++++++++++++++ .../sds_opensearch_query/query.py | 2 +- federation/sds_federation/main.py | 20 +++++++--- .../schemas/opensearch_indices.py | 2 +- .../tests/test_regression_fed_search.py | 2 +- .../api_methods/utils/opensearch_client.py | 37 ++++++------------ 6 files changed, 67 insertions(+), 35 deletions(-) create mode 100644 common/sds_opensearch_query/sds_opensearch_query/client.py diff --git a/common/sds_opensearch_query/sds_opensearch_query/client.py b/common/sds_opensearch_query/sds_opensearch_query/client.py new file mode 100644 index 000000000..30b16c620 --- /dev/null +++ b/common/sds_opensearch_query/sds_opensearch_query/client.py @@ -0,0 +1,39 @@ +from django.conf import settings +from loguru import logger as log +from opensearchpy import OpenSearch +from opensearchpy import RequestsHttpConnection +from requests.auth import HTTPBasicAuth + + +def build_opensearch_client( + *, + host: str, + port: int, + user: str = "", + password: str = "", + use_ssl: bool = False, + verify_certs: bool = False, + ca_certs: str | None = None, +) -> OpenSearch: + payload = { + "hosts": [{"host": host, "port": port}], + "http_auth": HTTPBasicAuth( + user, + password, + ), + "use_ssl": use_ssl, + "verify_certs": verify_certs, + "ssl_show_warn": False, + "connection_class": RequestsHttpConnection, + } + if verify_certs: + if not ca_certs: + msg = ( + "OPENSEARCH_VERIFY_CERTS is True but OPENSEARCH_CA_CERTS is not set." + "Not using SSL." + ) + log.warning(msg) + else: + log.info("OPENSEARCH_VERIFY_CERTS is True. Verifying certificates.") + payload["ca_certs"] = ca_certs + return OpenSearch(**payload) diff --git a/common/sds_opensearch_query/sds_opensearch_query/query.py b/common/sds_opensearch_query/sds_opensearch_query/query.py index 795751d43..4dbb26ae5 100644 --- a/common/sds_opensearch_query/sds_opensearch_query/query.py +++ b/common/sds_opensearch_query/sds_opensearch_query/query.py @@ -28,7 +28,7 @@ def multi_match_clause( def federation_not_deleted_clause() -> dict[str, Any]: - return {"term": {"is_federated_deleted": False}} + return {"term": {"is_deleted": False}} def bool_must_search_body( diff --git a/federation/sds_federation/main.py b/federation/sds_federation/main.py index ed44fc7cd..086ab9ef4 100644 --- a/federation/sds_federation/main.py +++ b/federation/sds_federation/main.py @@ -7,8 +7,8 @@ from fastapi import FastAPI from loguru import logger -from opensearchpy import OpenSearch +from common.sds_opensearch_query.client import build_opensearch_client from sds_federation.models import load_federation_config from sds_federation.routes.health import health_router from sds_federation.routes.search_index import search_index_router @@ -32,6 +32,9 @@ def _bootstrap_enabled() -> bool: ) +def get_setting(key: str) -> str: + return os.environ.get(key, "") + sync_app = FastAPI(title="SDS Federation Sync") sync_app.include_router(health_router) sync_app.include_router(search_index_router, prefix=API_PREFIX) @@ -43,15 +46,20 @@ async def lifespan(app: FastAPI): config = load_federation_config() http = build_gateway_http_client() - # TODO: Create a shared OpenSearch client for both apps in a later PR. - os_host = os.environ.get("OPENSEARCH_HOST", "opensearch") - os_port = os.environ.get("OPENSEARCH_PORT", "9200") - os_client = OpenSearch(hosts=[{"host": os_host, "port": int(os_port)}]) + os_client = build_opensearch_client( + host=get_setting("OPENSEARCH_HOST"), + port=get_setting("OPENSEARCH_PORT"), + user=get_setting("OPENSEARCH_USER"), + password=get_setting("OPENSEARCH_PASSWORD"), + use_ssl=get_setting("OPENSEARCH_USE_SSL"), + verify_certs=get_setting("OPENSEARCH_VERIFY_CERTS") == "true", + ca_certs=get_setting("OPENSEARCH_CA_CERTS"), + ) try: ensure_fed_indices(os_client) except Exception as exc: # noqa: BLE001 logger.error("Failed to ensure fed-* OpenSearch indices: {}", exc) - + peer_registry = PeerRegistry() fed_indexer = FederatedAssetIndexer(os_client) diff --git a/federation/sds_federation/schemas/opensearch_indices.py b/federation/sds_federation/schemas/opensearch_indices.py index e359056b8..9ad405c65 100644 --- a/federation/sds_federation/schemas/opensearch_indices.py +++ b/federation/sds_federation/schemas/opensearch_indices.py @@ -1,7 +1,7 @@ """OpenSearch index definitions for federated metadata (RFC fed-datasets / fed-captures). Explicit ``properties`` match the RFC search-facing field lists; export-only and -envelope fields (e.g. ``status``, ``capture_props``, ``is_federated_deleted``) rely on +envelope fields (e.g. ``status``, ``capture_props``, ``is_deleted``) rely on dynamic mapping. """ diff --git a/federation/tests/test_regression_fed_search.py b/federation/tests/test_regression_fed_search.py index 167428e22..b58e1d80a 100644 --- a/federation/tests/test_regression_fed_search.py +++ b/federation/tests/test_regression_fed_search.py @@ -38,7 +38,7 @@ def test_search_federated_datasets_builds_bool_query() -> None: call = client.search_calls[0] assert call["index"] == "fed-datasets" must = call["body"]["query"]["bool"]["must"] - assert {"term": {"is_federated_deleted": False}} in must + assert {"term": {"is_deleted": False}} in must assert {"term": {"site_name": "sds.crc.nd.edu"}} in must assert any("multi_match" in clause for clause in must) diff --git a/gateway/sds_gateway/api_methods/utils/opensearch_client.py b/gateway/sds_gateway/api_methods/utils/opensearch_client.py index 81380d8ca..272f27075 100644 --- a/gateway/sds_gateway/api_methods/utils/opensearch_client.py +++ b/gateway/sds_gateway/api_methods/utils/opensearch_client.py @@ -1,30 +1,15 @@ from django.conf import settings -from loguru import logger as log -from opensearchpy import OpenSearch -from opensearchpy import RequestsHttpConnection -from requests.auth import HTTPBasicAuth + +from common.sds_opensearch_query.client import build_opensearch_client def get_opensearch_client(): - payload = { - "hosts": [{"host": settings.OPENSEARCH_HOST, "port": settings.OPENSEARCH_PORT}], - "http_auth": HTTPBasicAuth( - settings.OPENSEARCH_USER, - settings.OPENSEARCH_PASSWORD, - ), - "use_ssl": settings.OPENSEARCH_USE_SSL, - "verify_certs": settings.OPENSEARCH_VERIFY_CERTS, - "ssl_show_warn": False, - "connection_class": RequestsHttpConnection, - } - if settings.OPENSEARCH_VERIFY_CERTS: - if not settings.OPENSEARCH_CA_CERTS: - msg = ( - "OPENSEARCH_VERIFY_CERTS is True but OPENSEARCH_CA_CERTS is not set." - "Not using SSL." - ) - log.warning(msg) - else: - log.info("OPENSEARCH_VERIFY_CERTS is True. Verifying certificates.") - payload["ca_certs"] = settings.OPENSEARCH_CA_CERTS - return OpenSearch(**payload) + return build_opensearch_client( + host=settings.OPENSEARCH_HOST, + port=settings.OPENSEARCH_PORT, + user=settings.OPENSEARCH_USER, + password=settings.OPENSEARCH_PASSWORD, + use_ssl=settings.OPENSEARCH_USE_SSL, + verify_certs=settings.OPENSEARCH_VERIFY_CERTS, + ca_certs=settings.OPENSEARCH_CA_CERTS, + ) From 75b1fba985cf9f8d4d0328a9c123eae2c419e1df Mon Sep 17 00:00:00 2001 From: klpoland Date: Thu, 16 Jul 2026 13:30:04 -0400 Subject: [PATCH 04/22] move search to gateway, add list helpers for serializing federated vs local data --- .../sds_opensearch_query/client.py | 1 - .../sds_opensearch_query/mapping.py | 53 + federation/sds_federation/main.py | 4 +- .../sds_federation/routes/search_index.py | 103 - federation/sds_federation/routes/webhooks.py | 2 +- .../schemas/opensearch_indices.py | 54 +- .../sds_federation/services/fed_index.py | 155 + .../sds_federation/services/fed_indices.py | 19 - .../sds_federation/services/fed_search.py | 286 - .../sds_federation/services/local_events.py | 2 +- .../tests/test_regression_fed_indices.py | 2 +- federation/tests/test_regression_fed_list.py | 4 +- .../tests/test_regression_fed_search.py | 35 +- .../api_methods/federation/search_helpers.py | 148 + .../api_methods/helpers/list_helpers.py | 323 + gateway/sds_gateway/templates/pages/home.html | 4 +- .../users/components/dataset_list_modals.html | 20 +- .../components/dataset_list_table_row.html | 15 +- .../search_published_datasets_tab.html | 16 +- gateway/sds_gateway/users/views/datasets.py | 200 +- .../sds_gateway/users/views/special_pages.py | 37 +- gateway/sds_gateway/users/views_deprecated.py | 5288 ----------------- 22 files changed, 783 insertions(+), 5988 deletions(-) delete mode 100644 federation/sds_federation/routes/search_index.py delete mode 100644 federation/sds_federation/services/fed_indices.py delete mode 100644 federation/sds_federation/services/fed_search.py create mode 100644 gateway/sds_gateway/api_methods/federation/search_helpers.py create mode 100644 gateway/sds_gateway/api_methods/helpers/list_helpers.py delete mode 100644 gateway/sds_gateway/users/views_deprecated.py diff --git a/common/sds_opensearch_query/sds_opensearch_query/client.py b/common/sds_opensearch_query/sds_opensearch_query/client.py index 30b16c620..f24b4ac99 100644 --- a/common/sds_opensearch_query/sds_opensearch_query/client.py +++ b/common/sds_opensearch_query/sds_opensearch_query/client.py @@ -1,4 +1,3 @@ -from django.conf import settings from loguru import logger as log from opensearchpy import OpenSearch from opensearchpy import RequestsHttpConnection diff --git a/common/sds_opensearch_query/sds_opensearch_query/mapping.py b/common/sds_opensearch_query/sds_opensearch_query/mapping.py index 13713240e..00f9b0a00 100644 --- a/common/sds_opensearch_query/sds_opensearch_query/mapping.py +++ b/common/sds_opensearch_query/sds_opensearch_query/mapping.py @@ -4,6 +4,59 @@ from typing import Any +FED_INDEX_SETTINGS: dict[str, Any] = { + "index": { + "number_of_shards": 1, + "number_of_replicas": 0, + }, +} + +# RFC §6 — fed-datasets search fields +RFC_FED_DATASET_PROPERTIES: dict[str, dict[str, Any]] = { + "uuid": {"type": "keyword"}, + "site_name": {"type": "keyword"}, # peer FQDN (federation.toml [site].fqdn) + "name": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, + "description": {"type": "text"}, + "abstract": {"type": "text"}, + "keywords": {"type": "keyword"}, + "owner_name": {"type": "keyword"}, + "created_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"}, + "updated_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"}, + "size": {"type": "long"}, + "capture_count": {"type": "integer"}, + "url": {"type": "keyword"}, +} + +# RFC §6 — fed-captures search fields +RFC_FED_CAPTURE_PROPERTIES: dict[str, dict[str, Any]] = { + "uuid": {"type": "keyword"}, + "site_name": {"type": "keyword"}, # peer FQDN (federation.toml [site].fqdn) + "capture_type": {"type": "keyword"}, + "channel": {"type": "keyword"}, + "center_frequency": {"type": "double"}, + "sample_rate": {"type": "double"}, + "start_time": {"type": "long"}, + "end_time": {"type": "long"}, + "dataset_ids": {"type": "keyword"}, + "url": {"type": "keyword"}, +} + + +def _fed_mappings(properties: dict[str, dict[str, Any]]) -> dict[str, Any]: + return { + "dynamic": True, + "properties": properties, + } + + +def fed_dataset_mappings() -> dict[str, Any]: + return _fed_mappings(RFC_FED_DATASET_PROPERTIES) + + +def fed_capture_mappings() -> dict[str, Any]: + return _fed_mappings(RFC_FED_CAPTURE_PROPERTIES) + + def flatten_property_paths( properties: Mapping[str, Any], *, diff --git a/federation/sds_federation/main.py b/federation/sds_federation/main.py index 086ab9ef4..68768df54 100644 --- a/federation/sds_federation/main.py +++ b/federation/sds_federation/main.py @@ -11,11 +11,10 @@ from common.sds_opensearch_query.client import build_opensearch_client from sds_federation.models import load_federation_config from sds_federation.routes.health import health_router -from sds_federation.routes.search_index import search_index_router from sds_federation.routes.webhooks import webhooks_router from sds_federation.services.bootstrap import run_bootstrap from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.fed_indices import ensure_fed_indices +from sds_federation.services.fed_index import ensure_fed_indices from sds_federation.services.local_events import build_gateway_http_client from sds_federation.services.local_events import run_federation_subscriber from sds_federation.services.peer_registry import PeerRegistry @@ -37,7 +36,6 @@ def get_setting(key: str) -> str: sync_app = FastAPI(title="SDS Federation Sync") sync_app.include_router(health_router) -sync_app.include_router(search_index_router, prefix=API_PREFIX) sync_app.include_router(webhooks_router, prefix=API_PREFIX) diff --git a/federation/sds_federation/routes/search_index.py b/federation/sds_federation/routes/search_index.py deleted file mode 100644 index cf5a0f989..000000000 --- a/federation/sds_federation/routes/search_index.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Federated metadata search (RFC GET /search/datasets, /search/captures).""" - -from __future__ import annotations - -import json -from typing import Any - -from fastapi import APIRouter -from fastapi import HTTPException -from fastapi import Query -from fastapi import Request -from opensearchpy import OpenSearch - -from sds_federation.services.fed_search import search_federated_captures -from sds_federation.services.fed_search import search_federated_datasets - -search_index_router = APIRouter(tags=["search"]) - - -def _opensearch_client(request: Request) -> OpenSearch: - client = getattr( - request.app.state, - "opensearch_client", - None, - ) - if client is None: - raise HTTPException( - status_code=503, - detail="OpenSearch client not ready", - ) - return client - - -def _parse_metadata_filters(raw: str | None) -> list[dict[str, Any]] | None: - if not raw: - return None - try: - parsed = json.loads(raw) - except json.JSONDecodeError as err: - raise HTTPException( - status_code=400, - detail="'metadata_filters' must be valid JSON", - ) from err - if not isinstance(parsed, list): - raise HTTPException( - status_code=400, - detail="'metadata_filters' must be a JSON list", - ) - return parsed - - -@search_index_router.get("/search/datasets") -async def search_datasets( - request: Request, - q: str | None = Query(default=None, description="Free-text search"), - site: str | None = Query( - default="*", - description="Peer site FQDN (federation.toml fqdn) or *", - ), - metadata_filters: str | None = Query( - default=None, - description="JSON list of metadata filter objects", - ), -) -> dict[str, Any]: - client = _opensearch_client(request) - filters = _parse_metadata_filters(metadata_filters) - try: - return search_federated_datasets( - client, - q=q, - site=site, - metadata_filters=filters, - ) - except ValueError as err: - raise HTTPException(status_code=400, detail=str(err)) from err - - -@search_index_router.get("/search/captures") -async def search_captures( - request: Request, - q: str | None = Query(default=None, description="Free-text search"), - site: str | None = Query( - default="*", - description="Peer site FQDN (federation.toml fqdn) or *", - ), - capture_type: str | None = Query(default=None), - metadata_filters: str | None = Query( - default=None, - description="JSON list of metadata filter objects", - ), -) -> dict[str, Any]: - client = _opensearch_client(request) - filters = _parse_metadata_filters(metadata_filters) - try: - return search_federated_captures( - client, - q=q, - site=site, - capture_type=capture_type, - metadata_filters=filters, - ) - except ValueError as err: - raise HTTPException(status_code=400, detail=str(err)) from err diff --git a/federation/sds_federation/routes/webhooks.py b/federation/sds_federation/routes/webhooks.py index b0e1ebc16..ebe995a1d 100644 --- a/federation/sds_federation/routes/webhooks.py +++ b/federation/sds_federation/routes/webhooks.py @@ -16,7 +16,7 @@ from sds_federation.services.bootstrap import backfill_peer_on_hello from sds_federation.services.bootstrap import peer_by_name from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.fed_search import alist_federated_assets_for_site +from sds_federation.services.fed_index import alist_federated_assets_for_site from sds_federation.services.peer_registry import PeerRegistry from sds_federation.services.peer_sync import peer_for_outbound diff --git a/federation/sds_federation/schemas/opensearch_indices.py b/federation/sds_federation/schemas/opensearch_indices.py index 9ad405c65..92b62577e 100644 --- a/federation/sds_federation/schemas/opensearch_indices.py +++ b/federation/sds_federation/schemas/opensearch_indices.py @@ -10,58 +10,10 @@ from typing import Any from sds_federation.schemas.webhooks import AssetTypeEnum +from common.sds_opensearch_query.mapping import fed_dataset_mappings +from common.sds_opensearch_query.mapping import fed_capture_mappings +from common.sds_opensearch_query.mapping import FED_INDEX_SETTINGS -FED_INDEX_SETTINGS: dict[str, Any] = { - "index": { - "number_of_shards": 1, - "number_of_replicas": 0, - }, -} - -# RFC §6 — fed-datasets search fields -RFC_FED_DATASET_PROPERTIES: dict[str, dict[str, Any]] = { - "uuid": {"type": "keyword"}, - "site_name": {"type": "keyword"}, # peer FQDN (federation.toml [site].fqdn) - "name": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, - "description": {"type": "text"}, - "abstract": {"type": "text"}, - "keywords": {"type": "keyword"}, - "owner_name": {"type": "keyword"}, - "created_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"}, - "updated_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"}, - "size": {"type": "long"}, - "capture_count": {"type": "integer"}, - "url": {"type": "keyword"}, -} - -# RFC §6 — fed-captures search fields -RFC_FED_CAPTURE_PROPERTIES: dict[str, dict[str, Any]] = { - "uuid": {"type": "keyword"}, - "site_name": {"type": "keyword"}, # peer FQDN (federation.toml [site].fqdn) - "capture_type": {"type": "keyword"}, - "channel": {"type": "keyword"}, - "center_frequency": {"type": "double"}, - "sample_rate": {"type": "double"}, - "start_time": {"type": "long"}, - "end_time": {"type": "long"}, - "dataset_ids": {"type": "keyword"}, - "url": {"type": "keyword"}, -} - - -def _fed_mappings(properties: dict[str, dict[str, Any]]) -> dict[str, Any]: - return { - "dynamic": True, - "properties": properties, - } - - -def fed_dataset_mappings() -> dict[str, Any]: - return _fed_mappings(RFC_FED_DATASET_PROPERTIES) - - -def fed_capture_mappings() -> dict[str, Any]: - return _fed_mappings(RFC_FED_CAPTURE_PROPERTIES) def index_body_for_asset(asset_type: AssetTypeEnum) -> dict[str, Any]: diff --git a/federation/sds_federation/services/fed_index.py b/federation/sds_federation/services/fed_index.py index c359cb727..2b57053fa 100644 --- a/federation/sds_federation/services/fed_index.py +++ b/federation/sds_federation/services/fed_index.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +import asyncio +from typing import Any from datetime import datetime from uuid import UUID @@ -7,6 +11,19 @@ from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import FederatedCaptureDoc from sds_federation.schemas.webhooks import FederatedDatasetDoc +from sds_federation.schemas.webhooks import asset_doc_class +from sds_federation.schemas.opensearch_indices import index_body_for_asset + + +def ensure_fed_indices(client: OpenSearch) -> None: + for asset_type in AssetTypeEnum: + index_name = asset_type.index_name + if client.indices.exists(index=index_name): + continue + client.indices.create( + index=index_name, + body=index_body_for_asset(asset_type), + ) def doc_id(site_name: str, uuid: UUID) -> str: @@ -21,6 +38,144 @@ def _parse_event_at(value: object) -> datetime | None: return None +_FEDERATION_META_KEYS = frozenset({"federation_event_at"}) + + +def _strip_federation_meta(source: dict) -> dict: + return { + key: value for key, value in source.items() if key not in _FEDERATION_META_KEYS + } + + + +_LIST_PAGE_SIZE = 1000 + + +def _parse_hit( + source: dict, + asset_type: AssetTypeEnum, +) -> FederatedDatasetDoc | FederatedCaptureDoc | None: + if not isinstance(source, dict): + return None + doc_class = asset_doc_class(asset_type) + return doc_class.model_validate(_strip_federation_meta(source)) + + +def load_federated_asset( + client: OpenSearch, + *, + site_name: str, + uuid: UUID, + asset_type: AssetTypeEnum, +) -> FederatedDatasetDoc | FederatedCaptureDoc | None: + """Return the indexed document for a site asset, or None if missing.""" + + def _get() -> dict | None: + try: + response = client.get( + index=asset_type.index_name, id=doc_id(site_name, uuid) + ) + except NotFoundError: + return None + source = response.get("_source") + if not isinstance(source, dict): + return None + return source + + source = _get() + if source is None: + return None + return _parse_hit(source, asset_type) + + +async def aload_federated_asset( + client: OpenSearch, + *, + site_name: str, + uuid: UUID, + asset_type: AssetTypeEnum, +) -> FederatedDatasetDoc | FederatedCaptureDoc | None: + return await asyncio.to_thread( + load_federated_asset, + client, + site_name=site_name, + uuid=uuid, + asset_type=asset_type, + ) + + + + +def _site_owned_query(site_name: str) -> dict[str, Any]: + return { + "bool": { + "should": [ + {"term": {"site_name.keyword": site_name}}, + {"term": {"site_name": site_name}}, + ], + "minimum_should_match": 1, + } + } + + +def list_federated_assets_for_site( + client: OpenSearch, + *, + site_name: str, + asset_type: AssetTypeEnum, +) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: + """Return all fed-* docs owned by ``site_name`` (paginated search_after).""" + docs: list[FederatedDatasetDoc | FederatedCaptureDoc] = [] + search_after: list[Any] | None = None + + while True: + body: dict[str, Any] = { + "size": _LIST_PAGE_SIZE, + "sort": [{"_id": "asc"}], + "query": _site_owned_query(site_name), + } + if search_after is not None: + body["search_after"] = search_after + + response = client.search(index=asset_type.index_name, body=body) + hits = (response.get("hits") or {}).get("hits") or [] + if not hits: + break + + for hit in hits: + source = hit.get("_source") + if not isinstance(source, dict): + continue + if source.get("site_name") != site_name: + continue + parsed = _parse_hit(source, asset_type) + if parsed is not None: + docs.append(parsed) + + if len(hits) < _LIST_PAGE_SIZE: + break + last_sort = hits[-1].get("sort") + if not isinstance(last_sort, list) or not last_sort: + break + search_after = last_sort + + return docs + + +async def alist_federated_assets_for_site( + client: OpenSearch, + *, + site_name: str, + asset_type: AssetTypeEnum, +) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: + return await asyncio.to_thread( + list_federated_assets_for_site, + client, + site_name=site_name, + asset_type=asset_type, + ) + + class FederatedAssetIndexer: def __init__(self, client: OpenSearch) -> None: self._client = client diff --git a/federation/sds_federation/services/fed_indices.py b/federation/sds_federation/services/fed_indices.py deleted file mode 100644 index c60a35550..000000000 --- a/federation/sds_federation/services/fed_indices.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Create fed-datasets / fed-captures OpenSearch indices if missing.""" - -from __future__ import annotations - -from opensearchpy import OpenSearch - -from sds_federation.schemas.opensearch_indices import index_body_for_asset -from sds_federation.schemas.webhooks import AssetTypeEnum - - -def ensure_fed_indices(client: OpenSearch) -> None: - for asset_type in AssetTypeEnum: - index_name = asset_type.index_name - if client.indices.exists(index=index_name): - continue - client.indices.create( - index=index_name, - body=index_body_for_asset(asset_type), - ) diff --git a/federation/sds_federation/services/fed_search.py b/federation/sds_federation/services/fed_search.py deleted file mode 100644 index a8086ac6f..000000000 --- a/federation/sds_federation/services/fed_search.py +++ /dev/null @@ -1,286 +0,0 @@ -"""Read and search federated documents in shared fed-* OpenSearch indices.""" - -from __future__ import annotations - -import asyncio -from collections.abc import Mapping -from typing import Any -from uuid import UUID - -from opensearchpy import OpenSearch -from opensearchpy.exceptions import NotFoundError -from sds_opensearch_query import bool_must_search_body -from sds_opensearch_query import build_metadata_filter_clauses -from sds_opensearch_query import federation_not_deleted_clause -from sds_opensearch_query import flatten_property_paths -from sds_opensearch_query import multi_match_clause -from sds_opensearch_query import run_search -from sds_opensearch_query import term_clause -from sds_opensearch_query.query import DEFAULT_SEARCH_SIZE - -from sds_federation.schemas.opensearch_indices import RFC_FED_CAPTURE_PROPERTIES -from sds_federation.schemas.opensearch_indices import RFC_FED_DATASET_PROPERTIES -from sds_federation.schemas.webhooks import AssetTypeEnum -from sds_federation.schemas.webhooks import FederatedCaptureDoc -from sds_federation.schemas.webhooks import FederatedDatasetDoc -from sds_federation.schemas.webhooks import asset_doc_class -from sds_federation.services.fed_index import doc_id - -_FEDERATION_META_KEYS = frozenset({"federation_event_at"}) -_LIST_PAGE_SIZE = 1000 - - -def _strip_federation_meta(source: dict) -> dict: - return { - key: value for key, value in source.items() if key not in _FEDERATION_META_KEYS - } - - -def _parse_hit( - source: dict, - asset_type: AssetTypeEnum, -) -> FederatedDatasetDoc | FederatedCaptureDoc | None: - if not isinstance(source, dict): - return None - doc_class = asset_doc_class(asset_type) - return doc_class.model_validate(_strip_federation_meta(source)) - - -def load_federated_asset( - client: OpenSearch, - *, - site_name: str, - uuid: UUID, - asset_type: AssetTypeEnum, -) -> FederatedDatasetDoc | FederatedCaptureDoc | None: - """Return the indexed document for a site asset, or None if missing.""" - - def _get() -> dict | None: - try: - response = client.get( - index=asset_type.index_name, id=doc_id(site_name, uuid) - ) - except NotFoundError: - return None - source = response.get("_source") - if not isinstance(source, dict): - return None - return source - - source = _get() - if source is None: - return None - return _parse_hit(source, asset_type) - - -def _site_owned_query(site_name: str) -> dict[str, Any]: - return { - "bool": { - "should": [ - {"term": {"site_name.keyword": site_name}}, - {"term": {"site_name": site_name}}, - ], - "minimum_should_match": 1, - } - } - - -def list_federated_assets_for_site( - client: OpenSearch, - *, - site_name: str, - asset_type: AssetTypeEnum, -) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: - """Return all fed-* docs owned by ``site_name`` (paginated search_after).""" - docs: list[FederatedDatasetDoc | FederatedCaptureDoc] = [] - search_after: list[Any] | None = None - - while True: - body: dict[str, Any] = { - "size": _LIST_PAGE_SIZE, - "sort": [{"_id": "asc"}], - "query": _site_owned_query(site_name), - } - if search_after is not None: - body["search_after"] = search_after - - response = client.search(index=asset_type.index_name, body=body) - hits = (response.get("hits") or {}).get("hits") or [] - if not hits: - break - - for hit in hits: - source = hit.get("_source") - if not isinstance(source, dict): - continue - if source.get("site_name") != site_name: - continue - parsed = _parse_hit(source, asset_type) - if parsed is not None: - docs.append(parsed) - - if len(hits) < _LIST_PAGE_SIZE: - break - last_sort = hits[-1].get("sort") - if not isinstance(last_sort, list) or not last_sort: - break - search_after = last_sort - - return docs - - -async def aload_federated_asset( - client: OpenSearch, - *, - site_name: str, - uuid: UUID, - asset_type: AssetTypeEnum, -) -> FederatedDatasetDoc | FederatedCaptureDoc | None: - return await asyncio.to_thread( - load_federated_asset, - client, - site_name=site_name, - uuid=uuid, - asset_type=asset_type, - ) - - -async def alist_federated_assets_for_site( - client: OpenSearch, - *, - site_name: str, - asset_type: AssetTypeEnum, -) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: - return await asyncio.to_thread( - list_federated_assets_for_site, - client, - site_name=site_name, - asset_type=asset_type, - ) - -FED_DATASET_TEXT_FIELDS = [ - "name^2", - "description", - "abstract", - "keywords", - "owner_name", -] - -FED_CAPTURE_TEXT_FIELDS = [ - "name", - "channel", - "capture_type", -] - -_WILDCARD_SITES = frozenset({"", "*", "all"}) - - -def _site_clause(site: str | None) -> dict[str, Any] | None: - if site is None or site.strip().lower() in _WILDCARD_SITES: - return None - return term_clause("site_name", site.strip()) - - -def _text_clause(q: str | None, fields: list[str]) -> dict[str, Any] | None: - if q is None or not q.strip(): - return None - return multi_match_clause(q.strip(), fields) - - -def _hits_to_response(hits: list[dict[str, Any]]) -> dict[str, Any]: - return { - "total": len(hits), - "hits": [ - { - "id": hit["_id"], - "score": hit.get("_score"), - "source": hit.get("_source", {}), - } - for hit in hits - ], - } - - -def _build_fed_must_clauses( - *, - q: str | None, - site: str | None, - metadata_filters: list[dict[str, Any]] | None, - rfc_properties: Mapping[str, dict[str, Any]], - text_fields: list[str], - extra_terms: list[tuple[str, Any]] | None = None, -) -> list[dict[str, Any]]: - known = flatten_property_paths(rfc_properties) - must: list[dict[str, Any]] = [federation_not_deleted_clause()] - - site_filter = _site_clause(site) - if site_filter is not None: - must.append(site_filter) - - for field, value in extra_terms or (): - if value is not None and value != "": - must.append(term_clause(field, value)) - - text = _text_clause(q, text_fields) - if text is not None: - must.append(text) - - must.extend( - build_metadata_filter_clauses( - metadata_filters, - known_field_paths=known, - ), - ) - return must - - -def search_federated_datasets( - client: OpenSearch, - *, - q: str | None = None, - site: str | None = None, - metadata_filters: list[dict[str, Any]] | None = None, - size: int = DEFAULT_SEARCH_SIZE, -) -> dict[str, Any]: - must = _build_fed_must_clauses( - q=q, - site=site, - metadata_filters=metadata_filters, - rfc_properties=RFC_FED_DATASET_PROPERTIES, - text_fields=FED_DATASET_TEXT_FIELDS, - ) - body = bool_must_search_body(*must) - hits = run_search( - client, - index=AssetTypeEnum.DATASET.index_name, - body=body, - size=size, - ) - return _hits_to_response(hits) - - -def search_federated_captures( - client: OpenSearch, - *, - q: str | None = None, - site: str | None = None, - metadata_filters: list[dict[str, Any]] | None = None, - capture_type: str | None = None, - size: int = DEFAULT_SEARCH_SIZE, -) -> dict[str, Any]: - must = _build_fed_must_clauses( - q=q, - site=site, - metadata_filters=metadata_filters, - rfc_properties=RFC_FED_CAPTURE_PROPERTIES, - text_fields=FED_CAPTURE_TEXT_FIELDS, - extra_terms=[("capture_type", capture_type)], - ) - body = bool_must_search_body(*must) - hits = run_search( - client, - index=AssetTypeEnum.CAPTURE.index_name, - body=body, - size=size, - ) - return _hits_to_response(hits) diff --git a/federation/sds_federation/services/local_events.py b/federation/sds_federation/services/local_events.py index 5bfa75663..fc327df6a 100644 --- a/federation/sds_federation/services/local_events.py +++ b/federation/sds_federation/services/local_events.py @@ -17,7 +17,7 @@ from sds_federation.schemas.webhooks import FederatedCaptureDoc from sds_federation.schemas.webhooks import FederatedDatasetDoc from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.fed_search import aload_federated_asset +from sds_federation.services.fed_index import aload_federated_asset from sds_federation.services.peer_registry import PeerRegistry from sds_federation.services.peer_sync import push_asset_updated_to_peers from sds_federation.services.redis_channel import resolve_federation_events_channel diff --git a/federation/tests/test_regression_fed_indices.py b/federation/tests/test_regression_fed_indices.py index 82463468e..12dac6ac8 100644 --- a/federation/tests/test_regression_fed_indices.py +++ b/federation/tests/test_regression_fed_indices.py @@ -6,7 +6,7 @@ import pytest from sds_federation.schemas.webhooks import AssetTypeEnum -from sds_federation.services.fed_indices import ensure_fed_indices +from sds_federation.services.fed_index import ensure_fed_indices class _IndicesStub: diff --git a/federation/tests/test_regression_fed_list.py b/federation/tests/test_regression_fed_list.py index 0fc52f74e..8c6df2a4b 100644 --- a/federation/tests/test_regression_fed_list.py +++ b/federation/tests/test_regression_fed_list.py @@ -13,8 +13,8 @@ from sds_federation.services.bootstrap import _index_export_docs from sds_federation.services.bootstrap import _parse_doc_event_at from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.fed_search import _LIST_PAGE_SIZE -from sds_federation.services.fed_search import list_federated_assets_for_site +from sds_federation.services.fed_index import _LIST_PAGE_SIZE +from sds_federation.services.fed_index import list_federated_assets_for_site from sds_federation.testing.sample_data import sample_federated_dataset_doc from tests.support.federation_mesh import peer_one_config diff --git a/federation/tests/test_regression_fed_search.py b/federation/tests/test_regression_fed_search.py index b58e1d80a..a5f1f5305 100644 --- a/federation/tests/test_regression_fed_search.py +++ b/federation/tests/test_regression_fed_search.py @@ -1,49 +1,19 @@ -"""Regression: federated search query assembly and OpenSearch document reads.""" +"""Regression tests for fed_* OpenSearch document reads.""" from __future__ import annotations from datetime import UTC from datetime import datetime -import pytest from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.fed_search import load_federated_asset -from sds_federation.services.fed_search import search_federated_datasets +from sds_federation.services.fed_index import load_federated_asset from sds_federation.testing.sample_data import TEST_DATASET_UUID from sds_federation.testing.sample_data import sample_federated_dataset_doc from tests.support.mock_opensearch import RecordingOpenSearch -pytest.importorskip("sds_opensearch_query") - -@pytest.mark.regression -def test_search_federated_datasets_builds_bool_query() -> None: - client = RecordingOpenSearch() - result = search_federated_datasets( - client, - q="rf survey", - site="sds.crc.nd.edu", - metadata_filters=[ - { - "field_path": "capture_count", - "query_type": "range", - "filter_value": {"gte": 1}, - }, - ], - ) - assert result["total"] == 0 - assert client.search_calls - call = client.search_calls[0] - assert call["index"] == "fed-datasets" - must = call["body"]["query"]["bool"]["must"] - assert {"term": {"is_deleted": False}} in must - assert {"term": {"site_name": "sds.crc.nd.edu"}} in must - assert any("multi_match" in clause for clause in must) - - -@pytest.mark.regression def test_load_federated_asset_returns_indexed_doc() -> None: opensearch = RecordingOpenSearch() site = "localhost" @@ -67,7 +37,6 @@ def test_load_federated_asset_returns_indexed_doc() -> None: assert loaded.uuid == TEST_DATASET_UUID -@pytest.mark.regression def test_load_federated_asset_missing_returns_none() -> None: opensearch = RecordingOpenSearch() assert ( diff --git a/gateway/sds_gateway/api_methods/federation/search_helpers.py b/gateway/sds_gateway/api_methods/federation/search_helpers.py new file mode 100644 index 000000000..7da68e766 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/search_helpers.py @@ -0,0 +1,148 @@ +"""Gateway-side OpenSearch search against shared fed-* indices.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from opensearchpy import OpenSearch +from sds_opensearch_query import bool_must_search_body +from sds_opensearch_query import build_metadata_filter_clauses +from sds_opensearch_query import federation_not_deleted_clause +from sds_opensearch_query import flatten_property_paths +from sds_opensearch_query import multi_match_clause +from sds_opensearch_query import run_search +from sds_opensearch_query import term_clause +from sds_opensearch_query.mapping import RFC_FED_CAPTURE_PROPERTIES +from sds_opensearch_query.mapping import RFC_FED_DATASET_PROPERTIES +from sds_opensearch_query.query import DEFAULT_SEARCH_SIZE + +from sds_gateway.api_methods.federation.fed_index import FED_CAPTURES_INDEX +from sds_gateway.api_methods.federation.fed_index import FED_DATASETS_INDEX + +FED_DATASET_TEXT_FIELDS = [ + "name^2", + "description", + "abstract", + "keywords", + "owner_name", +] + +FED_CAPTURE_TEXT_FIELDS = [ + "name", + "channel", + "capture_type", +] + +_WILDCARD_SITES = frozenset({"", "*", "all"}) + + +def _site_clause(site: str | None) -> dict[str, Any] | None: + if site is None or site.strip().lower() in _WILDCARD_SITES: + return None + return term_clause("site_name", site.strip()) + + +def _text_clause(q: str | None, fields: list[str]) -> dict[str, Any] | None: + if q is None or not q.strip(): + return None + return multi_match_clause(q.strip(), fields) + + +def _hits_to_response(hits: list[dict[str, Any]]) -> dict[str, Any]: + return { + "total": len(hits), + "hits": [ + { + "id": hit["_id"], + "score": hit.get("_score"), + "source": hit.get("_source", {}), + } + for hit in hits + ], + } + + +def _build_fed_must_clauses( + *, + q: str | None, + site: str | None, + metadata_filters: list[dict[str, Any]] | None, + rfc_properties: Mapping[str, dict[str, Any]], + text_fields: list[str], + extra_terms: list[tuple[str, Any]] | None = None, +) -> list[dict[str, Any]]: + known = flatten_property_paths(rfc_properties) + must: list[dict[str, Any]] = [federation_not_deleted_clause()] + + site_filter = _site_clause(site) + if site_filter is not None: + must.append(site_filter) + + for field, value in extra_terms or (): + if value is not None and value != "": + must.append(term_clause(field, value)) + + text = _text_clause(q, text_fields) + if text is not None: + must.append(text) + + must.extend( + build_metadata_filter_clauses( + metadata_filters, + known_field_paths=known, + ), + ) + return must + + +def search_federated_datasets( + client: OpenSearch, + *, + q: str | None = None, + site: str | None = None, + metadata_filters: list[dict[str, Any]] | None = None, + size: int = DEFAULT_SEARCH_SIZE, +) -> dict[str, Any]: + must = _build_fed_must_clauses( + q=q, + site=site, + metadata_filters=metadata_filters, + rfc_properties=RFC_FED_DATASET_PROPERTIES, + text_fields=FED_DATASET_TEXT_FIELDS, + ) + body = bool_must_search_body(*must) + hits = run_search( + client, + index=FED_DATASETS_INDEX, + body=body, + size=size, + ) + return _hits_to_response(hits) + + +def search_federated_captures( + client: OpenSearch, + *, + q: str | None = None, + site: str | None = None, + metadata_filters: list[dict[str, Any]] | None = None, + capture_type: str | None = None, + size: int = DEFAULT_SEARCH_SIZE, +) -> dict[str, Any]: + must = _build_fed_must_clauses( + q=q, + site=site, + metadata_filters=metadata_filters, + rfc_properties=RFC_FED_CAPTURE_PROPERTIES, + text_fields=FED_CAPTURE_TEXT_FIELDS, + extra_terms=[("capture_type", capture_type)], + ) + body = bool_must_search_body(*must) + hits = run_search( + client, + index=FED_CAPTURES_INDEX, + body=body, + size=size, + ) + return _hits_to_response(hits) diff --git a/gateway/sds_gateway/api_methods/helpers/list_helpers.py b/gateway/sds_gateway/api_methods/helpers/list_helpers.py new file mode 100644 index 000000000..fe23529a4 --- /dev/null +++ b/gateway/sds_gateway/api_methods/helpers/list_helpers.py @@ -0,0 +1,323 @@ +"""Lightweight list-row adapters for local and federated dataset UI lists.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from typing import Any + +from django.conf import settings +from django.urls import reverse +from django.utils import dateparse +from django.utils import timezone + +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.models import ItemType +from sds_gateway.api_methods.models import PermissionLevel +from sds_gateway.api_methods.models import UserSharePermission + +if TYPE_CHECKING: + from collections.abc import Iterable + + from django.db.models import QuerySet + + from sds_gateway.api_methods.models import Dataset + from sds_gateway.users.models import User + + +def local_site_name() -> str: + """FQDN used as ``site_name`` for local rows (RFC peer identity).""" + return str(getattr(settings, "SDS_SITE_FQDN", "") or "").strip() + + +def _parse_datetime(value: Any) -> datetime | None: + if value is None or value == "": + return None + if isinstance(value, datetime): + if timezone.is_naive(value): + return timezone.make_aware(value, timezone.get_current_timezone()) + return value + if isinstance(value, str): + parsed = dateparse.parse_datetime(value) + if parsed is None: + return None + if timezone.is_naive(parsed): + return timezone.make_aware(parsed, timezone.get_current_timezone()) + return parsed + return None + + +def dataset_list_dropdown_menu_items(row: dict[str, Any]) -> list[dict[str, Any]]: + """Build dropdown_menu.html items for a serialized dataset list row.""" + if row.get("is_federated"): + return [] + + uuid = str(row.get("uuid") or "") + if not uuid: + return [] + + is_owner = row.get("is_owner") + permission_level = row.get("permission_level") + is_contributor = permission_level == PermissionLevel.CONTRIBUTOR + is_co_owner = permission_level == PermissionLevel.CO_OWNER + dataset_published = row.get("status") == DatasetStatus.FINAL and row.get( + "is_public" + ) + + items: list[dict[str, Any]] = [] + if is_owner or is_contributor or is_co_owner: + items.append( + { + "label": "Share", + "icon": "person-plus", + "type": "button", + "modal_toggle": True, + "modal_target": f"#shareModal-{uuid}", + "data_attrs": {}, + } + ) + + if not dataset_published: + items.append( + { + "label": "Edit", + "icon": "pencil", + "type": "link", + "href": f"{reverse('users:group_captures')}?dataset_uuid={uuid}", + "data_attrs": {}, + } + ) + + if is_owner or is_co_owner: + items.append( + { + "label": "Create New Version", + "icon": "folder-symlink", + "type": "button", + "modal_toggle": True, + "modal_target": f"#versioningModal-{uuid}", + "data_attrs": {}, + } + ) + if not dataset_published: + items.append( + { + "label": "Publish", + "icon": "globe", + "type": "button", + "modal_toggle": True, + "modal_target": f"#publish-dataset-modal-{uuid}", + "data_attrs": {"dataset-uuid": uuid}, + "extra_class": "publish-dataset-btn", + } + ) + + status = row.get("status") + is_public = bool(row.get("is_public")) + is_deletable_dataset = status == DatasetStatus.DRAFT and not is_public + if is_owner and is_deletable_dataset: + dataset_name = str(row.get("name") or "").strip() or "Dataset" + items.append( + { + "label": "Delete", + "icon": "trash", + "type": "button", + "extra_class": "delete-asset-btn", + "data_attrs": { + "asset-type": "dataset", + "asset-uuid": uuid, + "asset-name": dataset_name[:200], + **({"asset-shared": "true"} if row.get("is_shared") else {}), + }, + } + ) + + items.append( + { + "label": "Web Download", + "icon": "download", + "type": "button", + "modal_toggle": True, + "modal_target": f"#webDownloadModal-{uuid}", + "data_attrs": {}, + } + ) + items.append( + { + "label": "SDK Instructions", + "icon": "code-slash", + "type": "button", + "modal_toggle": True, + "modal_target": f"#sdkDownloadModal-{uuid}", + "data_attrs": {}, + } + ) + return items + + +def _keyword_names(dataset: Dataset) -> list[str]: + return [ + kw.name + for kw in dataset.keywords.all() + if not getattr(kw, "is_deleted", False) + ] + + +def _permission_maps_for_user( + datasets: list[Dataset], + user: User | None, +) -> tuple[dict[Any, str], set[Any], set[Any]]: + """Batch-load permission_level, is_shared, is_shared_with_me maps.""" + uuids = [ds.uuid for ds in datasets] + if not uuids: + return {}, set(), set() + + shared_uuids = set( + UserSharePermission.objects.filter( + item_uuid__in=uuids, + item_type=ItemType.DATASET, + is_deleted=False, + is_enabled=True, + ).values_list("item_uuid", flat=True) + ) + + perm_by_uuid: dict[Any, str] = {} + shared_with_me: set[Any] = set() + if user is not None and getattr(user, "is_authenticated", False): + for ds in datasets: + if ds.owner_id == user.id: + perm_by_uuid[ds.uuid] = PermissionLevel.OWNER + + user_perms = UserSharePermission.objects.filter( + item_uuid__in=uuids, + item_type=ItemType.DATASET, + shared_with=user, + is_deleted=False, + is_enabled=True, + ).values_list("item_uuid", "permission_level") + for item_uuid, level in user_perms: + shared_with_me.add(item_uuid) + if item_uuid not in perm_by_uuid: + perm_by_uuid[item_uuid] = level + + return perm_by_uuid, shared_uuids, shared_with_me + + +def serialize_local_dataset_row( + dataset: Dataset, + user: User | None = None, + *, + permission_level: str | None = None, + is_shared: bool = False, + is_shared_with_me: bool = False, + include_actions: bool = True, +) -> dict[str, Any]: + """Build a lightweight list-row dict for a local Dataset ORM instance.""" + is_owner = bool( + user is not None + and getattr(user, "is_authenticated", False) + and dataset.owner_id == user.id + ) + if permission_level is None and is_owner: + permission_level = PermissionLevel.OWNER + + row: dict[str, Any] = { + "uuid": dataset.uuid, + "name": dataset.name, + "version": dataset.version, + "authors": dataset.get_authors_display(), + "keywords": _keyword_names(dataset), + "created_at": dataset.created_at, + "updated_at": dataset.updated_at, + "site_name": local_site_name(), + "is_federated": False, + "is_public": dataset.is_public, + "status": dataset.status, + "status_display": dataset.get_status_display(), + "owner_name": dataset.owner.name if dataset.owner else "Owner", + "is_owner": is_owner, + "is_shared": is_shared, + "is_shared_with_me": is_shared_with_me and not is_owner, + "permission_level": permission_level, + # Keep ORM for existing list modals that still read ``row.dataset``. + "dataset": dataset, + } + row["dropdown_menu_items"] = ( + dataset_list_dropdown_menu_items(row) if include_actions else [] + ) + return row + + +def serialize_federated_dataset_row(doc: dict[str, Any]) -> dict[str, Any]: + """Normalize a fed-datasets OpenSearch ``_source`` into a list-row dict.""" + return { + "uuid": doc.get("uuid"), + "name": doc.get("name") or "", + "version": doc.get("version", 1), + "authors": doc.get("authors") or [], + "keywords": doc.get("keywords") or [], + "created_at": _parse_datetime(doc.get("created_at")), + "updated_at": _parse_datetime(doc.get("updated_at")), + "site_name": doc.get("site_name") or "", + "is_federated": True, + "is_public": bool(doc.get("is_public", True)), + "status": doc.get("status") or DatasetStatus.FINAL, + "status_display": doc.get("status_display") or "Final", + "owner_name": doc.get("owner_name") or "", + "abstract": doc.get("abstract") or "", + "description": doc.get("description") or "", + "is_owner": False, + "is_shared": False, + "is_shared_with_me": False, + "permission_level": None, + "can_edit": False, + "can_share": False, + "dropdown_menu_items": [], + } + + +def serialize_datasets_for_user( + datasets: QuerySet[Dataset] | Iterable[Dataset], + user: User | None, + *, + include_actions: bool = True, +) -> list[dict[str, Any]]: + """Serialize local datasets into list-row dicts (no heavy API serializer).""" + dataset_list = list(datasets) + perm_by_uuid, shared_uuids, shared_with_me = _permission_maps_for_user( + dataset_list, + user, + ) + return [ + serialize_local_dataset_row( + dataset, + user, + permission_level=perm_by_uuid.get(dataset.uuid), + is_shared=dataset.uuid in shared_uuids, + is_shared_with_me=dataset.uuid in shared_with_me, + include_actions=include_actions, + ) + for dataset in dataset_list + ] + + +def _sort_key_value(row: dict[str, Any], key: str) -> tuple[bool, Any]: + value = row.get(key) + # None sorts after real values when ascending; reverse flips that. + return (value is None, value) + + +def merge_dataset_list_rows( + local_rows: list[dict[str, Any]], + federated_rows: list[dict[str, Any]], + *, + sort_by: str = "created_at", + descending: bool = True, +) -> list[dict[str, Any]]: + """Merge local + federated list rows and sort by a shared field.""" + merged = [*local_rows, *federated_rows] + merged.sort( + key=lambda row: _sort_key_value(row, sort_by), + reverse=descending, + ) + return merged diff --git a/gateway/sds_gateway/templates/pages/home.html b/gateway/sds_gateway/templates/pages/home.html index 8c552f007..01e338e49 100644 --- a/gateway/sds_gateway/templates/pages/home.html +++ b/gateway/sds_gateway/templates/pages/home.html @@ -68,8 +68,8 @@
- {% if dataset.dataset.created_at %} - {{ dataset.dataset.created_at|date:"Y-m-d" }} + {% if dataset.created_at %} + {{ dataset.created_at|date:"Y-m-d" }} {% endif %}
diff --git a/gateway/sds_gateway/templates/users/components/dataset_list_modals.html b/gateway/sds_gateway/templates/users/components/dataset_list_modals.html index c9223469e..bfecb87f8 100644 --- a/gateway/sds_gateway/templates/users/components/dataset_list_modals.html +++ b/gateway/sds_gateway/templates/users/components/dataset_list_modals.html @@ -4,15 +4,17 @@ Context: page_obj (paginated dataset list) {% endcomment %} {% for dataset in page_obj %} - {% include "users/partials/web_download_modal.html" with item=dataset item_type="dataset" %} - {% include "users/partials/sdk_download_modal.html" with dataset=dataset %} - {% if dataset.is_owner or dataset.permission_level == 'co-owner' or dataset.permission_level == 'contributor' %} - {% include "users/partials/share_modal.html" with item=dataset item_type="dataset" %} - {% endif %} - {% if dataset.is_owner or dataset.permission_level == 'co-owner' %} - {% include "users/partials/dataset_version_control.html" with dataset=dataset %} - {% if not dataset.dataset.status == 'final' or not dataset.is_public %} - {% include "users/partials/publish_dataset_modal.html" with dataset=dataset %} + {% if not dataset.is_federated %} + {% include "users/partials/web_download_modal.html" with item=dataset item_type="dataset" %} + {% include "users/partials/sdk_download_modal.html" with dataset=dataset %} + {% if dataset.is_owner or dataset.permission_level == 'co-owner' or dataset.permission_level == 'contributor' %} + {% include "users/partials/share_modal.html" with item=dataset item_type="dataset" %} + {% endif %} + {% if dataset.is_owner or dataset.permission_level == 'co-owner' %} + {% include "users/partials/dataset_version_control.html" with dataset=dataset %} + {% if not dataset.status == 'final' or not dataset.is_public %} + {% include "users/partials/publish_dataset_modal.html" with dataset=dataset %} + {% endif %} {% endif %} {% endif %} {% endfor %} diff --git a/gateway/sds_gateway/templates/users/components/dataset_list_table_row.html b/gateway/sds_gateway/templates/users/components/dataset_list_table_row.html index 33cb6aae0..6775bf3e8 100644 --- a/gateway/sds_gateway/templates/users/components/dataset_list_table_row.html +++ b/gateway/sds_gateway/templates/users/components/dataset_list_table_row.html @@ -29,6 +29,9 @@ {{ dataset.status_display }} {% endif %} {% if dataset.is_public %}{% endif %} + {% if dataset.is_federated %} + {{ dataset.site_name }} + {% endif %} {% if dataset.is_shared_with_me %} - {% if dataset.dataset.created_at %} + {% if dataset.created_at %} {% localtime on %}
- {{ dataset.dataset.created_at|date:"Y-m-d" }} - {{ dataset.dataset.created_at|date:"H:i:s T" }} + {{ dataset.created_at|date:"Y-m-d" }} + {{ dataset.created_at|date:"H:i:s T" }}
{% endlocaltime %} {% else %} - {% endif %} -{% include "users/components/dropdown_menu.html" with items=dropdown_menu_items %} + + {% if dropdown_menu_items %} + {% include "users/components/dropdown_menu.html" with items=dropdown_menu_items %} + {% endif %} + diff --git a/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html b/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html index 91b1dd4b1..a0a8f9eef 100644 --- a/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html +++ b/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html @@ -53,8 +53,12 @@

No datasets found

class="dataset-name-link" data-dataset-uuid="{{ dataset.uuid }}" data-item-uuid="{{ dataset.uuid }}" + data-item-type="dataset" + {% if dataset.is_federated %}data-is-federated="true" data-site-name="{{ dataset.site_name }}"{% endif %} aria-label="View details for {{ dataset.name }}">{{ dataset.name }} - {% if dataset.is_public %} + {% if dataset.is_federated %} + {{ dataset.site_name }} + {% elif dataset.is_public %} {% endif %} @@ -98,18 +102,18 @@

No datasets found

{% endif %} - {% if dataset.dataset.created_at %} + {% if dataset.created_at %} {% localtime on %}
- {{ dataset.dataset.created_at|date:"Y-m-d" }} - {{ dataset.dataset.created_at|date:"H:i:s T" }} + {{ dataset.created_at|date:"Y-m-d" }} + {{ dataset.created_at|date:"H:i:s T" }}
{% endlocaltime %} {% else %} - {% endif %} - {% if user.is_authenticated %} + {% if user.is_authenticated and not dataset.is_federated %}
+ {% elif user.is_authenticated %} + — {% endif %} {% endfor %} diff --git a/gateway/sds_gateway/users/views/datasets.py b/gateway/sds_gateway/users/views/datasets.py index da7967018..28cbfc8de 100644 --- a/gateway/sds_gateway/users/views/datasets.py +++ b/gateway/sds_gateway/users/views/datasets.py @@ -1,9 +1,7 @@ import json import uuid from pathlib import Path -from typing import TYPE_CHECKING from typing import Any -from typing import cast from uuid import UUID from django.contrib import messages @@ -31,6 +29,12 @@ from django.views.generic import TemplateView from loguru import logger as log +from sds_gateway.api_methods.federation.availability import is_federation_operational +from sds_gateway.api_methods.federation.search_helpers import search_federated_datasets +from sds_gateway.api_methods.helpers.list_helpers import local_site_name +from sds_gateway.api_methods.helpers.list_helpers import merge_dataset_list_rows +from sds_gateway.api_methods.helpers.list_helpers import serialize_datasets_for_user +from sds_gateway.api_methods.helpers.list_helpers import serialize_federated_dataset_row from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset from sds_gateway.api_methods.models import DatasetStatus @@ -42,10 +46,10 @@ from sds_gateway.api_methods.models import get_shared_users_for_item from sds_gateway.api_methods.models import get_user_permission_level from sds_gateway.api_methods.models import user_has_access_to_item -from sds_gateway.api_methods.serializers.dataset_serializers import DatasetGetSerializer from sds_gateway.api_methods.serializers.dataset_serializers import ( get_dataset_serializer, ) +from sds_gateway.api_methods.utils.opensearch_client import get_opensearch_client from sds_gateway.api_methods.utils.sds_files import sanitize_path_rel_to_user from sds_gateway.users.forms import CaptureSearchForm from sds_gateway.users.forms import DatasetInfoForm @@ -59,9 +63,6 @@ from .captures import _apply_frequency_filters_to_list -if TYPE_CHECKING: - from rest_framework.utils.serializer_helpers import ReturnDict - class GroupCapturesView( Auth0LoginRequiredMixin, FormSearchMixin, FileTreeMixin, TemplateView @@ -1035,151 +1036,33 @@ def filter_by_frequency_range( return datasets.filter(uuid__in=matching_dataset_uuids) -def _dataset_list_dropdown_menu_items(row: dict[str, Any]) -> list[dict[str, Any]]: - """Build dropdown_menu.html items for a serialized dataset list row.""" - uuid = str(row.get("uuid") or "") - if not uuid: - return [] - - is_owner = row.get("is_owner") - permission_level = row.get("permission_level") - is_contributor = permission_level == PermissionLevel.CONTRIBUTOR - is_co_owner = permission_level == PermissionLevel.CO_OWNER - dataset_published = row.get("status") == DatasetStatus.FINAL and row.get( - "is_public" - ) - - items: list[dict[str, Any]] = [] - if is_owner or is_contributor or is_co_owner: - # Share button - items.extend( - ( - { - "label": "Share", - "icon": "person-plus", - "type": "button", - "modal_toggle": True, - "modal_target": f"#shareModal-{uuid}", - "data_attrs": {}, - }, - ) - ) - - if not dataset_published: - # Edit button - items.append( - { - "label": "Edit", - "icon": "pencil", - "type": "link", - "href": f"{reverse('users:group_captures')}?dataset_uuid={uuid}", - "data_attrs": {}, - } - ) - - if is_owner or is_co_owner: - # Create new version button - items.append( - { - "label": "Create New Version", - "icon": "folder-symlink", - "type": "button", - "modal_toggle": True, - "modal_target": f"#versioningModal-{uuid}", - "data_attrs": {}, - } - ) - # Publish button - if not dataset_published: - items.append( - { - "label": "Publish", - "icon": "globe", - "type": "button", - "modal_toggle": True, - "modal_target": f"#publish-dataset-modal-{uuid}", - "data_attrs": {"dataset-uuid": uuid}, - "extra_class": "publish-dataset-btn", - } - ) - - status = row.get("status") - is_public = bool(row.get("is_public")) - is_deletable_dataset = status == DatasetStatus.DRAFT and not is_public - if is_owner and is_deletable_dataset: - dataset_name = str(row.get("name") or "").strip() or "Dataset" - items.append( - { - "label": "Delete", - "icon": "trash", - "type": "button", - "extra_class": "delete-asset-btn", - "data_attrs": { - "asset-type": "dataset", - "asset-uuid": uuid, - "asset-name": dataset_name[:200], - **({"asset-shared": "true"} if row.get("is_shared") else {}), - }, - } - ) - - # Web download button - items.append( - { - "label": "Web Download", - "icon": "download", - "type": "button", - "modal_toggle": True, - "modal_target": f"#webDownloadModal-{uuid}", - "data_attrs": {}, - } - ) - - # SDK download instructions button - items.append( - { - "label": "SDK Instructions", - "icon": "code-slash", - "type": "button", - "modal_toggle": True, - "modal_target": f"#sdkDownloadModal-{uuid}", - "data_attrs": {}, - } - ) - return items - - -def serialize_datasets_for_user( - datasets: QuerySet[Dataset], user: User | None +def _federated_published_dataset_rows( + *, + query: str | None = None, + site: str | None = None, ) -> list[dict[str, Any]]: - """Serialize datasets for display with user context. + """Fetch federated dataset docs and map them to list-row dicts.""" + if not is_federation_operational(): + return [] - Args: - datasets: QuerySet of Dataset objects to serialize - user: User object or None for anonymous users + try: + client = get_opensearch_client() + result = search_federated_datasets(client, q=query, site=site) + except Exception: # noqa: BLE001 + log.exception("Federated dataset search failed; returning local results only") + return [] - Returns: - List of serialized dataset dictionaries - """ - serialized_datasets = [] - for dataset in datasets: - # Create a mock request object for the serializer context - context_req = { - "request": type( - "Request", - (), - {"user": user if user and user.is_authenticated else None}, - )() - } - dataset_data = cast( - "ReturnDict", DatasetGetSerializer(dataset, context=context_req).data - ) - dataset_data["dataset"] = dataset - dataset_data["dropdown_menu_items"] = _dataset_list_dropdown_menu_items( - dataset_data - ) - serialized_datasets.append(dataset_data) - return serialized_datasets + local_site = local_site_name() + rows: list[dict[str, Any]] = [] + for hit in result.get("hits") or []: + source = hit.get("source") if isinstance(hit, dict) else None + if not isinstance(source, dict): + continue + # Local public datasets come from Postgres; skip same-site fed docs. + if local_site and source.get("site_name") == local_site: + continue + rows.append(serialize_federated_dataset_row(source)) + return rows def get_published_datasets() -> QuerySet[Dataset]: @@ -1190,7 +1073,8 @@ def get_published_datasets() -> QuerySet[Dataset]: is_public=True, is_deleted=False, ) - .prefetch_related("keywords", "owner") + .select_related("owner") + .prefetch_related("keywords") .distinct() .order_by("-created_at") ) @@ -1243,6 +1127,8 @@ def get(self, request: HttpRequest, *args, **kwargs) -> HttpResponse: """Handle GET request for dataset search.""" form = PublishedDatasetSearchForm(request.GET) datasets = get_published_datasets() + query: str | None = None + site = request.GET.get("site") # Apply search filters if form.is_valid(): @@ -1250,11 +1136,17 @@ def get(self, request: HttpRequest, *args, **kwargs) -> HttpResponse: datasets, form.cleaned_data, ) - - # Serialize datasets - serialized_datasets = serialize_datasets_for_user( - datasets, request.user if request.user.is_authenticated else None + query = (form.cleaned_data.get("query") or "").strip() or None + + user = request.user if request.user.is_authenticated else None + # Published search is read-only discovery; skip action dropdowns. + local_rows = serialize_datasets_for_user( + datasets, + user, + include_actions=False, ) + federated_rows = _federated_published_dataset_rows(query=query, site=site) + serialized_datasets = merge_dataset_list_rows(local_rows, federated_rows) # Paginate results paginator = Paginator(serialized_datasets, per_page=15) @@ -1373,6 +1265,7 @@ def _get_owned_datasets(self, user: User, order_by: str) -> QuerySet[Dataset]: """Get datasets owned by the user.""" return ( user.datasets.filter(is_deleted=False) + .select_related("owner") .prefetch_related("keywords") .order_by(order_by) ) @@ -1390,6 +1283,7 @@ def _get_shared_datasets(self, user: User, order_by: str) -> QuerySet[Dataset]: return ( Dataset.objects.filter(uuid__in=shared_dataset_uuids, is_deleted=False) .exclude(owner=user) + .select_related("owner") .prefetch_related("keywords") .order_by(order_by) ) diff --git a/gateway/sds_gateway/users/views/special_pages.py b/gateway/sds_gateway/users/views/special_pages.py index 024679cde..43c46cdfd 100644 --- a/gateway/sds_gateway/users/views/special_pages.py +++ b/gateway/sds_gateway/users/views/special_pages.py @@ -1,5 +1,3 @@ -from typing import TYPE_CHECKING -from typing import cast from uuid import UUID from django.conf import settings @@ -9,12 +7,12 @@ from django.views.generic import TemplateView from loguru import logger as log +from sds_gateway.api_methods.helpers.list_helpers import serialize_datasets_for_user from sds_gateway.api_methods.models import Dataset from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.models import KeySources from sds_gateway.api_methods.models import PermissionLevel from sds_gateway.api_methods.models import UserSharePermission -from sds_gateway.api_methods.serializers.dataset_serializers import DatasetGetSerializer from sds_gateway.users.forms import PublishedDatasetSearchForm from sds_gateway.users.mixins import Auth0LoginRequiredMixin from sds_gateway.users.models import UserAPIKey @@ -22,9 +20,6 @@ from .api_keys import MAX_API_KEY_COUNT from .api_keys import get_active_api_key_count -if TYPE_CHECKING: - from rest_framework.utils.serializer_helpers import ReturnDict - class HomePageView(TemplateView): """View for the home page with search form and latest datasets.""" @@ -41,30 +36,20 @@ def get_context_data(self, **kwargs): is_public=True, is_deleted=False, ) - .prefetch_related("keywords", "owner") + .select_related("owner") + .prefetch_related("keywords") .distinct() .order_by("-created_at")[:5] ) - # Serialize datasets - serialized_datasets = [] - for dataset in latest_datasets: - context_req = { - "request": type( - "Request", - (), - { - "user": self.request.user - if self.request.user.is_authenticated - else None - }, - )() - } - dataset_data = cast( - "ReturnDict", DatasetGetSerializer(dataset, context=context_req).data - ) - dataset_data["dataset"] = dataset - serialized_datasets.append(dataset_data) + user = ( + self.request.user if self.request.user.is_authenticated else None + ) + serialized_datasets = serialize_datasets_for_user( + latest_datasets, + user, + include_actions=False, + ) context["search_form"] = PublishedDatasetSearchForm() context["latest_datasets"] = serialized_datasets diff --git a/gateway/sds_gateway/users/views_deprecated.py b/gateway/sds_gateway/users/views_deprecated.py deleted file mode 100644 index ab67417b3..000000000 --- a/gateway/sds_gateway/users/views_deprecated.py +++ /dev/null @@ -1,5288 +0,0 @@ -import datetime -import json -import uuid -from dataclasses import dataclass -from itertools import chain -from pathlib import Path -from typing import TYPE_CHECKING -from typing import Any -from typing import cast -from uuid import UUID - -from django.conf import settings -from django.contrib import messages -from django.contrib.auth.models import AbstractBaseUser -from django.contrib.auth.models import AnonymousUser -from django.contrib.messages.views import SuccessMessageMixin -from django.core.paginator import EmptyPage -from django.core.paginator import Page -from django.core.paginator import PageNotAnInteger -from django.core.paginator import Paginator -from django.core.serializers.json import DjangoJSONEncoder -from django.db import DatabaseError -from django.db import transaction -from django.db.models import Q -from django.db.models import Sum -from django.db.models.query import QuerySet -from django.db.utils import IntegrityError -from django.db.utils import OperationalError -from django.http import Http404 -from django.http import HttpRequest -from django.http import HttpResponse -from django.http import HttpResponseRedirect -from django.http import JsonResponse -from django.shortcuts import get_object_or_404 -from django.shortcuts import redirect -from django.shortcuts import render -from django.template.defaultfilters import slugify -from django.template.loader import render_to_string -from django.urls import reverse -from django.utils import timezone -from django.utils.translation import gettext_lazy as _ -from django.views import View -from django.views.generic import DetailView -from django.views.generic import RedirectView -from django.views.generic import TemplateView -from django.views.generic import UpdateView -from loguru import logger as log -from minio.error import MinioException -from rest_framework import status - -from sds_gateway.api_methods.helpers.download_file import FileDownloadError -from sds_gateway.api_methods.helpers.download_file import download_file -from sds_gateway.api_methods.helpers.file_helpers import ( - check_file_contents_exist_helper, -) -from sds_gateway.api_methods.helpers.file_helpers import create_capture_helper_simple -from sds_gateway.api_methods.helpers.file_helpers import upload_file_helper_simple -from sds_gateway.api_methods.models import Capture -from sds_gateway.api_methods.models import CaptureType -from sds_gateway.api_methods.models import Dataset -from sds_gateway.api_methods.models import DatasetStatus -from sds_gateway.api_methods.models import File -from sds_gateway.api_methods.models import ItemType -from sds_gateway.api_methods.models import KeySources -from sds_gateway.api_methods.models import Keyword -from sds_gateway.api_methods.models import PermissionLevel -from sds_gateway.api_methods.models import ShareGroup -from sds_gateway.api_methods.models import TemporaryZipFile -from sds_gateway.api_methods.models import UserSharePermission -from sds_gateway.api_methods.models import get_shared_users_for_item -from sds_gateway.api_methods.models import get_user_permission_level -from sds_gateway.api_methods.models import user_has_access_to_item -from sds_gateway.api_methods.serializers.capture_serializers import ( - serialize_capture_or_composite, -) -from sds_gateway.api_methods.serializers.dataset_serializers import DatasetGetSerializer -from sds_gateway.api_methods.serializers.dataset_serializers import ( - get_dataset_serializer, -) -from sds_gateway.api_methods.serializers.file_serializers import FileGetSerializer -from sds_gateway.api_methods.tasks import is_user_locked -from sds_gateway.api_methods.tasks import notify_shared_users -from sds_gateway.api_methods.tasks import send_item_files_email -from sds_gateway.api_methods.utils.asset_access_control import user_has_access_to_file -from sds_gateway.api_methods.utils.metadata_schemas import infer_index_name -from sds_gateway.api_methods.utils.relationship_utils import ( - get_dataset_files_including_captures, -) -from sds_gateway.api_methods.utils.sds_files import sanitize_path_rel_to_user -from sds_gateway.users.file_utils import get_file_content_response -from sds_gateway.users.file_utils import validate_file_preview_request -from sds_gateway.users.files_utils import add_capture_files -from sds_gateway.users.files_utils import add_root_items -from sds_gateway.users.files_utils import add_shared_items -from sds_gateway.users.files_utils import add_user_files -from sds_gateway.users.files_utils import build_breadcrumbs -from sds_gateway.users.files_utils import items_to_dicts -from sds_gateway.users.forms import CaptureSearchForm -from sds_gateway.users.forms import DatasetInfoForm -from sds_gateway.users.forms import FileSearchForm -from sds_gateway.users.forms import PublishedDatasetSearchForm -from sds_gateway.users.forms import UserUpdateForm -from sds_gateway.users.h5_service import H5PreviewService -from sds_gateway.users.item_models import Item -from sds_gateway.users.mixins import ApprovedUserRequiredMixin -from sds_gateway.users.mixins import Auth0LoginRequiredMixin -from sds_gateway.users.mixins import FileTreeMixin -from sds_gateway.users.mixins import FormSearchMixin -from sds_gateway.users.mixins import UserSearchMixin -from sds_gateway.users.models import User -from sds_gateway.users.models import UserAPIKey -from sds_gateway.users.navigation_models import NavigationContext -from sds_gateway.users.navigation_models import NavigationType -from sds_gateway.users.utils import deduplicate_composite_captures -from sds_gateway.users.utils import render_html_fragment -from sds_gateway.users.utils import update_or_create_user_group_share_permissions -from sds_gateway.visualizations.config import get_visualization_compatibility - -if TYPE_CHECKING: - from rest_framework.utils.serializer_helpers import ReturnDict - -# Constants -MAX_API_KEY_COUNT = 10 - - -class ShareOperationError(Exception): - """Custom exception for share operation errors with HTTP status codes.""" - - def __init__(self, message: str, status_code: int = 400): - self.message = message - self.status_code = status_code - super().__init__(message) - - -def get_active_api_key_count(api_keys) -> int: - """ - Calculate the number of active (non-revoked and non-expired) API keys. - - Args: - api_keys: QuerySet of UserAPIKey objects - - Returns: - int: Number of active API keys - """ - now = datetime.datetime.now(datetime.UTC) - return sum( - 1 - for key in api_keys - if not key.revoked and (not key.expiry_date or key.expiry_date >= now) - ) - - -def validate_uuid(uuid_string: str) -> bool: - """Validate if a string is a valid UUID.""" - try: - uuid.UUID(uuid_string) - except (ValueError, TypeError): - return False - else: - return True - - -def _is_safe_template_path(template_name: str) -> bool: - """Check if the template path is safe (within users/components/).""" - normalized_path = Path(template_name).resolve() - base_dir = Path("users/components").resolve() - try: - normalized_path.relative_to(base_dir) - except ValueError: - return False - else: - return True - - -class UserDetailView(Auth0LoginRequiredMixin, DetailView): # pyright: ignore[reportMissingTypeArgument] - model = User - slug_field = "id" - slug_url_arg = "id" - - -user_detail_view = UserDetailView.as_view() - - -class UserUpdateView(Auth0LoginRequiredMixin, SuccessMessageMixin, UpdateView): # pyright: ignore[reportMissingTypeArgument] - model = User - form_class = UserUpdateForm - success_message = _("Information successfully updated") - - def get_success_url(self): - # for mypy to know that the user is authenticated - user = cast("User", self.request.user) - assert user.is_authenticated - return user.get_absolute_url() - - def get_object(self, queryset=None) -> AbstractBaseUser | AnonymousUser: - return self.request.user - - -user_update_view = UserUpdateView.as_view() - - -class UserRedirectView(Auth0LoginRequiredMixin, RedirectView): - permanent = False - - def get_redirect_url(self) -> str: - return reverse("users:view_api_key") - - -user_redirect_view = UserRedirectView.as_view() - - -class GenerateAPIKeyView(ApprovedUserRequiredMixin, Auth0LoginRequiredMixin, View): - template_name = "users/user_api_key.html" - - def get(self, request, *args, **kwargs): - # Get all API keys for the user (except SVIBackend) - api_keys = ( - UserAPIKey.objects.filter(user=request.user) - .exclude(source=KeySources.SVIBackend) - .order_by("revoked", "-created") - ) # Active keys first, then by creation date (recent first) - now = datetime.datetime.now(datetime.UTC) - active_api_key_count = get_active_api_key_count(api_keys) - context = { - "api_key": False, - "expires_at": None, - "expired": False, - "current_api_keys": api_keys, - "now": now, - "active_api_key_count": active_api_key_count, - } - if not api_keys.exists(): - return render( - request, - template_name=self.template_name, - context=context, - ) - - context.update( - { - "api_key": True, # return True if API key exists - "current_api_keys": api_keys, - "now": now, - "active_api_key_count": active_api_key_count, - } - ) - return render( - request, - template_name=self.template_name, - context=context, - ) - - def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: - """ - Creates a new API key for the authenticated user without deleting existing keys. - Enforces the maximum API key count (MAX_API_KEY_COUNT) per user. - """ - # Check if user has reached the maximum number of active API keys - api_keys = UserAPIKey.objects.filter(user=request.user).exclude( - source=KeySources.SVIBackend - ) - active_api_key_count = get_active_api_key_count(api_keys) - if active_api_key_count >= MAX_API_KEY_COUNT: - messages.error( - request, - f"You have reached the maximum number of API keys " - f"({MAX_API_KEY_COUNT}). Please revoke an existing key " - "before creating a new one.", - ) - return redirect("users:view_api_key") - - # Get the name and description from the form - api_key_name = request.POST.get("api_key_name", "") - api_key_description = request.POST.get("api_key_description", "") - api_key_expiry_date_str = request.POST.get("api_key_expiry_date", "") - - expiry_date = None - if api_key_expiry_date_str: - try: - expiry_date = datetime.datetime.strptime( - api_key_expiry_date_str, "%Y-%m-%d" - ).replace(tzinfo=datetime.UTC) - except ValueError: - messages.error(request, "Invalid expiration date format.") - return redirect("users:view_api_key") - - # create an API key for the user - _, raw_key = UserAPIKey.objects.create_key( - name=api_key_name, - description=api_key_description, - user=request.user, - source=KeySources.SDSWebUI, - expiry_date=expiry_date, - ) - request.session["new_api_key"] = raw_key - return redirect("users:new_api_key") - - -user_api_key_view = GenerateAPIKeyView.as_view() - - -class NewAPIKeyView(ApprovedUserRequiredMixin, Auth0LoginRequiredMixin, View): - template_name = "users/new_api_key.html" - - def get(self, request, *args, **kwargs): - api_key = request.session.pop("new_api_key", None) - return render(request, self.template_name, {"api_key": api_key}) - - -new_api_key_view = NewAPIKeyView.as_view() - - -class RevokeAPIKeyView(ApprovedUserRequiredMixin, Auth0LoginRequiredMixin, View): - def post(self, request, *args, **kwargs): - key_id = request.POST.get("key_id") - api_key = get_object_or_404(UserAPIKey, id=key_id, user=request.user) - if not api_key.revoked: - api_key.revoked = True - api_key.save() - messages.success(request, "API key revoked successfully.") - else: - messages.info(request, "API key is already revoked.") - return redirect("users:view_api_key") - - -revoke_api_key_view = RevokeAPIKeyView.as_view() - - -class GenerateAPIKeyFormView(ApprovedUserRequiredMixin, Auth0LoginRequiredMixin, View): - template_name = "users/generate_api_key_form.html" - - def get(self, request, *args, **kwargs): - api_keys = UserAPIKey.objects.filter(user=request.user).exclude( - source=KeySources.SVIBackend - ) - active_api_key_count = get_active_api_key_count(api_keys) - is_allowed_to_generate_key = active_api_key_count < MAX_API_KEY_COUNT - context = { - "is_allowed_to_generate_key": is_allowed_to_generate_key, - } - return render(request, self.template_name, context) - - -generate_api_key_form_view = GenerateAPIKeyFormView.as_view() - - -class ShareItemView(Auth0LoginRequiredMixin, UserSearchMixin, View): - """ - View to handle item sharing functionality using - the generalized UserSharePermission model. - - This view is used to search for users to share with, - add users to item sharing, and remove users from item sharing. - - It also handles the notification of shared users. - """ - - # Map item types to their corresponding models - ITEM_MODELS = { - ItemType.DATASET: Dataset, - ItemType.CAPTURE: Capture, - } - - def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: - """Handle user search requests.""" - try: - item_uuid_unknown = kwargs.get("item_uuid") - item_type_unknown = kwargs.get("item_type") - - item_uuid = ( - UUID(item_uuid_unknown) - if not isinstance(item_uuid_unknown, UUID) - else item_uuid_unknown - ) - item_type = ( - ItemType(item_type_unknown) - if not isinstance(item_type_unknown, ItemType) - else item_type_unknown - ) - except (ValueError, TypeError): - return JsonResponse( - {"error": "Invalid item UUID or item type"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - val_errors = "" - if not item_uuid: - val_errors += "Invalid item UUID\n" - if not item_type or not isinstance(item_type, ItemType): - val_errors += "Invalid item type format\n" - if item_type not in self.ITEM_MODELS: - val_errors += "Invalid item type\n" - - if val_errors: - return JsonResponse( - {"error": val_errors}, status=status.HTTP_400_BAD_REQUEST - ) - - # Check if user has access to the item (either as owner or shared user) - if not user_has_access_to_item( - request.user, item_uuid=item_uuid, item_type=item_type - ): - return JsonResponse( - {"error": f"{item_type.capitalize()} not found or access denied"}, - status=404, - ) - - # Get the item to check existing shared users - try: - model_class = self.ITEM_MODELS[item_type] - # Get the item (we know it exists and user has access) - - # Get exclusion lists for search - excluded_user_ids, excluded_group_ids = self._get_exclusion_lists( - request.user, item_uuid=item_uuid, item_type=item_type - ) - - except model_class.DoesNotExist: - return JsonResponse( - {"error": f"{item_type.capitalize()} not found"}, - status=status.HTTP_404_NOT_FOUND, - ) - - # Use the enhanced mixin method with exclusions and include groups - return self.search_users( - request, - exclude_user_ids=excluded_user_ids, - exclude_group_ids=excluded_group_ids, - include_groups=True, - ) - - def _get_exclusion_lists( - self, user: User, item_uuid: UUID, item_type: ItemType - ) -> tuple[list[int], list[str]]: - """Get lists of user IDs and group UUIDs to exclude from search results.""" - # Get individual users already shared with this item - shared_user_ids = list( - UserSharePermission.objects.filter( - item_uuid=item_uuid, - item_type=item_type, - is_deleted=False, - is_enabled=True, - ) - .exclude(share_groups__isnull=False) - .values_list("shared_with__id", flat=True) - ) - - # Get groups already shared with this item - shared_group_ids = list( - UserSharePermission.objects.filter( - item_uuid=item_uuid, - item_type=item_type, - is_deleted=False, - is_enabled=True, - ) - .filter(share_groups__isnull=False) - .values_list("share_groups__uuid", flat=True) - .distinct() - ) - - # Get users who are members of already shared groups - # (to exclude them from individual search) - shared_group_member_ids = self._get_group_member_ids(shared_group_ids) - - # Combine individual shared users and group members - # to exclude from individual search - all_excluded_user_ids = shared_user_ids + shared_group_member_ids - - return all_excluded_user_ids, shared_group_ids - - def _get_group_member_ids(self, group_uuids: list[str]) -> list[int]: - """Get user IDs of members in the given groups.""" - if not group_uuids: - return [] - - shared_groups = ShareGroup.objects.filter(uuid__in=group_uuids) - member_ids = [] - for group in shared_groups: - member_ids.extend(group.members.values_list("id", flat=True)) - return list(set(member_ids)) # Remove duplicates - - def _add_group_to_item( - self, - group_identifier: str, - item_uuid: UUID, - item_type: ItemType, - request_user: User, - message: str, - permission_level: PermissionLevel = PermissionLevel.VIEWER, - ) -> tuple[list[str], list[str]]: - """Add a group to item sharing.""" - group_uuid = group_identifier.split(":")[1] # Remove "group:" prefix - shared_users: list[str] = [] - errors: list[str] = [] - - try: - group = ShareGroup.objects.get( - uuid=group_uuid, owner=request_user, is_deleted=False - ) - - # Validate group has members - group_members = group.members.all() - if not group_members.exists(): - errors.append(f"Group '{group.name}' has no members") - return shared_users, errors - - # Create individual permissions for each group member - # Users who are already shared individually - # will have their permissions updated - for member in group_members: - update_or_create_user_group_share_permissions( - request_user=request_user, - group=group, - share_user=member, - item_uuid=item_uuid, - item_type=item_type, - message=message, - permission_level=permission_level, - ) - shared_users.append(member.email) - - except ShareGroup.DoesNotExist: - errors.append("Group not found or you don't own it") - - return shared_users, errors - - def _add_individual_user_to_item( - self, - email: str, - item_uuid: UUID, - item_type: ItemType, - request_user: User, - message: str, - permission_level: PermissionLevel = PermissionLevel.VIEWER, - ) -> tuple[str | None, str | None]: - """Add an individual user to item sharing. Returns (shared_user, error).""" - try: - user_to_share_with = User.objects.get(email=email, is_approved=True) - - if user_to_share_with.id == request_user.id: - return ( - None, - f"You cannot share a {item_type.lower()} with yourself ({email})", - ) - - # Check if already shared - existing_permission = self._get_existing_user_permission( - user_to_share_with, item_uuid, item_type, request_user - ) - - if existing_permission: - if ( - existing_permission.is_enabled - and existing_permission.is_individual_share - ): - return ( - None, - f"{item_type.capitalize()} is already shared with {email}", - ) - # Re-enable and mark as explicitly individually shared - existing_permission.is_enabled = True - existing_permission.is_individual_share = True - existing_permission.message = message - existing_permission.permission_level = permission_level - existing_permission.save() - return email, None - - # Create the share permission - UserSharePermission.objects.create( - owner=request_user, - shared_with=user_to_share_with, - item_type=item_type, - item_uuid=item_uuid, - message=message, - permission_level=permission_level, - is_enabled=True, - is_individual_share=True, - ) - except User.DoesNotExist: - return None, f"User with email {email} not found or not approved" - else: - return email, None - - def _get_existing_user_permission( - self, - user: User, - item_uuid: UUID, - item_type: ItemType, - request_user: User, - ) -> UserSharePermission | None: - """Get existing share permission for a user and item.""" - return UserSharePermission.objects.filter( - item_uuid=item_uuid, - item_type=item_type, - shared_with=user, - is_deleted=False, - ).first() - - def _validate_share_request( - self, request: HttpRequest, item_uuid: UUID, item_type: ItemType - ) -> JsonResponse | None: - """ - Validate the share request. - Returns error response if invalid, None if valid. - """ - # Validate item type - if item_type not in self.ITEM_MODELS: - return JsonResponse({"error": "Invalid item type"}, status=400) - - # Check if user has access to the item (either as owner or shared user) - if not user_has_access_to_item( - request.user, item_uuid=item_uuid, item_type=item_type - ): - return JsonResponse( - {"error": f"{item_type.capitalize()} not found or access denied"}, - status=404, - ) - - # For sharing operations, user must be owner or co-owner - if not UserSharePermission.user_can_share( - request.user, item_uuid=item_uuid, item_type=item_type - ): - return JsonResponse( - {"error": "Only owners and co-owners can manage sharing"}, status=403 - ) - - return None - - def _notify_shared_users_if_requested( - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - shared_users: list[str], - message: str, - ) -> None: - """Send notifications to shared users if requested.""" - notify = request.POST.get("notify_users") == "1" - if shared_users and notify: - notify_shared_users.delay( - item_uuid, item_type, shared_users, notify=True, message=message - ) - - def post( - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - *args: Any, - **kwargs: Any, - ) -> HttpResponse: - """ - Unified endpoint for sharing operations: adding users, updating permissions, - and removing users. - - Args: - request: The HTTP request object - item_uuid: The UUID of the item to share - item_type: The type of item to share from ItemType enum - Returns: - A JSON response containing the response message - """ - # Validate request - validation_error = self._validate_share_request(request, item_uuid, item_type) - if validation_error: - return validation_error - - try: - results = self._process_share_operations(request, item_uuid, item_type) - except ShareOperationError as e: - return JsonResponse({"error": e.message}, status=e.status_code) - except (ValueError, json.JSONDecodeError) as e: - return JsonResponse({"error": str(e)}, status=400) - - return self._build_share_response( - request, - item_uuid, - item_type, - results, - ) - - def _process_share_operations( - self, request: HttpRequest, item_uuid: UUID, item_type: ItemType - ) -> dict[str, list[str]]: - """Process all sharing operations and return results.""" - # Parse all change types from the request - new_users = self._parse_new_users(request) - permission_changes = self._parse_permission_changes(request) - removals = self._parse_removals(request) - - # Track results - results: dict[str, list[str]] = { - "added": [], - "updated": [], - "removed": [], - "errors": [], - } - - # Process new user additions - if new_users: - added_users, add_errors = self._add_users_to_item( - item_uuid, - item_type, - new_users, - request.user, - request.POST.get("notify_message", "").strip() or "", - ) - results["added"].extend(added_users) - results["errors"].extend(add_errors) - - # Process permission changes - for change in permission_changes: - change_result = self._process_permission_change( - request, item_uuid, item_type, change - ) - if change_result.get("success"): - results["updated"].append(change_result["message"]) - else: - results["errors"].append(change_result["error"]) - - # Process removals - for removal in removals: - removal_result = self._process_removal( - request, item_uuid, item_type, removal - ) - if removal_result.get("success"): - results["removed"].append(removal_result["message"]) - else: - results["errors"].append(removal_result["error"]) - - return results - - def _build_share_response( - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - results: dict[str, list[str]], - ) -> JsonResponse: - """Build the final JSON response for sharing operations.""" - # Send notifications if requested - if results["added"]: - self._notify_shared_users_if_requested( - request, - item_uuid, - item_type, - results["added"], - request.POST.get("notify_message", "").strip() or "", - ) - - # Build response message - messages = [] - if results["added"]: - messages.append(f"Added {len(results['added'])} user(s)") - if results["updated"]: - messages.append(f"Updated {len(results['updated'])} permission(s)") - if results["removed"]: - messages.append(f"Removed {len(results['removed'])} user(s)") - - success_message = "; ".join(messages) if messages else "No changes made" - - return JsonResponse( - { - "success": len(results["errors"]) == 0, - "message": success_message, - "details": results, - } - ) - - def _parse_new_users(self, request: HttpRequest) -> dict: - """Parse new users to add from the request.""" - user_emails_str = request.POST.get("user-search", "").strip() - if not user_emails_str: - return {} - - # Parse user permissions if provided - user_permissions = {} - user_permissions_str = request.POST.get("user_permissions", "") - if user_permissions_str: - try: - user_permissions = json.loads(user_permissions_str) - except json.JSONDecodeError as err: - msg = "Invalid user_permissions format" - raise ValueError(msg) from err - else: - # Validate all permission levels - valid_permissions = [ - PermissionLevel.VIEWER, - PermissionLevel.CONTRIBUTOR, - PermissionLevel.CO_OWNER, - ] - for email, perm_level in user_permissions.items(): - if perm_level not in valid_permissions: - msg = ( - f"Invalid permission level '{perm_level}' for user {email}" - ) - raise ValueError(msg) from None - - # Parse user emails and their permissions - users = {} - identifiers = [ - identifier.strip() - for identifier in user_emails_str.split(",") - if identifier.strip() - ] - - for identifier in identifiers: - permission = user_permissions.get(identifier, PermissionLevel.VIEWER) - users[identifier] = permission - - return users - - def _parse_permission_changes(self, request: HttpRequest) -> list[dict]: - """Parse permission changes from the request.""" - permission_changes_json = request.POST.get("permission_changes", "") - if not permission_changes_json: - return [] - - try: - changes_list = json.loads(permission_changes_json) - # Convert from [["email", {change_data}], ...] to list of dicts - return [ - {"user_email": email, **change_data} - for email, change_data in changes_list - ] - except json.JSONDecodeError as err: - msg = "Invalid permission_changes format" - raise ValueError(msg) from err - - def _parse_removals(self, request: HttpRequest) -> list[str]: - """Parse user removals from the request.""" - remove_users_json = request.POST.get("remove_users", "") - if not remove_users_json: - return [] - - try: - return json.loads(remove_users_json) - except json.JSONDecodeError as err: - msg = "Invalid remove_users format" - raise ValueError(msg) from err - - def _process_permission_change( - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - change: dict, # pyright: ignore[reportMissingTypeArgument] - ) -> dict: # pyright: ignore[reportMissingTypeArgument] - """Process a single permission change.""" - user_email = change.get("user_email") - new_permission = change.get("permissionLevel", PermissionLevel.VIEWER) - - if not user_email or not new_permission: - return {"success": False, "error": "Missing email or permission level"} - if new_permission == "remove": - return self._process_removal(request, item_uuid, item_type, user_email) - - # Validate permission level - valid_permissions = [ - PermissionLevel.VIEWER, - PermissionLevel.CONTRIBUTOR, - PermissionLevel.CO_OWNER, - ] - if new_permission not in valid_permissions: - error_msg = f"Invalid permission level: {new_permission}" - raise ShareOperationError( - error_msg, status_code=status.HTTP_400_BAD_REQUEST - ) - - # Handle group vs individual user - if user_email.startswith("group:"): - return self._update_group_permission( - request, item_uuid, item_type, user_email, new_permission - ) - return self._update_individual_permission( - request, item_uuid, item_type, user_email, new_permission - ) - - def _process_removal( - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - user_email: str, - ) -> dict: - """Process a single user removal.""" - if user_email.startswith("group:"): - return self._remove_group_access(request, item_uuid, item_type, user_email) - return self._remove_individual_access(request, item_uuid, item_type, user_email) - - def _update_individual_permission( - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - user_email: str, - new_permission: PermissionLevel, - ) -> dict: - """Update permission for an individual user.""" - try: - user_to_update = User.objects.get(email=user_email) - share_permission = self._get_existing_user_permission( - user_to_update, item_uuid, item_type, request.user - ) - - if not share_permission: - return { - "success": False, - "error": ( - f"User {user_email} is not shared with this {item_type.lower()}" - ), - } - old_permission = share_permission.permission_level - share_permission.permission_level = new_permission - share_permission.is_enabled = True # Re-enable if it was disabled - share_permission.save() - - return { # noqa: TRY300 - "success": True, - "message": ( - f"Updated {user_email} permission from {old_permission} " - f"to {new_permission}" - ), - } - - except User.DoesNotExist as err: - error_msg = f"User with email {user_email} not found" - raise ShareOperationError( - error_msg, status_code=status.HTTP_400_BAD_REQUEST - ) from err - - def _update_group_permission( - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - group_identifier: str, - new_permission: PermissionLevel, - ) -> dict: - """Update permission for a group.""" - try: - group_uuid = group_identifier.split(":")[1] - group = ShareGroup.objects.get( - uuid=group_uuid, owner=request.user, is_deleted=False - ) - - group_permissions = UserSharePermission.objects.filter( - item_uuid=item_uuid, - item_type=item_type, - share_groups=group, - is_deleted=False, - is_enabled=True, - ) - - if not group_permissions.exists(): - return { - "success": False, - "error": f"Group is not shared with this {item_type.lower()}", - } - updated_count = 0 - for permission in group_permissions: - permission.permission_level = new_permission - permission.save() - updated_count += 1 - - return { # noqa: TRY300 - "success": True, - "message": ( - f"Updated {updated_count} group members to {new_permission} " - "permission" - ), - } - - except ShareGroup.DoesNotExist as err: - error_msg = "Group not found or you don't own it" - raise ShareOperationError( - error_msg, status_code=status.HTTP_404_NOT_FOUND - ) from err - - def _remove_individual_access( - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - user_email: str, - ) -> dict: - """Remove access for an individual user.""" - try: - user_to_remove = User.objects.get(email=user_email) - share_permission = self._get_existing_user_permission( - user_to_remove, item_uuid, item_type, request.user - ) - - if not share_permission: - return { - "success": False, - "error": ( - f"User {user_email} is not shared with this {item_type.lower()}" - ), - } - - share_permission.is_enabled = False - share_permission.save() - - return { - "success": True, - "message": f"Removed {user_email} from {item_type.lower()} sharing", - } - - except User.DoesNotExist: - return { - "success": False, - "error": f"User with email {user_email} not found", - } - - def _remove_group_access( - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - group_identifier: str, - ) -> dict: - """Remove access for a group.""" - try: - group_uuid = group_identifier.split(":")[1] - group = ShareGroup.objects.get( - uuid=group_uuid, owner=request.user, is_deleted=False - ) - - group_permissions = UserSharePermission.objects.filter( - item_uuid=item_uuid, - item_type=item_type, - share_groups=group, - is_deleted=False, - is_enabled=True, - ) - - if not group_permissions.exists(): - return { - "success": False, - "error": f"Group is not shared with this {item_type.lower()}", - } - - removed_count = 0 - for permission in group_permissions: - permission.share_groups.remove(group) - permission.update_enabled_status() - permission.message = "Unshared from group" - permission.save() - removed_count += 1 - - return { - "success": True, - "message": ( - f"Removed {removed_count} group members from " - f"{item_type.lower()} sharing" - ), - } - - except ShareGroup.DoesNotExist: - return {"success": False, "error": "Group not found or you don't own it"} - - def _add_users_to_item( - self, - item_uuid: UUID, - item_type: ItemType, - users: dict, # {email: permission_level} - request_user: User, - message: str, - ) -> tuple[list[str], list[str]]: - """ - Add users and groups to item sharing. - Args: - item_uuid: The UUID of the item to share - item_type: The type of item to share - users: Dictionary mapping user emails to permission levels - request_user: The user sharing the item - Returns: - A tuple containing a list of shared users and a list of errors - """ - shared_users: list[str] = [] - errors: list[str] = [] - - for email, permission_level in users.items(): - if email.startswith("group:"): - group_shared_users, group_errors = self._add_group_to_item( - email, item_uuid, item_type, request_user, message, permission_level - ) - shared_users.extend(group_shared_users) - errors.extend(group_errors) - else: - user_shared, user_error = self._add_individual_user_to_item( - email, item_uuid, item_type, request_user, message, permission_level - ) - if user_shared: - shared_users.append(user_shared) - if user_error: - errors.append(user_error) - - return shared_users, errors - - -user_share_item_view = ShareItemView.as_view() - - -class ListFilesView(Auth0LoginRequiredMixin, View): - template_name = "users/file_list.html" - items_per_page = 25 - - def get(self, request, *args, **kwargs) -> HttpResponse: - # Get query parameters - page = int(request.GET.get("page", 1)) - sort_by = request.GET.get("sort_by", "created_at") - sort_order = request.GET.get("sort_order", "desc") - - # Get filter parameters - search = request.GET.get("search", "") - date_start = request.GET.get("date_start", "") - date_end = request.GET.get("date_end", "") - center_freq = request.GET.get("center_freq", "") - bandwidth = request.GET.get("bandwidth", "") - location = request.GET.get("location", "") - - # Base queryset - files_qs = request.user.files.filter(is_deleted=False) - - # Apply search filter - if search: - files_qs = files_qs.filter(name__icontains=search) - - # Apply date range filter - if date_start: - files_qs = files_qs.filter(created_at__gte=date_start) - if date_end: - files_qs = files_qs.filter(created_at__lte=date_end) - - # Apply other filters - if center_freq: - files_qs = files_qs.filter(center_frequency=center_freq) - if bandwidth: - files_qs = files_qs.filter(bandwidth=bandwidth) - if location: - files_qs = files_qs.filter(location=location) - - # Handle sorting - if sort_by: - if sort_order == "desc": - files_qs = files_qs.order_by(f"-{sort_by}") - else: - files_qs = files_qs.order_by(sort_by) - - # Paginate the results - paginator = Paginator(files_qs, self.items_per_page) - try: - files_page = paginator.page(page) - except (EmptyPage, PageNotAnInteger): - files_page = paginator.page(1) - - # Get visualization compatibility data - visualization_compatibility = get_visualization_compatibility() - - return render( - request, - template_name=self.template_name, - context={ - "files": files_page, - "total_pages": paginator.num_pages, - "current_page": page, - "total_items": paginator.count, - "sort_by": sort_by, - "sort_order": sort_order, - "visualization_compatibility": visualization_compatibility, - }, - ) - - -user_file_list_view = ListFilesView.as_view() - - -class FileDetailView(Auth0LoginRequiredMixin, DetailView): # pyright: ignore[reportMissingTypeArgument] - model = File - slug_field = "uuid" - slug_url_kwarg = "uuid" - template_name = "users/file_detail.html" - - def get_queryset(self) -> QuerySet[File]: - return self.request.user.files.filter(is_deleted=False).all() - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - target_file = cast("File", self.get_object()) - if target_file is None: - return context - serializer = FileGetSerializer(target_file) - context["returning_page"] = self.request.GET.get("returning_page", default=1) - context["file"] = serializer.data - context["skip_fields"] = [ - "bucket_name", - "deleted_at", - "file", - "is_deleted", - "name", - ] - return context - - -user_file_detail_view = FileDetailView.as_view() - - -class FileDownloadView(Auth0LoginRequiredMixin, View): - """Session-authenticated file download for the Users UI.""" - - def get(self, request: HttpRequest, uuid: str, *args, **kwargs) -> HttpResponse: - file_obj = get_object_or_404(File, uuid=uuid, is_deleted=False) - - # Access control: owner or shared via capture/dataset - has_access = user_has_access_to_file(request.user, file_obj) - - if not has_access: - return JsonResponse({"error": "Not found or access denied"}, status=404) - - try: - content = download_file(file_obj) - except (MinioException, FileDownloadError) as e: - log.warning(f"Error downloading file {file_obj.name}: {e}") - return JsonResponse({"error": "Failed to download file"}, status=500) - - response = HttpResponse( - content, - content_type=file_obj.media_type or "application/octet-stream", - ) - response["Content-Disposition"] = f'attachment; filename="{file_obj.name}"' - return response - - -class FileContentView(Auth0LoginRequiredMixin, View): - """Serve small text content of a file for modal previews. - - Supports rendering JSON as pretty-printed text. Enforces basic access - control: owners or users with access to the parent capture/dataset. - """ - - MAX_BYTES = 1024 * 1024 # 1 MiB safety limit for previews - - def get(self, request: HttpRequest, *args, **kwargs) -> HttpResponse: - """Get file content for preview.""" - file_uuid = kwargs.get("uuid") - if not file_uuid: - return JsonResponse({"error": "File UUID required"}, status=400) - - file_obj = get_object_or_404(File, uuid=file_uuid, is_deleted=False) - - # Validate request (access control and size checks) - error_response = validate_file_preview_request( - request.user, file_obj, self.MAX_BYTES - ) - if error_response is not None: - return error_response - - # Get file content response - try: - return get_file_content_response(file_obj, self.MAX_BYTES) - except OSError as e: - log.warning(f"Error reading file content for preview: {e}") - return JsonResponse({"error": "Error reading file"}, status=500) - - -class FileH5InfoView(Auth0LoginRequiredMixin, View): - """Return a summarized structure for an HDF5 file as JSON for modal preview.""" - - def get(self, request: HttpRequest, *args, **kwargs) -> JsonResponse | None: - file_uuid = kwargs.get("uuid") - if not file_uuid: - return JsonResponse({"error": "File UUID required"}, status=400) - - file_obj = get_object_or_404(File, uuid=file_uuid, is_deleted=False) - - # Use the H5 service to handle all the complex logic - h5_service = H5PreviewService() - return h5_service.get_preview(file_obj, request.user) - - -def _get_captures_for_template( - captures: QuerySet[Capture] | list[Capture] | Page[Capture], - request: HttpRequest, -) -> list[dict[str, Any]]: - """Get enhanced captures for the template.""" - enhanced_captures = [] - - # Bulk-load OpenSearch metadata before serialization loop - captures_list = list(captures) - if captures_list: - temp_qs = Capture.objects.filter(uuid__in=[c.uuid for c in captures_list]) - bulk_metadata = Capture.bulk_load_frequency_metadata(temp_qs) - Capture.set_bulk_metadata_cache(captures_list, bulk_metadata) - else: - bulk_metadata = {} - - for capture in captures_list: - # Use composite serialization to handle multi-channel captures properly - capture_data = serialize_capture_or_composite( - capture, context={"bulk_metadata": bulk_metadata} - ) - - # Composite serialization omits top-level name; - # templates and API need a display name - if not capture_data.get("name"): - capture_data["name"] = capture.name or "" - - # Add ownership flags for template display - capture_data["is_owner"] = capture.owner == request.user - capture_data["is_shared_with_me"] = capture.owner != request.user - capture_data["owner_name"] = capture.owner.name or "Owner" - capture_data["owner_email"] = capture.owner.email or "" - - # Add the original model instance for template use - capture_data["capture"] = capture - - # Add shared users data for share modal - if user_has_access_to_item(request.user, capture.uuid, ItemType.CAPTURE): - # Get shared users and groups using the new model - shared_permissions = ( - UserSharePermission.objects.filter( - item_uuid=capture.uuid, - item_type=ItemType.CAPTURE, - is_deleted=False, - is_enabled=True, - ) - .select_related("shared_with") - .prefetch_related("share_groups__members") - ) - - shared_users = [] - group_permissions = {} - - for perm in shared_permissions: - if perm.share_groups.exists(): - # Group member - collect by group - for group in perm.share_groups.all(): - group_uuid = str(group.uuid) - if group_uuid not in group_permissions: - group_permissions[group_uuid] = { - "name": group.name, - "email": f"group:{group_uuid}", - "type": "group", - "members": [], - "permission_level": perm.permission_level, - "owner": group.owner.name, - "owner_email": group.owner.email, - "is_group_owner": group.owner == request.user, - } - group_permissions[group_uuid]["members"].append( - { - "name": perm.shared_with.name, - "email": perm.shared_with.email, - } - ) - else: - # Individual user - shared_users.append( - { - "name": perm.shared_with.name, - "email": perm.shared_with.email, - "type": "user", - "permission_level": perm.permission_level, - } - ) - - # Add groups with member counts - for group_data in group_permissions.values(): - group_data["member_count"] = len(group_data["members"]) - shared_users.append(group_data) - capture_data["shared_users"] = shared_users - else: - capture_data["shared_users"] = [] - - enhanced_captures.append(capture_data) - - return enhanced_captures - - -# API performance constant: maximum number of captures to return in API responses -API_CAPTURES_LIMIT = 25 - - -def _get_user_captures_querysets( - user: User, -) -> tuple[QuerySet[Capture], QuerySet[Capture]]: - """Get owned and shared capture querysets for a user.""" - # Get captures owned by the user - owned_captures = user.captures.filter(is_deleted=False) - - # Get captures shared with the user using the new UserSharePermission model - shared_permissions = UserSharePermission.objects.filter( - shared_with=user, - item_type=ItemType.CAPTURE, - is_deleted=False, - is_enabled=True, - ).values_list("item_uuid", flat=True) - - shared_captures = Capture.objects.filter( - uuid__in=shared_permissions, is_deleted=False - ).exclude(owner=user) - - return owned_captures, shared_captures - - -def _apply_frequency_filters_to_list( # noqa: C901 - captures_list: list[Capture], - min_freq: str | float | None, - max_freq: str | float | None, -) -> list[Capture]: - """Apply frequency filters to a list of captures.""" - if not captures_list or (not min_freq and not max_freq): - return captures_list - - try: - # Convert list to queryset for bulk frequency loading - temp_qs = Capture.objects.filter(uuid__in=[c.uuid for c in captures_list]) - # Bulk load frequency metadata - frequency_data = Capture.bulk_load_frequency_metadata(temp_qs) - - # Parse frequency values - min_freq_str = str(min_freq).strip() if min_freq else "" - max_freq_str = str(max_freq).strip() if max_freq else "" - - try: - min_freq_val = float(min_freq_str) if min_freq_str else None - except ValueError: - min_freq_val = None - - try: - max_freq_val = float(max_freq_str) if max_freq_str else None - except ValueError: - max_freq_val = None - - if min_freq_val is None and max_freq_val is None: - return captures_list - - # Filter captures by frequency range - filtered_captures: list[Capture] = [] - for capture in captures_list: - capture_uuid = str(capture.uuid) - freq_info = frequency_data.get(capture_uuid, {}) - center_freq_hz = freq_info.get("center_frequency") - - if center_freq_hz is None: - continue - - try: - center_freq_hz = float(center_freq_hz) - except (ValueError, TypeError): - continue - - center_freq_ghz = center_freq_hz / 1e9 - - if min_freq_val is not None and center_freq_ghz < min_freq_val: - continue - if max_freq_val is not None and center_freq_ghz > max_freq_val: - continue - - filtered_captures.append(capture) - - except (DatabaseError, AttributeError) as e: - log.warning(f"Error in frequency filtering: {e}", exc_info=True) - # Continue with unfiltered list on error - return captures_list - - else: - return filtered_captures - - -def _apply_sorting_to_list( - captures_list: list[Capture], - sort_by: str, - sort_order: str, -) -> list[Capture]: - """Apply sorting to a list of captures.""" - if not sort_by or not captures_list: - return captures_list - - reverse = sort_order == "desc" - try: - allowed_sort_fields: set[str] = { - "uuid", - "created_at", - "updated_at", - "deleted_at", - "is_deleted", - "is_public", - "channel", - "scan_group", - "capture_type", - "top_level_dir", - "index_name", - } - if sort_by in allowed_sort_fields: - captures_list = sorted( - captures_list, - key=lambda c: ( - getattr(c, sort_by, None) is None, - getattr(c, sort_by, ""), - ), - reverse=reverse, - ) - except (TypeError, AttributeError) as e: - log.warning(f"Sorting failed: {e}") - - return captures_list - - -def _get_filtered_and_sorted_captures( - user: User, - params: dict[str, Any], - limit: int | None = None, -) -> list[Capture]: - """ - Get filtered and sorted captures for a user based on parameters. - - Args: - user: The user to get captures for - params: Dictionary of filter parameters - limit: Optional limit to apply to each queryset before union - - Returns: - List of filtered, sorted, and deduplicated Capture objects - """ - # Get owned and shared captures - owned_captures, shared_captures = _get_user_captures_querysets(user) - - # Apply basic filters to each queryset - owned_captures = _apply_basic_filters( - qs=owned_captures, - search=params["search"], - date_start=params["date_start"], - date_end=params["date_end"], - cap_type=params["cap_type"], - ) - shared_captures = _apply_basic_filters( - qs=shared_captures, - search=params["search"], - date_start=params["date_start"], - date_end=params["date_end"], - cap_type=params["cap_type"], - ) - - # Apply limit to each queryset before union to reduce memory usage - if limit is not None: - # Add buffer to ensure we have enough after filtering/deduplication - queryset_limit = int(limit * 1.5) # 50% buffer - owned_captures = owned_captures[:queryset_limit] - shared_captures = shared_captures[:queryset_limit] - - # Union the querysets (all basic filters already applied) - qs = owned_captures.union(shared_captures) - - # Convert to list (single DB query for union) - captures_list: list[Capture] = list(qs) - - # Apply frequency filters to the combined list - captures_list = _apply_frequency_filters_to_list( - captures_list, params["min_freq"], params["max_freq"] - ) - - # Apply sorting to the combined list (union doesn't preserve order) - captures_list = _apply_sorting_to_list( - captures_list, params["sort_by"], params["sort_order"] - ) - - # Deduplicate composite captures - unique_captures = deduplicate_composite_captures(captures_list) - - # Apply final limit if specified (after deduplication) - if limit is not None: - unique_captures = unique_captures[:limit] - - return unique_captures - - -class ListCapturesView(Auth0LoginRequiredMixin, View): - """Handle HTML requests for the captures list page.""" - - template_name = "users/file_list.html" - default_items_per_page = 25 - max_items_per_page = 100 - - def _extract_request_params(self, request): - """Extract and return request parameters for HTML view.""" - return { - "page": int(request.GET.get("page", 1)), - "sort_by": request.GET.get("sort_by", "created_at"), - "sort_order": request.GET.get("sort_order", "desc"), - "search": request.GET.get("search", ""), - "date_start": request.GET.get("date_start", ""), - "date_end": request.GET.get("date_end", ""), - "cap_type": request.GET.get("capture_type", ""), - "min_freq": request.GET.get("min_freq", ""), - "max_freq": request.GET.get("max_freq", ""), - "items_per_page": min( - int(request.GET.get("items_per_page", self.default_items_per_page)), - self.max_items_per_page, - ), - } - - def get(self, request, *args, **kwargs) -> HttpResponse: - """Handle HTML page requests for captures list.""" - # Extract request parameters - params = self._extract_request_params(request) - - # Get filtered and sorted captures - unique_captures = _get_filtered_and_sorted_captures(request.user, params) - - # Paginate the unique captures - paginator = Paginator(unique_captures, params["items_per_page"]) - try: - page_obj = paginator.page(params["page"]) - except (EmptyPage, PageNotAnInteger): - page_obj = paginator.page(1) - - # Update the page_obj with enhanced captures - page_obj.object_list = _get_captures_for_template(page_obj, request) - - # Get visualization compatibility data - visualization_compatibility = get_visualization_compatibility() - - return render( - request, - self.template_name, - { - "captures": page_obj, - "sort_by": params["sort_by"], - "sort_order": params["sort_order"], - "search": params["search"], - "date_start": params["date_start"], - "date_end": params["date_end"], - "capture_type": params["cap_type"], - "min_freq": params["min_freq"], - "max_freq": params["max_freq"], - "items_per_page": params["items_per_page"], - "visualization_compatibility": visualization_compatibility, - }, - ) - - -class CapturesAPIView(Auth0LoginRequiredMixin, View): - """Handle API/JSON requests for captures search.""" - - def _extract_request_params(self, request): - """Extract and return request parameters for API view.""" - return { - "sort_by": request.GET.get("sort_by", "created_at"), - "sort_order": request.GET.get("sort_order", "desc"), - "search": request.GET.get("search", ""), - "date_start": request.GET.get("date_start", ""), - "date_end": request.GET.get("date_end", ""), - "cap_type": request.GET.get("capture_type", ""), - "min_freq": request.GET.get("min_freq", ""), - "max_freq": request.GET.get("max_freq", ""), - } - - def get(self, request, *args, **kwargs) -> JsonResponse: - """Handle AJAX requests for the captures API.""" - - try: - # Extract and validate parameters - params = self._extract_request_params(request) - - # Get filtered and sorted captures with API limit applied before union - captures_list = _get_filtered_and_sorted_captures( - request.user, params, limit=API_CAPTURES_LIMIT - ) - - try: - captures_data = _get_captures_for_template(captures_list, request) - # remove the Capture model instance from each - # capture_data dict for JSON serialization - for capture_data in captures_data: - capture_data.pop("capture", None) - except Exception as e: - log.exception(f"Error in _get_captures_for_template: {e}") - msg = f"Error getting capture data: {e!s}" - raise ValueError(msg) from e - - response_data = { - "captures": captures_data, - "has_results": len(captures_data) > 0, - "total_count": len(captures_data), - } - return JsonResponse(response_data) - - except (ValueError, TypeError) as e: - error_msg = str(e) - log.warning( - f"Invalid parameter in captures API request: {error_msg}", - exc_info=True, - ) - return JsonResponse( - {"error": f"Invalid search parameters: {error_msg}"}, - status=400, - ) - except DatabaseError: - log.exception("Database error in captures API request") - return JsonResponse({"error": "Database error occurred"}, status=500) - - -user_capture_list_view = ListCapturesView.as_view() -user_captures_api_view = CapturesAPIView.as_view() - - -class KeywordAutocompleteAPIView(Auth0LoginRequiredMixin, View): - """Handle API requests for keyword autocomplete suggestions.""" - - def get(self, request, *args, **kwargs) -> JsonResponse: - """ - Return keyword suggestions based on search query. - - Returns up to 10 unique keyword suggestions that match the query - anywhere in the keyword. - """ - query = request.GET.get("q", "").strip() - - if not query: - return JsonResponse({"suggestions": []}) - - try: - # Search for keywords that contain the query anywhere (case-insensitive) - keywords = Keyword.objects.filter( - name__icontains=query, - is_deleted=False, - ).values_list("name", flat=True)[:10] - - return JsonResponse({"suggestions": list(keywords)}) - - except DatabaseError: - log.exception("Database error in keyword autocomplete") - return JsonResponse({"error": "Database error occurred"}, status=500) - - -keyword_autocomplete_api_view = KeywordAutocompleteAPIView.as_view() - - -@dataclass -class AddToDatasetReport: - """Result of adding capture(s) to a dataset.""" - - added: list[UUID] - skipped: list[UUID] - errors: list[str] - - -def _get_dataset_editable_by_user( - user: "User", dataset_uuid: UUID | str -) -> tuple["Dataset | None", tuple[str, int] | None]: - """ - Validate that the dataset exists, is editable (not deleted/public/final), - and the user has permission to edit or add assets. - - Returns (dataset, None) on success, (None, (error_message, status_code)) on failure. - Caller may format the error as redirect + messages or as JsonResponse. - """ - uuid_val = ( - dataset_uuid if isinstance(dataset_uuid, UUID) else UUID(str(dataset_uuid)) - ) - - if not user_has_access_to_item(user, uuid_val, ItemType.DATASET): - return (None, ("Dataset not found or access denied.", 404)) - - dataset = Dataset.objects.filter(uuid=uuid_val, is_deleted=False).first() - if not dataset: - return (None, ("Dataset not found or access denied.", 404)) - - if dataset.status == DatasetStatus.FINAL or dataset.is_public: - return (None, ("This dataset is published and cannot be edited.", 403)) - - if not UserSharePermission.user_can_edit_dataset( - user, uuid_val, ItemType.DATASET - ) and not UserSharePermission.user_can_add_assets(user, uuid_val, ItemType.DATASET): - return (None, ("You don't have permission to edit this dataset.", 403)) - - return (dataset, None) - - -class QuickAddCaptureToDatasetView(Auth0LoginRequiredMixin, View): - """ - Quick-add view: add a single capture (and all its channels if multi-channel) - owned by the user to a dataset the user can modify. - Expects POST with JSON body: - {"dataset_uuid": "", "capture_uuid": ""} - Returns error if capture_uuid or dataset_uuid is missing/invalid. - Returns error if user does not have permission to add captures to the dataset. - On success returns added/skipped UUIDs and errors (skipped = already in dataset). - """ - - def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> JsonResponse: - data, error = self._parse_json(request) - if error: - return error - assert data is not None # type narrowing: success path only - - # Validate both UUIDs before any ORM access - dataset_uuid, err = self._validate_uuid( - data.get("dataset_uuid"), "dataset_uuid" - ) - if err: - return err - capture_uuid, err = self._validate_uuid( - data.get("capture_uuid"), "capture_uuid" - ) - if err: - return err - assert dataset_uuid is not None - assert capture_uuid is not None - - # Fetch capture: must be owned by user and not deleted - capture = Capture.objects.filter( - uuid=capture_uuid, - owner=request.user, - is_deleted=False, - ).first() - if not capture: - return JsonResponse( - {"error": "Capture not found"}, - status=status.HTTP_404_NOT_FOUND, - ) - - # Dataset: must exist, not deleted, editable, and user must have permission - dataset, err = _get_dataset_editable_by_user( - cast("User", request.user), dataset_uuid - ) - if err: - message, status_code = err - return JsonResponse({"error": message}, status=status_code) - assert dataset is not None - - report = self._add_capture_to_dataset_with_report( - capture=capture, - dataset=dataset, - user=cast("User", request.user), - ) - return JsonResponse( - { - "success": True, - "added": [str(u) for u in report.added], - "skipped": [str(u) for u in report.skipped], - "errors": report.errors, - }, - status=status.HTTP_200_OK, - ) - - def _add_capture_to_dataset_with_report( - self, - capture: "Capture", - dataset: "Dataset", - user: "User", - ) -> AddToDatasetReport: - """ - Add a single capture (and its multi-channel siblings if applicable) to a - dataset. Skips captures already in the dataset. Internal use only. - - Returns an AddToDatasetReport with added, skipped, and errors lists. - """ - added: list[UUID] = [] - skipped: list[UUID] = [] - errors: list[str] = [] - - if capture.is_multi_channel: - candidates = list( - Capture.objects.filter( - top_level_dir=capture.top_level_dir, - owner=user, - is_deleted=False, - ) - ) - else: - candidates = [capture] - - existing_pks = set( - dataset.captures.filter(pk__in=[c.pk for c in candidates]).values_list( - "pk", flat=True - ) - ) - - for c in candidates: - if c.pk in existing_pks: - skipped.append(c.uuid) - continue - try: - dataset.captures.add(c) - added.append(c.uuid) - except OperationalError as e: - errors.append(f"{c.uuid}: {e}") - except IntegrityError as e: - errors.append(f"{c.uuid}: {e}") - except Exception as e: # noqa: BLE001 - catch-all for unexpected errors - errors.append(f"{c.uuid}: {e}") - - return AddToDatasetReport(added=added, skipped=skipped, errors=errors) - - def _parse_json( - self, request: HttpRequest - ) -> tuple[dict[str, Any] | None, JsonResponse | None]: - """ - Parse request body as JSON and require it to be an object. - Returns (data, None) on success, (None, error_response) on error. - Caller should do: data, err = self._parse_json(request); if err: return err - """ - if not request.content_type or "application/json" not in request.content_type: - return None, JsonResponse( - {"error": "Content-Type must be application/json"}, - status=status.HTTP_400_BAD_REQUEST, - ) - try: - data = json.loads(request.body) - except (json.JSONDecodeError, TypeError, ValueError): - return None, JsonResponse( - {"error": "Invalid JSON body"}, status=status.HTTP_400_BAD_REQUEST - ) - if not isinstance(data, dict): - return None, JsonResponse( - {"error": "Request body must be a JSON object"}, - status=status.HTTP_400_BAD_REQUEST, - ) - return data, None - - def _validate_uuid( - self, value: Any, field_name: str = "uuid" - ) -> tuple[UUID | None, JsonResponse | None]: - """ - Validate capture_uuid or dataset_uuid from request data. - Returns (uuid, None) on success, - (None, error_response) on validation failure. - """ - if value is None: - return None, JsonResponse( - {"error": f"{field_name} is required"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - try: - s = value.strip() # will fail if not a string - return UUID(s), None # handles validation and conversion to UUID - - except (AttributeError, ValueError, TypeError): - return None, JsonResponse( - {"error": f"Invalid {field_name}"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - -quick_add_capture_to_dataset_view = QuickAddCaptureToDatasetView.as_view() - - -class GroupCapturesView( - Auth0LoginRequiredMixin, FormSearchMixin, FileTreeMixin, TemplateView -): - template_name = "users/group_captures.html" - - # ========== Helper Methods ========== - - def _parse_dataset_uuid( - self, dataset_uuid_str: str, *, raise_on_error: bool = False - ) -> UUID | None: - """ - Parse dataset UUID string with consistent error handling. - - Args: - dataset_uuid_str: String representation of UUID - raise_on_error: If True, raises Http404 on error. If False, returns None. - - Returns: - UUID object if valid, None if invalid (when raise_on_error=False) - - Raises: - Http404: If raise_on_error=True and UUID is invalid - """ - try: - return UUID(dataset_uuid_str) - except ValueError as err: - if raise_on_error: - msg = "Invalid dataset UUID." - raise Http404(msg) from err - return None - - def _parse_comma_separated_ids(self, value: str) -> list[str]: - """ - Parse comma-separated IDs from a string. - - Args: - value: Comma-separated string of IDs - - Returns: - List of trimmed, non-empty IDs - """ - if not value: - return [] - return [item_id.strip() for item_id in value.split(",") if item_id.strip()] - - def _get_error_response( - self, - message: str | None = None, - errors: dict | None = None, - status_code: int = 400, - ) -> JsonResponse: - """ - Create standardized error response. - - Args: - message: Single error message (for non_field_errors) - errors: Dictionary of field errors - status_code: HTTP status code - - Returns: - JsonResponse with standardized error format - """ - if message: - return JsonResponse( - { - "success": False, - "errors": {"non_field_errors": [message]}, - }, - status=status_code, - ) - if errors: - return JsonResponse( - {"success": False, "errors": errors}, - status=status_code, - ) - return JsonResponse( - {"success": False, "errors": {"non_field_errors": ["An error occurred."]}}, - status=status_code, - ) - - def _get_dataset( - self, dataset_uuid: UUID, user: User | None = None, *, raise_404: bool = True - ) -> Dataset | None: - """ - Safely retrieve a dataset with consistent error handling. - - Args: - dataset_uuid: UUID of the dataset - user: Optional user to filter by owner - raise_404: If True, raises Http404 if not found. If False, returns None. - - Returns: - Dataset object if found, None if not found (when raise_404=False) - - Raises: - Http404: If raise_404=True and dataset not found - """ - if raise_404: - if user: - return get_object_or_404(Dataset, uuid=dataset_uuid, owner=user) - return get_object_or_404(Dataset, uuid=dataset_uuid) - - # When raise_404=False, use objects.get() to return None instead of raising - try: - filters = {"uuid": dataset_uuid} - if user: - filters["owner"] = user - return Dataset.objects.get(**filters) - except Dataset.DoesNotExist: - return None - - def _get_capture( - self, capture_id: str, user: User | None = None, *, require_owner: bool = False - ) -> Capture | None: - """ - Safely retrieve a capture with consistent error handling. - - Args: - capture_id: UUID string of the capture - user: Optional user to filter by owner - require_owner: If True, only returns captures owned by user - - Returns: - Capture object if found and accessible, None otherwise - """ - try: - filters = {"uuid": capture_id, "is_deleted": False} - if require_owner and user: - filters["owner"] = user - # Additional check if user provided but require_owner is False - # Still allow if user has access (for shared captures) - return Capture.objects.get(**filters) - except Capture.DoesNotExist: - return None - - def _get_file( - self, file_id: str, user: User | None = None, *, require_owner: bool = False - ) -> File | None: - """ - Safely retrieve a file with consistent error handling. - - Args: - file_id: UUID string of the file - user: Optional user to filter by owner - require_owner: If True, only returns files owned by user - - Returns: - File object if found and accessible, None otherwise - """ - try: - filters = {"uuid": file_id, "is_deleted": False} - if require_owner and user: - filters["owner"] = user - return File.objects.get(**filters) - except File.DoesNotExist: - return None - - def _process_keywords(self, dataset: Dataset, raw_keywords: str) -> None: - """ - Process and associate keywords with a dataset. - - Args: - dataset: Dataset to associate keywords with - raw_keywords: Comma-separated string of keywords - """ - if not raw_keywords: - return - - # Slugify and deduplicate keywords - slugified_keywords = { - slugify(p.strip()) - for p in raw_keywords.split(",") - if p.strip() and slugify(p.strip()) - } - - # Get or create keywords and associate them with the dataset - for slug in slugified_keywords: - keyword, _created = Keyword.objects.get_or_create(name=slug) - keyword.datasets.add(dataset) - - def _get_permission_cache( - self, user: User, dataset_uuid: UUID - ) -> dict[str, bool | PermissionLevel | None]: - """ - Get and cache permission information for a user and dataset. - - Args: - user: User to check permissions for - dataset_uuid: UUID of the dataset - - Returns: - Dictionary with permission information - """ - return { - "has_access": user_has_access_to_item(user, dataset_uuid, ItemType.DATASET), - "permission_level": get_user_permission_level( - user, dataset_uuid, ItemType.DATASET - ), - "can_edit_dataset": UserSharePermission.user_can_edit_dataset( - user, dataset_uuid, ItemType.DATASET - ), - "can_add_assets": UserSharePermission.user_can_add_assets( - user, dataset_uuid, ItemType.DATASET - ), - "can_remove_assets": UserSharePermission.user_can_remove_assets( - user, dataset_uuid, ItemType.DATASET - ), - "can_remove_others_assets": ( - UserSharePermission.user_can_remove_others_assets( - user, dataset_uuid, ItemType.DATASET - ) - ), - } - - # ========== View Methods ========== - - def get(self, request, *args, **kwargs): - """Handle GET request with permission checking and AJAX requests.""" - dataset_uuid = request.GET.get("dataset_uuid") - - # Validate dataset permissions if editing - if dataset_uuid: - validation_error = self._validate_dataset_edit_permissions( - request, dataset_uuid - ) - if validation_error: - return validation_error - - # Handle AJAX requests - if request.headers.get("X-Requested-With") == "XMLHttpRequest": - ajax_response = self._handle_ajax_request(request) - if ajax_response: - return ajax_response - - return super().get(request, *args, **kwargs) - - def _validate_dataset_edit_permissions( - self, request: HttpRequest, dataset_uuid: str - ) -> HttpResponseRedirect | None: - """Validate user permissions for editing a dataset.""" - try: - uuid_val = UUID(dataset_uuid) - except (ValueError, TypeError): - messages.error(request, "Dataset not found or access denied.") - return redirect("users:dataset_list") - - _, err = _get_dataset_editable_by_user(request.user, uuid_val) - if err: - message, _ = err - messages.error(request, message) - return redirect("users:dataset_list") - - return None - - def _handle_ajax_request(self, request: HttpRequest) -> JsonResponse | None: - """Handle AJAX requests for search operations.""" - try: - if "search_captures" in request.GET: - return self._handle_capture_search(request) - - if "search_files" in request.GET: - return self._handle_file_search(request) - - except (OSError, DatabaseError) as e: - return JsonResponse({"error": str(e)}, status=500) - - return None - - def _handle_capture_search(self, request: HttpRequest) -> JsonResponse: - """Handle AJAX request for capture search.""" - form = CaptureSearchForm(request.GET) - if form.is_valid(): - captures = self.search_captures(form.cleaned_data, request) - return JsonResponse(self.get_paginated_response(captures, request)) - return self._get_error_response(errors=form.errors, status_code=400) - - def _handle_file_search(self, request: HttpRequest) -> JsonResponse: - """Handle AJAX request for file search.""" - base_dir = sanitize_path_rel_to_user( - unsafe_path="/", - request=request, - ) - - form = FileSearchForm(request.GET, user=request.user) - if form.is_valid(): - files = self.search_files(form.cleaned_data, request) - tree_data = self._get_directory_tree(files, str(base_dir)) - - return JsonResponse( - { - "tree": tree_data, - "extension_choices": form.fields["file_extension"].choices, - "search_values": { - "file_name": form.cleaned_data.get("file_name", ""), - "file_extension": form.cleaned_data.get("file_extension", ""), - "directory": form.cleaned_data.get("directory", ""), - }, - }, - ) - return self._get_error_response(errors=form.errors, status_code=400) - - def search_captures(self, search_data, request) -> list[Capture]: - """Override to only return captures owned by the user for dataset creation.""" - # Only get captures owned by the user (no shared captures) - queryset = Capture.objects.filter( - owner=request.user, - is_deleted=False, - ) - - # Build a Q object for complex queries - q_objects = Q() - - if search_data.get("directory"): - q_objects &= Q(top_level_dir__icontains=search_data["directory"]) - if search_data.get("capture_type"): - q_objects &= Q(capture_type=search_data["capture_type"]) - if search_data.get("scan_group"): - q_objects &= Q(scan_group__icontains=search_data["scan_group"]) - if search_data.get("channel"): - q_objects &= Q(channel__icontains=search_data["channel"]) - - queryset = queryset.filter(q_objects).order_by("-created_at") - - # Use utility function to deduplicate composite captures - return deduplicate_composite_captures(list(queryset)) - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - base_dir = sanitize_path_rel_to_user( - unsafe_path="/", - request=self.request, - ) - - # Check if we're editing an existing dataset - dataset_uuid_str = self.request.GET.get("dataset_uuid", None) - existing_dataset = None - permission_level = None - is_owner = False - dataset_uuid = None - - if dataset_uuid_str: - dataset_uuid = self._parse_dataset_uuid( - dataset_uuid_str, raise_on_error=True - ) - - # Check if user has access to this dataset - if not user_has_access_to_item( - self.request.user, dataset_uuid, ItemType.DATASET - ): - msg = "Dataset not found or access denied." - raise Http404(msg) - - # Get the dataset - it exists and user has access - existing_dataset = self._get_dataset(dataset_uuid, raise_404=True) - permission_level = get_user_permission_level( - self.request.user, dataset_uuid, ItemType.DATASET - ) - is_owner = existing_dataset.owner == self.request.user - else: - # For new dataset creation, user is always the owner - permission_level = PermissionLevel.OWNER - is_owner = True - - # Get form - if self.request.method == "POST": - dataset_form = DatasetInfoForm(self.request.POST, user=self.request.user) - else: - initial_data = {} - if existing_dataset: - authors_json = self._set_authors_el_ids( - existing_dataset.get_authors_display() - ) - - initial_data = { - "name": existing_dataset.name, - "description": existing_dataset.description, - "keywords": ", ".join( - existing_dataset.keywords.values_list("name", flat=True) - ), - "authors": authors_json, - "status": existing_dataset.status, - "is_public": existing_dataset.is_public, - } - dataset_form = DatasetInfoForm(user=self.request.user, initial=initial_data) - - selected_files, selected_files_details = self._get_file_context( - base_dir=base_dir, existing_dataset=existing_dataset - ) - selected_captures, selected_captures_details = self._get_capture_context( - existing_dataset=existing_dataset - ) - - # Add to context - context.update( - { - "dataset_form": dataset_form, - "capture_search_form": CaptureSearchForm(), - "file_search_form": FileSearchForm(user=self.request.user), - "selected_captures": json.dumps( - selected_captures, cls=DjangoJSONEncoder - ), - "selected_files": json.dumps(selected_files, cls=DjangoJSONEncoder), - "form": dataset_form, - "existing_dataset": existing_dataset, - "selected_captures_details_json": json.dumps( - selected_captures_details, cls=DjangoJSONEncoder - ), - "selected_files_details_json": json.dumps( - selected_files_details, cls=DjangoJSONEncoder - ), - "permission_level": permission_level, - "is_owner": is_owner, - "can_edit_metadata": ( - True - if not dataset_uuid - else UserSharePermission.user_can_edit_dataset( - self.request.user, - dataset_uuid, - ItemType.DATASET, - ) - ), - "can_add_assets": ( - True - if not dataset_uuid - else UserSharePermission.user_can_add_assets( - self.request.user, - dataset_uuid, - ItemType.DATASET, - ) - ), - "can_remove_assets": ( - True - if not dataset_uuid - else UserSharePermission.user_can_remove_assets( - self.request.user, - dataset_uuid, - ItemType.DATASET, - ) - ), - } - ) - return context - - def post(self, request, *args, **kwargs): - """Handle dataset creation/update with selected captures and files.""" - try: - dataset_uuid_str = request.GET.get("dataset_uuid") - dataset_form = DatasetInfoForm(request.POST, user=request.user) - - # Validate form and get selected items - validation_result = self._validate_dataset_form( - request, - dataset_form, - dataset_uuid_str, - ) - if validation_result: - return validation_result - - if dataset_uuid_str: - # Get dataset UUID format - dataset_uuid = self._parse_dataset_uuid( - dataset_uuid_str, raise_on_error=False - ) - if not dataset_uuid: - return self._get_error_response( - message="Invalid dataset UUID.", status_code=400 - ) - - # Handle dataset editing - return self._handle_dataset_edit(request, dataset_form, dataset_uuid) - # Handle dataset creation - return self._handle_dataset_creation(request, dataset_form) - - except (DatabaseError, IntegrityError) as e: - log.exception("Database error in dataset creation") - return self._get_error_response(message=str(e), status_code=500) - except ValueError: - # Handle UUID parsing errors - return self._get_error_response( - message="Invalid dataset UUID.", status_code=400 - ) - - def _validate_dataset_form( - self, - request: HttpRequest, - dataset_form: DatasetInfoForm, - dataset_uuid_str: str | None = None, - ) -> JsonResponse | None: - """Validate the dataset form and return error response if invalid.""" - # Check if this is an edit operation first - - if dataset_uuid_str: - dataset_uuid = self._parse_dataset_uuid( - dataset_uuid_str, raise_on_error=False - ) - if not dataset_uuid: - messages.error(request, "Invalid dataset UUID.") - return redirect("users:dataset_list") - - # For editing, validate permissions first - permission_level = get_user_permission_level( - request.user, dataset_uuid, ItemType.DATASET - ) - - if not permission_level: - return self._get_error_response( - message="Access denied.", status_code=403 - ) - - # Only validate form if user can edit metadata - can_edit = UserSharePermission.user_can_edit_dataset( - request.user, dataset_uuid, ItemType.DATASET - ) - - if can_edit: - if not dataset_form.is_valid(): - return self._get_error_response( - errors=dataset_form.errors, status_code=400 - ) - # If user can't edit metadata, skip form validation - else: - # For new dataset creation, always validate form - if not dataset_form.is_valid(): - return self._get_error_response( - errors=dataset_form.errors, status_code=400 - ) - - # Get selected assets - selected_captures, selected_files = self._get_asset_selections(request) - - # Validate that at least one capture or file is selected - if len(selected_captures) == 0 and len(selected_files) == 0: - return self._get_error_response( - message="Please select at least one capture or file.", - status_code=400, - ) - - return None - - def _set_authors_el_ids(self, authors: list) -> str: - """Set the author element IDs for the page lifecycle in edit mode.""" - for author in authors: - author["_stableId"] = str(uuid.uuid4()) - return json.dumps(authors) - - def _handle_dataset_creation( - self, - request: HttpRequest, - dataset_form: DatasetInfoForm, - ) -> JsonResponse: - """Handle dataset creation.""" - - # Create dataset - dataset = self._create_or_update_dataset(request, dataset_form, dataset=None) - - # Get selected assets - selected_captures, selected_files = self._get_asset_selections(request) - - # Add captures to dataset - capture_error = self._add_captures_to_dataset( - dataset, selected_captures, request - ) - if capture_error: - return capture_error - - # Add files to dataset - self._add_files_to_dataset(dataset, selected_files) - - # Return success response with redirect URL - return JsonResponse( - {"success": True, "redirect_url": reverse("users:dataset_list")}, - ) - - def _handle_dataset_edit( - self, request, dataset_form: DatasetInfoForm, dataset_uuid: UUID - ) -> JsonResponse: - """Handle dataset editing with asset management.""" - - # Get dataset - dataset = self._get_dataset(dataset_uuid, user=request.user, raise_404=True) - - # Update metadata if user has permission - if UserSharePermission.user_can_edit_dataset( - request.user, dataset_uuid, ItemType.DATASET - ): - self._create_or_update_dataset(request, dataset_form, dataset) - - # Handle asset changes - asset_changes = self._parse_asset_changes(request) - - # Apply asset changes based on permissions - self._apply_asset_changes( - dataset, - asset_changes, - request.user, - ) - - return JsonResponse( - {"success": True, "redirect_url": reverse("users:dataset_list")}, - ) - - def _parse_asset_changes(self, request) -> dict: - """Parse asset changes from the request.""" - changes: dict[str, dict[str, list[str]]] = { - "captures": {"add": [], "remove": []}, - "files": {"add": [], "remove": []}, - } - - # Parse captures changes - changes["captures"]["add"] = self._parse_comma_separated_ids( - request.POST.get("captures_add", "") - ) - changes["captures"]["remove"] = self._parse_comma_separated_ids( - request.POST.get("captures_remove", "") - ) - - # Parse files changes - changes["files"]["add"] = self._parse_comma_separated_ids( - request.POST.get("files_add", "") - ) - changes["files"]["remove"] = self._parse_comma_separated_ids( - request.POST.get("files_remove", "") - ) - - return changes - - def _apply_asset_changes( # noqa: C901 - self, - dataset: Dataset, - changes: dict, - user: User, - ): - """Apply asset changes based on user permissions.""" - # Cache permissions to avoid repeated queries - permissions = self._get_permission_cache(user, dataset.uuid) - - # Process captures and files using the same pattern - asset_types = [ - ("captures", Capture, dataset.captures), - ("files", File, dataset.files), - ] - - for asset_type_name, _asset_model, asset_relation in asset_types: - # Add assets - if permissions["can_add_assets"]: - for asset_id in changes[asset_type_name]["add"]: - if asset_type_name == "captures": - asset = self._get_capture( - asset_id, user=user, require_owner=True - ) - else: - asset = self._get_file(asset_id, user=user, require_owner=True) - - if asset: - asset_relation.add(asset) - - # Remove assets - if permissions["can_remove_assets"]: - for asset_id in changes[asset_type_name]["remove"]: - if asset_type_name == "captures": - asset = self._get_capture( - asset_id, user=None, require_owner=False - ) - else: - asset = self._get_file(asset_id, user=None, require_owner=False) - - if asset: - # Check if user can remove this asset - can_remove = ( - asset.owner == user - or permissions["can_remove_others_assets"] - ) - if can_remove: - asset_relation.remove(asset) - - def _apply_author_changes(self, authors: list, changes: dict) -> list: - """Apply author changes based on the changes tracking.""" - result = [] - - # Process each author index - for i, author in enumerate(authors): - # Skip if marked for removal - if i in changes.get("removed", []): - continue - - # Apply modifications if any - if i in changes.get("modified", {}): - modified_author = ( - author.copy() - if isinstance(author, dict) - else {"name": author, "orcid_id": ""} - ) - for field, change_data in changes["modified"][i].items(): - modified_author[field] = change_data.get( - "new", modified_author.get(field, "") - ) - result.append(modified_author) - else: - result.append(author) - - # Add new authors - only add those that aren't already in the result - # The 'added' array contains indices of newly added authors in the - # current authors array - added_indices = changes.get("added", []) - for i in added_indices: - if i < len(authors): - new_author = authors[i] - # Check if this author is already in result (shouldn't be, - # but safety check) - # Convert to comparable format - new_author_name = ( - new_author.get("name", "") - if isinstance(new_author, dict) - else str(new_author) - ) - new_author_orcid = ( - new_author.get("orcid_id", "") - if isinstance(new_author, dict) - else "" - ) - - # Only add if not already present (by name and orcid) - is_duplicate = any( - ( - isinstance(a, dict) - and a.get("name") == new_author_name - and a.get("orcid_id") == new_author_orcid - ) - or (not isinstance(a, dict) and str(a) == new_author_name) - for a in result - ) - - if not is_duplicate: - result.append(new_author) - - return result - - def _get_asset_selections( - self, - request: HttpRequest, - ) -> tuple[list[str], list[str]]: - """ - Get selected assets from the request. - This function is used to get the selected assets on creation only. - """ - selected_captures = request.POST.get("selected_captures", "").split(",") - selected_files = request.POST.get("selected_files", "").split(",") - return selected_captures, selected_files - - def _create_or_update_dataset( - self, - request: HttpRequest, - dataset_form: DatasetInfoForm, - dataset: Dataset | None = None, - ) -> Dataset: - """Create a new dataset or update an existing one.""" - if dataset: - dataset.name = dataset_form.cleaned_data["name"] - dataset.description = dataset_form.cleaned_data["description"] - - # Parse authors from JSON string - authors_json = dataset_form.cleaned_data["authors"] - authors = json.loads(authors_json) - - # Parse author changes if provided - author_changes_json = request.POST.get("author_changes", "") - if author_changes_json: - try: - author_changes = json.loads(author_changes_json) - # Apply author changes - authors = self._apply_author_changes(authors, author_changes) - except json.JSONDecodeError: - # Fallback to direct authors if parsing fails - pass - - dataset.authors = authors - dataset.status = dataset_form.cleaned_data["status"] - dataset.is_public = dataset_form.cleaned_data.get("is_public", False) - dataset.save() - - # Clear existing keyword relationships (not the keywords themselves) - dataset.keywords.clear() - else: - # Create new dataset - # Parse authors from JSON string - authors_json = dataset_form.cleaned_data["authors"] - authors = json.loads(authors_json) - dataset = Dataset.objects.create( - name=dataset_form.cleaned_data["name"], - description=dataset_form.cleaned_data["description"], - authors=authors, - status=dataset_form.cleaned_data["status"], - is_public=dataset_form.cleaned_data.get("is_public", False), - owner=request.user, - ) - - # Persist keywords from form (comma-separated) - raw_keywords = dataset_form.cleaned_data.get("keywords", "") or "" - self._process_keywords(dataset, raw_keywords) - - return dataset - - def _add_captures_to_dataset( - self, dataset: Dataset, selected_captures: list[str], request - ) -> JsonResponse | None: - """Add selected captures to the dataset.""" - if not selected_captures[0]: - return None - - for capture_id in selected_captures: - if not capture_id: - continue - try: - # Only allow adding captures owned by the user - capture = Capture.objects.get( - uuid=capture_id, owner=request.user, is_deleted=False - ) - if capture.is_multi_channel: - # Add all captures in this composite - all_captures = Capture.objects.filter( - top_level_dir=capture.top_level_dir, - owner=request.user, - is_deleted=False, - ) - dataset.captures.add(*all_captures) - else: - dataset.captures.add(capture) - except Capture.DoesNotExist: - return JsonResponse( - { - "success": False, - "errors": { - "non_field_errors": [ - f"Capture {capture_id} not found or you don't have " - "permission to add it to a dataset.", - ], - }, - }, - status=400, - ) - - return None - - def _add_files_to_dataset( - self, dataset: Dataset, selected_files: list[str] - ) -> None: - """Add selected files to the dataset.""" - if selected_files[0]: - files = File.objects.filter( - uuid__in=selected_files, - owner=self.request.user, - ) - dataset.files.add(*files) - - def _get_file_context( - self, - base_dir: Path | None = None, - existing_dataset: Dataset | None = None, - ) -> tuple[list[dict[str, Any]], dict[str, Any]]: - selected_files: list[dict[str, Any]] = [] - selected_files_details: dict[str, Any] = {} - if not existing_dataset: - return selected_files, selected_files_details - - files_queryset = existing_dataset.files.filter( - is_deleted=False, - ) - - # Prepare file details for JavaScript - for selected_file in files_queryset: - rel_path = ( - f"{selected_file.directory.replace(str(base_dir), '')}" - if base_dir - else None - ) - file_dict = self.serialize_item(selected_file, rel_path) - selected_files.append(file_dict) - - selected_files_details[str(selected_file.uuid)] = file_dict - - return selected_files, selected_files_details - - def _get_capture_context( - self, existing_dataset: Dataset | None = None - ) -> tuple[list[dict[str, Any]], dict[str, Any]]: - selected_captures: list[dict[str, Any]] = [] - selected_captures_details: dict[str, Any] = {} - composite_capture_dirs: set[str] = set() - if existing_dataset: - captures_queryset = existing_dataset.captures.filter( - is_deleted=False, - ) - - # Only include one composite per group - for capture in captures_queryset.order_by("-created_at"): - if capture.is_multi_channel: - if capture.top_level_dir not in composite_capture_dirs: - capture_dict = self.serialize_item(capture) - capture_uuid = str(capture_dict["id"]) - selected_captures.append(capture_dict) - selected_captures_details[capture_uuid] = capture_dict - composite_capture_dirs.add(capture.top_level_dir) - else: - capture_dict = self.serialize_item(capture) - capture_uuid = str(capture_dict["id"]) - selected_captures.append(capture_dict) - selected_captures_details[capture_uuid] = capture_dict - - return selected_captures, selected_captures_details - - -user_group_captures_view = GroupCapturesView.as_view() - - -def filter_by_frequency_range( - datasets: QuerySet[Dataset], - min_freq: float | None, - max_freq: float | None, -) -> QuerySet[Dataset]: - """Filter datasets by frequency range of their captures. - - Reuses the existing _apply_frequency_filters_to_list function - to filter captures, then maps back to datasets. - """ - if min_freq is None and max_freq is None: - return datasets - - # Get dataset UUIDs - dataset_uuids = list(datasets.values_list("uuid", flat=True)) - if not dataset_uuids: - return datasets.none() - - # Get all captures for these datasets and convert to list - captures_qs = Capture.objects.filter( - dataset__uuid__in=dataset_uuids, is_deleted=False - ) - captures_list = list(captures_qs.iterator(chunk_size=1000)) - if not captures_list: - return datasets.none() - - # Use existing frequency filter function - filtered_captures = _apply_frequency_filters_to_list( - captures_list=captures_list, - min_freq=min_freq, - max_freq=max_freq, - ) - - # Get dataset IDs from filtered captures - matching_dataset_ids = { - capture.dataset_id - for capture in filtered_captures - if capture.dataset_id is not None - } - if not matching_dataset_ids: - return datasets.none() - - # Get dataset UUIDs from IDs and filter the queryset - matching_dataset_uuids = set( - Dataset.objects.filter(id__in=matching_dataset_ids).values_list( - "uuid", flat=True - ) - ) - return datasets.filter(uuid__in=matching_dataset_uuids) - - -def serialize_datasets_for_user( - datasets: QuerySet[Dataset], user: User | None -) -> list[dict[str, Any]]: - """Serialize datasets for display with user context. - - Args: - datasets: QuerySet of Dataset objects to serialize - user: User object or None for anonymous users - - Returns: - List of serialized dataset dictionaries - """ - serialized_datasets = [] - for dataset in datasets: - # Create a mock request object for the serializer context - context_req = { - "request": type( - "Request", - (), - {"user": user if user and user.is_authenticated else None}, - )() - } - dataset_data = cast( - "ReturnDict", DatasetGetSerializer(dataset, context=context_req).data - ) - dataset_data["dataset"] = dataset - serialized_datasets.append(dataset_data) - return serialized_datasets - - -def get_published_datasets() -> QuerySet[Dataset]: - """Get all published datasets (status=FINAL or is_public=True).""" - return ( - Dataset.objects.filter( - status=DatasetStatus.FINAL, - is_public=True, - is_deleted=False, - ) - .prefetch_related("keywords", "owner") - .distinct() - .order_by("-created_at") - ) - - -def apply_search_filters( - datasets: QuerySet[Dataset], - form_data: dict[str, Any], -) -> QuerySet[Dataset]: - """Apply search filters to the dataset queryset.""" - query = form_data.get("query", "").strip() - keywords_str = form_data.get("keywords", "").strip() - min_freq = form_data.get("min_frequency") - max_freq = form_data.get("max_frequency") - - # Apply text search - if query: - datasets = datasets.filter( - Q(name__icontains=query) - | Q(abstract__icontains=query) - | Q(description__icontains=query) - | Q(authors__icontains=query) - | Q(doi__icontains=query) - ) - - # Apply keyword filter - if keywords_str: - # Split and slugify keywords - keyword_slugs = { - slugify(k.strip()) - for k in keywords_str.split(",") - if k.strip() and slugify(k.strip()) - } - if keyword_slugs: - datasets = datasets.filter(keywords__name__in=keyword_slugs).distinct() - - # Apply frequency range filter - if min_freq is not None or max_freq is not None: - datasets = filter_by_frequency_range(datasets, min_freq, max_freq) - - return datasets - - -class ListDatasetsView(Auth0LoginRequiredMixin, View): - template_name = "users/dataset_list.html" - - def get(self, request, *args, **kwargs) -> HttpResponse: - """Handle GET request for dataset list.""" - - sort_by, sort_order = self._get_sort_parameters(request) - order_by = self._build_order_by(sort_by, sort_order) - - owned_datasets = self._get_owned_datasets(request.user, order_by) - shared_datasets = self._get_shared_datasets(request.user, order_by) - - datasets_with_shared_users: list[dict] = [] # pyright: ignore[reportMissingTypeArgument] - datasets_with_shared_users.extend( - serialize_datasets_for_user(owned_datasets, request.user) - ) - datasets_with_shared_users.extend( - serialize_datasets_for_user(shared_datasets, request.user) - ) - page_obj = self._paginate_datasets(datasets_with_shared_users, request) - - # Check if this is an AJAX request - if request.headers.get("X-Requested-With") == "XMLHttpRequest": - # Return table and modals so the client can update both after list refresh - table_html = render_to_string( - "users/components/dataset_list_table.html", - { - "page_obj": page_obj, - "sort_by": sort_by, - "sort_order": sort_order, - "ajax_fragment": True, - }, - request=request, - ) - modals_html = render_to_string( - "users/components/dataset_list_modals.html", - {"page_obj": page_obj}, - request=request, - ) - # Separator used by ListRefreshManager to split table vs modals - list_refresh_sep = "" - return HttpResponse(table_html + list_refresh_sep + modals_html) - - return render( - request, - template_name=self.template_name, - context={ - "page_obj": page_obj, - "sort_by": sort_by, - "sort_order": sort_order, - }, - ) - - def _get_sort_parameters(self, request: HttpRequest) -> tuple[str, str]: - """Get sort parameters from request.""" - sort_by = request.GET.get("sort_by", "created_at") - sort_order = request.GET.get("sort_order", "desc") - return sort_by, sort_order - - def _build_order_by(self, sort_by: str, sort_order: str) -> str: - """Build order_by string for queryset.""" - allowed_sort_fields = {"name", "created_at", "updated_at", "authors"} - - if sort_by in allowed_sort_fields: - order_prefix = "-" if sort_order == "desc" else "" - return f"{order_prefix}{sort_by}" - - return "-created_at" - - def _get_owned_datasets(self, user: User, order_by: str) -> QuerySet[Dataset]: - """Get datasets owned by the user.""" - return ( - user.datasets.filter(is_deleted=False) - .prefetch_related("keywords") - .order_by(order_by) - ) - - def _get_shared_datasets(self, user: User, order_by: str) -> QuerySet[Dataset]: - """Get datasets shared with the user.""" - shared_permissions = UserSharePermission.objects.filter( - shared_with=user, - item_type=ItemType.DATASET, - is_deleted=False, - is_enabled=True, - ).select_related("owner") - - shared_dataset_uuids = [perm.item_uuid for perm in shared_permissions] - return ( - Dataset.objects.filter(uuid__in=shared_dataset_uuids, is_deleted=False) - .exclude(owner=user) - .prefetch_related("keywords") - .order_by(order_by) - ) - - def _paginate_datasets( - self, datasets: list[dict[str, Any]], request: HttpRequest - ) -> Any: - """Paginate the datasets list.""" - paginator = Paginator(datasets, per_page=15) - page_number = request.GET.get("page") - return paginator.get_page(page_number) - - -class SearchPublishedDatasetsView(Auth0LoginRequiredMixin, View): - """View for searching published datasets (public, no auth required).""" - - template_name = "users/published_datasets_list.html" - - def get(self, request: HttpRequest, *args, **kwargs) -> HttpResponse: - """Handle GET request for dataset search.""" - form = PublishedDatasetSearchForm(request.GET) - datasets = get_published_datasets() - - # Apply search filters - if form.is_valid(): - datasets = apply_search_filters( - datasets, - form.cleaned_data, - ) - - # Serialize datasets - serialized_datasets = serialize_datasets_for_user( - datasets, request.user if request.user.is_authenticated else None - ) - - # Paginate results - paginator = Paginator(serialized_datasets, per_page=15) - page_number = request.GET.get("page", 1) - try: - page_obj = paginator.get_page(page_number) - except (PageNotAnInteger, EmptyPage): - page_obj = paginator.get_page(1) - - return render( - request, - template_name=self.template_name, - context={ - "search_form": form, - "page_obj": page_obj, - }, - ) - - -def _apply_basic_filters( - qs: QuerySet[Capture], - search: str | None = None, - date_start: str | None = None, - date_end: str | None = None, - cap_type: str | None = None, -) -> QuerySet[Capture]: - """Apply basic filters: search, date range, and capture type.""" - if search: - # First get the base queryset with direct field matches - base_filter = ( - Q(name__icontains=search) - | Q(channel__icontains=search) - | Q(index_name__icontains=search) - | Q(capture_type__icontains=search) - | Q(uuid__icontains=search) - ) - - # Then add any captures where the display value matches - display_matches = [ - capture.pk - for capture in qs - if search.lower() in capture.get_capture_type_display().lower() - ] - - if display_matches: - base_filter |= Q(pk__in=display_matches) - - qs = qs.filter(base_filter) - - if date_start: - qs = qs.filter(created_at__gte=date_start) - if date_end: - qs = qs.filter(created_at__lte=date_end) - if cap_type: - qs = qs.filter(capture_type=cap_type) - - return qs - - -def _apply_sorting( - qs: QuerySet[Capture], - sort_by: str, - sort_order: str = "desc", -): - """Apply sorting to the queryset.""" - # Define allowed sort fields (actual database fields only) - allowed_sort_fields = { - "uuid", - "created_at", - "updated_at", - "deleted_at", - "is_deleted", - "is_public", - "channel", - "scan_group", - "capture_type", - "top_level_dir", - "index_name", - "owner", - "origin", - "dataset", - } - - # Handle computed properties with meaningful fallbacks - computed_field_fallbacks = { - # Could be enhanced with OpenSearch sorting later - "center_frequency_ghz": "created_at", - "sample_rate_mhz": "created_at", - } - - # Check if it's a computed field first - if sort_by in computed_field_fallbacks: - # For now, fall back to a meaningful sort field - # In the future, this could be enhanced to sort by OpenSearch data - fallback_field = computed_field_fallbacks[sort_by] - if sort_order == "desc": - return qs.order_by(f"-{fallback_field}") - return qs.order_by(fallback_field) - - # Only apply sorting if the field is allowed - if sort_by in allowed_sort_fields: - if sort_order == "desc": - return qs.order_by(f"-{sort_by}") - return qs.order_by(sort_by) - - # Default sorting if field is not recognized - return qs.order_by("-created_at") - - -user_dataset_list_view = ListDatasetsView.as_view() - - -class UserDatasetsForQuickAddView(Auth0LoginRequiredMixin, View): - """Return JSON list of datasets the user can add captures to (quick-add modal).""" - - def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> JsonResponse: - user = cast("User", request.user) - owned = ( - user.datasets.filter(is_deleted=False, is_public=False) - .exclude(status=DatasetStatus.FINAL) - .order_by("name") - .values("uuid", "name") - ) - shared_uuids = list( - UserSharePermission.objects.filter( - shared_with=user, - item_type=ItemType.DATASET, - is_deleted=False, - is_enabled=True, - permission_level__in=[ - PermissionLevel.CO_OWNER, - PermissionLevel.CONTRIBUTOR, - ], - ).values_list("item_uuid", flat=True) - ) - shared = ( - Dataset.objects.filter( - uuid__in=shared_uuids, is_deleted=False, is_public=False - ) - .exclude(owner=user) - .exclude(status=DatasetStatus.FINAL) - .order_by("name") - .values("uuid", "name") - ) - datasets = [ - {"uuid": str(d["uuid"]), "name": (d["name"] or "Unnamed")} - for d in chain(owned, shared) - ] - return JsonResponse({"datasets": datasets}) - - -user_datasets_for_quick_add_view = UserDatasetsForQuickAddView.as_view() - - -class HomePageView(TemplateView): - """View for the home page with search form and latest datasets.""" - - template_name = "pages/home.html" - - def get_context_data(self, **kwargs): - """Add search form and latest 5 public datasets to context.""" - context = super().get_context_data(**kwargs) - - # Get latest 5 public published datasets (is_public=True only) - latest_datasets = ( - Dataset.objects.filter( - is_public=True, - is_deleted=False, - ) - .prefetch_related("keywords", "owner") - .distinct() - .order_by("-created_at")[:5] - ) - - # Serialize datasets - serialized_datasets = [] - for dataset in latest_datasets: - context_req = { - "request": type( - "Request", - (), - { - "user": self.request.user - if self.request.user.is_authenticated - else None - }, - )() - } - dataset_data = cast( - "ReturnDict", DatasetGetSerializer(dataset, context=context_req).data - ) - dataset_data["dataset"] = dataset - serialized_datasets.append(dataset_data) - - context["search_form"] = PublishedDatasetSearchForm() - context["latest_datasets"] = serialized_datasets - return context - - -home_page_view = HomePageView.as_view() -user_search_datasets_view = SearchPublishedDatasetsView.as_view() - - -class PublishDatasetView(Auth0LoginRequiredMixin, View): - """View to handle dataset publishing (updating status and is_public).""" - - def post(self, request, dataset_uuid: str) -> JsonResponse: - """Handle POST request to publish a dataset.""" - # Get the dataset - dataset = get_object_or_404(Dataset, uuid=dataset_uuid) - - # Check if user has access - if not user_has_access_to_item(request.user, dataset_uuid, ItemType.DATASET): - return JsonResponse( - {"success": False, "error": "Access denied."}, status=403 - ) - - can_publish = UserSharePermission.user_can_edit_dataset( - request.user, dataset_uuid, ItemType.DATASET - ) - - if not can_publish: - return JsonResponse( - { - "success": False, - "error": "You do not have permission to publish this dataset.", - }, - status=403, - ) - - # Get status and is_public from request - status_value = request.POST.get("status") - - is_public_raw = request.POST.get("is_public") - if is_public_raw is None: - is_public_value = None - else: - try: - is_public_value = json.loads(is_public_raw) - except (json.JSONDecodeError, TypeError): - return JsonResponse( - { - "success": False, - "error": "Could not determine dataset visibility.", - }, - status=400, - ) - - error_messages = self._handle_400_errors( - dataset, - status_value, - is_public_value=is_public_value, - ) - if len(error_messages) > 0: - return JsonResponse( - {"success": False, "errors": {"non_field_errors": error_messages}}, - status=400, - ) - - # Update status if provided and dataset is not already final - if status_value: - dataset.status = status_value - - # Update is_public if provided and dataset is not already public - if is_public_value is not None: - dataset.is_public = is_public_value - - dataset.save() - - return JsonResponse( - { - "success": True, - "message": "Dataset updated successfully.", - "status": dataset.status, - "is_public": dataset.is_public, - } - ) - - def _handle_400_errors( - self, - dataset: Dataset, - status_value: str | None, - *, - is_public_value: bool | None, - ) -> list[str]: - """Handle status change.""" - - # Initialize error message - error_messages = [] - - # Validate that at least one field is being updated - if not status_value and is_public_value is None: - error_messages.append("No fields to update.") - # Validate status value - if status_value and status_value not in [ - DatasetStatus.DRAFT, - DatasetStatus.FINAL, - ]: - error_messages.append("Invalid status value.") - - # Update status if provided and dataset is not already final - if status_value: - if ( - dataset.status == DatasetStatus.FINAL - and status_value == DatasetStatus.DRAFT - ): - error_messages.append( - "Cannot change published dataset status back to Draft." - ) - - # Cannot make DRAFT dataset public - must be FINAL first - if is_public_value is True: - # Check if dataset will be DRAFT after this update - new_status = status_value or dataset.status - if new_status == DatasetStatus.DRAFT: - error_messages.append( - "Draft datasets cannot be made public. Status must be Final." - ) - - if dataset.is_public and is_public_value is False: - error_messages.append( - "Cannot change public dataset visibility back to Private." - ) - - return error_messages - - -user_publish_dataset_view = PublishDatasetView.as_view() - - -class TemporaryZipDownloadView(Auth0LoginRequiredMixin, View): - """View to display a temporary zip file download page and serve the file.""" - - template_name = "users/temporary_zip_download.html" - - def get(self, request, *args, **kwargs) -> HttpResponse: - """Display download page for a temporary zip file or serve the file.""" - zip_uuid = kwargs.get("uuid") - if not zip_uuid: - log.warning("No UUID provided in temporary zip download request") - error_msg = "UUID is required" - raise Http404(error_msg) - - # Check if this is a download request (automatic download from JavaScript) - if request.GET.get("download") == "true": - return self._serve_file_download(zip_uuid, request.user) - - try: - # Get the temporary zip file - temp_zip = get_object_or_404( - TemporaryZipFile, - uuid=zip_uuid, - owner=request.user, - ) - - # Check if file still exists on disk - file_exists = ( - Path(temp_zip.file_path).exists() if temp_zip.file_path else False - ) - - # Determine status and prepare context - if temp_zip.is_deleted: - status = "deleted" - message = "This file has been deleted and is no longer available." - elif temp_zip.is_expired: - status = "expired" - message = "This download link has expired and is no longer available." - elif not file_exists: - status = "file_missing" - message = "The file was not found on the server." - else: - status = "available" - message = None - - # Convert UTC expiry date to user's timezone for display - expires_at_local = ( - timezone.localtime(temp_zip.expires_at) if temp_zip.expires_at else None - ) - - context = { - "temp_zip": temp_zip, - "status": status, - "message": message, - "file_exists": file_exists, - "expires_at_local": expires_at_local, - } - - return render(request, template_name=self.template_name, context=context) - - except TemporaryZipFile.DoesNotExist as err: - log.warning( - f"Temporary zip file not found: {zip_uuid} for user: {request.user.id}" - ) - error_msg = "File not found." - raise Http404(error_msg) from err - - def _serve_file_download(self, zip_uuid: str, user) -> HttpResponse: - """Serve the zip file for download.""" - # Get the temporary zip file - temp_zip = get_object_or_404( - TemporaryZipFile, - uuid=zip_uuid, - owner=user, - ) - - log.info(f"Found temporary zip file: {temp_zip.filename}") - - file_path = Path(temp_zip.file_path) - if not file_path.exists(): - log.warning(f"File not found on disk: {temp_zip.file_path}") - return JsonResponse( - {"error": "The file was not found on the server."}, status=404 - ) - - try: - file_size = file_path.stat().st_size - - with file_path.open("rb") as f: - file_content = f.read() - response = HttpResponse(file_content, content_type="application/zip") - response["Content-Disposition"] = ( - f'attachment; filename="{temp_zip.filename}"' - ) - response["Content-Length"] = file_size - - # Mark the file as downloaded - temp_zip.mark_downloaded() - - return response - - except OSError: - log.exception(f"Error reading file: {temp_zip.file_path}") - return JsonResponse({"error": "Error reading file."}, status=500) - - -user_temporary_zip_download_view = TemporaryZipDownloadView.as_view() - - -def _parse_optional_time( - raw_value: str | None, param_name: str -) -> tuple[int | None, JsonResponse | None]: - """Parse optional start/end time. - - Returns (value, None), or (None, error JsonResponse). - """ - if raw_value in (None, ""): - return None, None - try: - value = int(raw_value) - except (TypeError, ValueError): - return None, JsonResponse( - { - "success": False, - "message": f"Invalid {param_name}; it must be an integer value.", - }, - status=400, - ) - if value < 0: - message = f"Invalid {param_name}; it must be greater than or equal to 0." - return None, JsonResponse( - { - "success": False, - "message": message, - }, - status=400, - ) - return value, None - - -def _validate_time_range( - start_time: int | None, end_time: int | None -) -> JsonResponse | None: - """Return 400 JsonResponse if both provided and start >= end; else None.""" - if start_time is not None and end_time is not None and start_time >= end_time: - return JsonResponse( - { - "success": False, - "message": "Invalid time range; start_time must be less than end_time.", - }, - status=400, - ) - return None - - -class DownloadItemView(Auth0LoginRequiredMixin, View): - """ - Unified view to handle item download requests for both datasets and captures. - - This view follows the same pattern as ShareItemView, accepting item_type - as a URL parameter and handling the download logic generically. - """ - - # Map item types to their corresponding models - ITEM_MODELS = { - ItemType.DATASET: Dataset, - ItemType.CAPTURE: Capture, - } - - def post( # noqa: PLR0911 - self, - request: HttpRequest, - item_uuid: UUID, - item_type: ItemType, - *args: Any, - **kwargs: Any, - ) -> HttpResponse: - """ - Handle item download request. - - Args: - request: The HTTP request object - item_uuid: The UUID of the item to download - item_type: The type of item to download from ItemType enum - - Returns: - A JSON response containing the download status - """ - # Optional start and end times for temporal filtering - raw_start_time = request.POST.get("start_time") - raw_end_time = request.POST.get("end_time") - - start_time, err = _parse_optional_time(raw_start_time, "start_time") - if err is not None: - return err - end_time, err = _parse_optional_time(raw_end_time, "end_time") - if err is not None: - return err - err = _validate_time_range(start_time, end_time) - if err is not None: - return err - - # Validate item type - if item_type not in self.ITEM_MODELS: - return JsonResponse( - {"success": False, "message": "Invalid item type"}, - status=400, - ) - - # Check if user has access to the item (either as owner or shared user) - if not user_has_access_to_item(request.user, item_uuid, item_type): - return JsonResponse( - { - "success": False, - "message": f"{item_type.capitalize()} not found or access denied", - "item_uuid": item_uuid, - }, - status=404, - ) - - # Get the item - model_class = self.ITEM_MODELS[item_type] - try: - item = get_object_or_404( - model_class, - uuid=item_uuid, - is_deleted=False, - ) - except model_class.DoesNotExist: - return JsonResponse( - { - "success": False, - "message": f"{item_type.capitalize()} not found", - "item_uuid": item_uuid, - }, - status=404, - ) - - # Get user email - user_email = request.user.email - if not user_email: - return JsonResponse( - { - "success": False, - "message": f"User email is required for sending {item_type} files.", - }, - status=400, - ) - - # Check if a user already has a task running - task_name = f"{item_type}_download" - if is_user_locked(str(request.user.id), task_name): - return JsonResponse( - { - "success": False, - "message": ( - f"You already have a {item_type} download in progress. " - "Please wait for it to complete." - ), - }, - status=400, - ) - - # Trigger the unified Celery task - task = send_item_files_email.delay( - str(item.uuid), - str(request.user.id), - item_type, - start_time=start_time, - end_time=end_time, - ) - - return JsonResponse( - { - "success": True, - "message": ( - f"{item_type.capitalize()} download request accepted. " - "You will receive an email with the files shortly." - ), - "task_id": task.id, - "item_name": getattr(item, "name", str(item)), - "user_email": user_email, - }, - status=202, - ) - - -user_download_item_view = DownloadItemView.as_view() - - -class DatasetDetailsView(FileTreeMixin, View): - """View to handle dataset details modal requests.""" - - def _get_dataset_files(self, dataset: Dataset) -> QuerySet[File]: - """ - Get all files associated with a dataset, - including files from linked captures. - - Supports both FK and M2M relationships (expand-contract pattern). - - Args: - dataset: The dataset to get files for - - Returns: - A QuerySet of files associated with the dataset - """ - return get_dataset_files_including_captures(dataset, include_deleted=False) - - def get(self, request, *args, **kwargs) -> JsonResponse: - """ - Get dataset details and files for the modal. - - Args: - request: The HTTP request object - *args: Variable length argument list - **kwargs: Arbitrary keyword arguments - - Returns: - A JSON response containing the dataset details and files - """ - dataset_uuid_str = request.GET.get("dataset_uuid") - - if not dataset_uuid_str: - return JsonResponse({"error": "Dataset UUID is required"}, status=400) - - try: - dataset_uuid = UUID(dataset_uuid_str) - except ValueError: - return JsonResponse({"error": "Invalid dataset UUID"}, status=400) - - try: - dataset = get_object_or_404(Dataset, uuid=dataset_uuid, is_deleted=False) - - has_public_access = ( - dataset.is_public and dataset.status == DatasetStatus.FINAL - ) - has_user_access = request.user.is_authenticated and user_has_access_to_item( - request.user, dataset_uuid, ItemType.DATASET - ) - - if not (has_public_access or has_user_access): - return JsonResponse( - {"error": "Dataset not found or access denied"}, status=404 - ) - - # Get dataset information - dataset_data = get_dataset_serializer( - dataset, has_user_access=has_user_access - ) - - # Get all files associated with the dataset - files_queryset = self._get_dataset_files(dataset) - - # Calculate statistics - total_files = files_queryset.count() - captures_count = files_queryset.filter(capture__isnull=False).count() - artifacts_count = files_queryset.filter(capture__isnull=True).count() - total_size = files_queryset.aggregate(total=Sum("size"))["total"] or 0 - - base_dir = sanitize_path_rel_to_user( - unsafe_path="/", - user=dataset.owner, - ) - - tree_data = self._get_directory_tree(files_queryset, str(base_dir)) - - response_data = { - "dataset": dataset_data, - "tree": tree_data, - "statistics": { - "total_files": total_files, - "captures": captures_count, - "artifacts": artifacts_count, - "total_size": total_size, - }, - } - - return JsonResponse(response_data) - - except Dataset.DoesNotExist: - return JsonResponse({"error": "Dataset not found"}, status=404) - except Exception: # noqa: BLE001 - log.exception("Error retrieving dataset details") - return JsonResponse({"error": "Internal server error"}, status=500) - - -user_dataset_details_view = DatasetDetailsView.as_view() - - -class DatasetVersioningView(Auth0LoginRequiredMixin, View): - """View to handle dataset versioning updates.""" - - def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: - dataset_uuid = request.POST.get("dataset_uuid") - copy_shared_users = request.POST.get("copy_shared_users", "false").lower() in ( - "true", - "1", - "on", - ) - if not dataset_uuid: - return JsonResponse({"error": "Dataset UUID is required"}, status=400) - - dataset = get_object_or_404(Dataset, uuid=dataset_uuid, is_deleted=False) - - # check if user has access to the dataset - if not UserSharePermission.user_can_advance_version( - request.user, dataset_uuid, ItemType.DATASET - ): - return JsonResponse( - { - "error": ( - "You do not have permission to advance " - "the version of this dataset" - ) - }, - status=403, - ) - - # copy dataset with relations - new_dataset = self._copy_dataset_with_relations( - dataset, - request_user=request.user, - copy_shared_users=copy_shared_users, - ) - - return JsonResponse({"success": True, "version": new_dataset.version}) - - def _copy_dataset_with_relations( - self, - original_dataset: Dataset, - *, - request_user: User, - copy_shared_users: bool = False, - ) -> Dataset: - """ - Copy a dataset along with all its related files and captures. - - Args: - original_dataset: The dataset to copy - request_user: The user creating the new version - - Returns: - The new dataset with copied related objects - """ - new_version = original_dataset.version + 1 - - # Use database transaction with locking to prevent race conditions - # when multiple requests try to create the same version simultaneously - with transaction.atomic(): - # Lock the original dataset to prevent concurrent version creation - locked_dataset = Dataset.objects.select_for_update().get( - uuid=original_dataset.uuid - ) - - # Check again for existing version within the locked transaction - existing_version = Dataset.objects.filter( - previous_version=locked_dataset, - version=new_version, - owner=request_user, - is_deleted=False, - ).first() - - if existing_version: - # Return existing version if it was already created - return existing_version - - # Fields that should not be copied from the original dataset - # These fields will be reset for the new version - no_copy_fields = [ - "uuid", - "created_at", - "updated_at", - "status", - "is_public", - "shared_with", - "previous_version", - "version", - "owner", - ] - - dataset_data = { - field.name: getattr(locked_dataset, field.name) - for field in locked_dataset._meta.get_fields() # noqa: SLF001 - if hasattr(field, "name") - and field.name not in no_copy_fields - and not field.many_to_many - and not field.one_to_many - and not field.one_to_one - } - dataset_data["owner"] = request_user - dataset_data["version"] = new_version - dataset_data["previous_version"] = locked_dataset - - # Ensure status is draft for new version - dataset_data["status"] = DatasetStatus.DRAFT.value - dataset_data["is_public"] = False - - new_dataset = Dataset.objects.create(**dataset_data) - - # Set the relationships on the new dataset - new_dataset.captures.set(locked_dataset.captures.all()) - new_dataset.files.set(locked_dataset.files.all()) - new_dataset.keywords.set(locked_dataset.keywords.all()) - if copy_shared_users: - self._copy_shared_users(locked_dataset, new_dataset) - - return new_dataset - - def _copy_shared_users( - self, original_dataset: Dataset, new_dataset: Dataset - ) -> None: - """ - Copy the shared users from the original dataset to the new dataset. - Args: - original_dataset: The original dataset - new_dataset: The new dataset - """ - shared_users = get_shared_users_for_item( - original_dataset.uuid, ItemType.DATASET - ) - for shared_user in shared_users: - UserSharePermission.objects.create( - owner=new_dataset.owner, - shared_with=shared_user.shared_with, - item_type=ItemType.DATASET, - item_uuid=new_dataset.uuid, - is_enabled=True, - is_deleted=False, - permission_level=shared_user.permission_level, - ) - - -user_dataset_versioning_view = DatasetVersioningView.as_view() - - -# Auth0LoginRequiredMixin is not used because this view might be called from the home -# page where users may not be authenticated, but we still want to allow rendering of -# public components. -# -# SECURITY MODEL: -# - Only templates in users/components/ directory are allowed (enforced by prefix check) -# - Context data is provided by the client, not pulled from the database -# - All data is rendered through Django templates with automatic HTML escaping -# - CSRF protection is still enforced by Django middleware -# - No sensitive server-side data is exposed - only client-provided data is rendered -# - Calling views (e.g., DatasetDetailsView) are responsible for authorization checks -# - Rate limiting should be configured at the infrastructure level -class RenderHTMLFragmentView(View): - """Generic view to render any HTML fragment from a Django template. - - This endpoint allows rendering of component templates with client-provided context. - It's designed to support both authenticated and unauthenticated users for rendering - public UI components (e.g., file trees for public datasets). - - Security: - - Restricted to users/components/ templates only - - Context is client-provided (no database queries) - - Django's automatic HTML escaping prevents XSS - - Authorization must be handled by calling views - """ - - def post(self, request: HttpRequest) -> JsonResponse: - """ - Render HTML fragment using server-side templates. - - Expects JSON body with: - ```json - { - "template": "users/components/my_component.html", - "context": { - "key": "value", - ... - } - } - ``` - Returns: - JsonResponse with rendered HTML - """ - try: - data = json.loads(request.body) - except json.JSONDecodeError: - return JsonResponse({"error": "Invalid JSON"}, status=400) - - template_name = data.get("template") - context = data.get("context", {}) - - if not template_name: - return JsonResponse({"error": "Template name is required"}, status=400) - - # Security: Only allow templates from users/components/ directory - # Resolves path traversal attempts like "../" - if not _is_safe_template_path(template_name): - log.warning(f"Invalid template path: {template_name}") - return JsonResponse( - {"error": "Cannot render component."}, - status=400, - ) - - try: - html = render_html_fragment( - template_name=template_name, - context=context, - request=request, - ) - - return JsonResponse({"html": html}) - except Exception: # noqa: BLE001 - log.exception(f"Error rendering template {data.get('template', 'unknown')}") - return JsonResponse( - {"error": "Failed to render component.", "code": "RENDER_ERROR"}, - status=500, - ) - - -render_html_fragment_view = RenderHTMLFragmentView.as_view() - - -class ShareGroupListView(Auth0LoginRequiredMixin, UserSearchMixin, View): - """ - View to handle ShareGroup management functionality. - - This view allows users to: - - View their owned ShareGroups - - Create new ShareGroups - - Add/remove members from ShareGroups - - Delete ShareGroups - """ - - template_name = "users/share_group_list.html" - - def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: - """Display the ShareGroup management page.""" - # Check if this is an AJAX request for group members - group_uuid = request.GET.get("group_uuid") - search_query = request.GET.get("q") - - if request.headers.get("X-Requested-With") == "XMLHttpRequest": - if not group_uuid: - return JsonResponse({"error": "Group not found."}, status=400) - - if search_query: - return self._search_users_for_group(request, group_uuid, search_query) - return self._get_group_members(request, group_uuid) - - return self._display_share_groups_page(request) - - def _search_users_for_group( - self, request: HttpRequest, group_uuid: str, search_query: str - ) -> HttpResponse: - """Search users for a specific group.""" - user = cast("User", request.user) - if not hasattr(user, "owned_share_groups"): - return JsonResponse( - {"error": "User does not have owned share groups"}, status=400 - ) - shared_groups = cast("QuerySet[ShareGroup]", user.owned_share_groups) - try: - group = shared_groups.get(uuid=group_uuid, is_deleted=False) - users_in_group = group.members.values_list("id", flat=True) - - return self.search_users( - request=request, - exclude_user_ids=users_in_group, - include_groups=False, - ) - except ShareGroup.DoesNotExist: - return JsonResponse({"error": "ShareGroup not found"}, status=404) - - def _display_share_groups_page(self, request: HttpRequest) -> HttpResponse: - """Display the main share groups page.""" - share_groups = ( - request.user.owned_share_groups.filter(is_deleted=False) - .prefetch_related("members") - .order_by("-created_at") - ) - - context = { - "share_groups": share_groups, - } - - return render(request, self.template_name, context) - - def _get_group_members(self, request: HttpRequest, group_uuid: str) -> JsonResponse: - """Get current members of a ShareGroup.""" - try: - share_group = request.user.owned_share_groups.get( - uuid=group_uuid, is_deleted=False - ) - - members = share_group.members.all().values("email", "name") - member_list = [ - {"email": member["email"], "name": member["name"]} for member in members - ] - - return JsonResponse( - {"success": True, "members": member_list, "count": len(member_list)} - ) - except ShareGroup.DoesNotExist: - return JsonResponse({"error": "ShareGroup not found"}, status=404) - - def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: - """Handle ShareGroup operations (create, update, delete).""" - action = request.POST.get("action") - if not action: - return JsonResponse({"error": "Action is required"}, status=400) - - action_handlers = { - "create": self._create_share_group, - "add_members": self._add_members_to_group, - "remove_members": self._remove_members_from_group, - "delete_group": self._delete_share_group, - "get_shared_assets": self._get_shared_assets_for_group_request, - } - - handler = action_handlers.get(action) - if handler: - return handler(request) - return JsonResponse({"error": "Invalid action"}, status=400) - - def _create_share_group(self, request: HttpRequest) -> JsonResponse: - """Create a new ShareGroup.""" - name = request.POST.get("name", "").strip() - - if not name: - return JsonResponse({"error": "Group name is required"}, status=400) - - # Check if group name already exists for this user - if request.user.owned_share_groups.filter(name=name, is_deleted=False).exists(): - return JsonResponse( - {"error": "A group with this name already exists"}, status=400 - ) - - try: - share_group = ShareGroup.objects.create(name=name, owner=request.user) - share_group.members.add(request.user) - - return JsonResponse( - { - "success": True, - "message": f'ShareGroup "{name}" created successfully', - "group": { - "uuid": str(share_group.uuid), - "name": share_group.name, - "created_at": share_group.created_at.isoformat(), - "member_count": 1, - }, - } - ) - except (ValueError, IntegrityError) as e: - return JsonResponse({"error": f"Failed to create group: {e!s}"}, status=500) - - def _add_members_to_group(self, request: HttpRequest) -> JsonResponse: - """Add members to a ShareGroup.""" - group_uuid = request.POST.get("group_uuid") - user_emails_str = request.POST.get("user_emails", "").strip() - - if not group_uuid or not user_emails_str: - return JsonResponse( - {"error": "Group UUID and user emails are required"}, status=400 - ) - - try: - share_group = request.user.owned_share_groups.get( - uuid=group_uuid, is_deleted=False - ) - except ShareGroup.DoesNotExist: - return JsonResponse({"error": "ShareGroup not found"}, status=404) - - # Get shared assets that will be accessible to new group members - # (commented out as not currently used) - - # Parse and validate user emails - user_emails = [ - email.strip() for email in user_emails_str.split(",") if email.strip() - ] - added_users, errors = self._process_user_addition( - request=request, - share_group=share_group, - user_emails=user_emails, - request_user=request.user, - ) - - return JsonResponse( - { - "success": True, - "message": ( - f'Added {len(added_users)} members to group "{share_group.name}"' - ), - "added_users": added_users, - "errors": errors, - "member_count": share_group.members.count(), - } - ) - - def _process_user_addition( - self, - request: HttpRequest, - share_group: ShareGroup, - user_emails: list[str], - request_user: User, - ) -> tuple[list[str], list[str]]: - """Process adding users to a group. Returns (added_users, errors).""" - added_users = [] - errors = [] - - for email in user_emails: - try: - user = User.objects.get(email=email, is_approved=True) - - if user == request_user: - errors.append(f"You cannot add yourself to a group ({email})") - continue - - if share_group.members.filter(id=user.id).exists(): - errors.append(f"User {email} is already a member of this group") - continue - - share_group.members.add(user) - added_users.append(email) - - # get the user_object from the email - user_object = User.objects.get(email=email) - message = ( - f"You have been added to the group {share_group.name} " - f"by {request.user.name}" - ) - - self._share_items_with_users_in_group_on_add( - request=request, - group=share_group, - user=user_object, - message=message, - ) - except User.DoesNotExist: - errors.append(f"User with email {email} not found or not approved") - - return added_users, errors - - def _share_items_with_users_in_group_on_add( - self, - request: HttpRequest, - group: ShareGroup, - user: User, - message: str, - ) -> None: - """Share items to new members of group on add""" - - # find share permissions for group members - shared_items = ( - UserSharePermission.objects.filter( - share_groups=group, - is_deleted=False, - is_enabled=True, - ) - .values_list("item_uuid", "item_type") - .distinct() - ) - # create share permissions for new member - for item_uuid, item_type in shared_items: - update_or_create_user_group_share_permissions( - request_user=request.user, - group=group, - share_user=user, - item_uuid=item_uuid, - item_type=item_type, - message=message, - ) - - def _remove_members_from_group(self, request: HttpRequest) -> JsonResponse: - """Remove members from a ShareGroup.""" - group_uuid = request.POST.get("group_uuid") - user_emails_str = request.POST.get("user_emails", "").strip() - - if not group_uuid or not user_emails_str: - return JsonResponse( - {"error": "Group UUID and user emails are required"}, status=400 - ) - - try: - share_group = request.user.owned_share_groups.get( - uuid=group_uuid, is_deleted=False - ) - except ShareGroup.DoesNotExist: - return JsonResponse({"error": "ShareGroup not found"}, status=404) - - # Parse user emails - user_emails = [ - email.strip() for email in user_emails_str.split(",") if email.strip() - ] - removed_users, errors = self._process_user_removal(share_group, user_emails) - - return JsonResponse( - { - "success": True, - "message": ( - f"Removed {len(removed_users)} members from group " - f'"{share_group.name}"' - ), - "removed_users": removed_users, - "errors": errors, - "member_count": share_group.members.count(), - } - ) - - def _process_user_removal( - self, share_group: ShareGroup, user_emails: list[str] - ) -> tuple[list[str], list[str]]: - """Process removing users from a group. Returns (removed_users, errors).""" - removed_users = [] - errors = [] - - for email in user_emails: - try: - user = User.objects.get(email=email) - - if not share_group.members.filter(id=user.id).exists(): - errors.append(f"User {email} is not a member of this group") - continue - - if user == share_group.owner: - errors.append( - f"User {email} is the owner of this group and cannot be removed" - ) - continue - - share_group.members.remove(user) - self._update_user_share_permissions_on_removal(user, share_group) - removed_users.append(email) - - except User.DoesNotExist: - errors.append(f"User with email {email} not found") - - return removed_users, errors - - def _get_shared_assets_for_group( - self, share_group: ShareGroup - ) -> list[dict[str, Any]]: - """Get list of shared assets that are accessible to group members.""" - # Find all share permissions where this group is associated - share_permissions = ( - UserSharePermission.objects.filter( - share_groups=share_group, - is_deleted=False, - is_enabled=True, - ) - .select_related("owner") - .distinct("item_uuid", "item_type") - ) - shared_assets = [] - for permission in share_permissions: - try: - # Get the actual item based on type - if permission.item_type == "dataset": - item = Dataset.objects.get(uuid=permission.item_uuid) - elif permission.item_type == "capture": - item = Capture.objects.get(uuid=permission.item_uuid) - else: - continue # Skip unknown item types - - shared_assets.append( - { - "uuid": str(item.uuid), - "name": getattr(item, "name", str(item)), - "type": permission.item_type, - "owner_name": permission.owner.name, - "owner_email": permission.owner.email, - } - ) - except (Dataset.DoesNotExist, Capture.DoesNotExist): - # Skip if item no longer exists - continue - - # Sort assets: datasets first (alphabetically), then captures (alphabetically) - shared_assets.sort( - key=lambda asset: (asset["type"] != "dataset", asset["name"].lower()) - ) - - return shared_assets - - def _update_user_share_permissions_on_removal( - self, user: User, share_group: ShareGroup - ) -> None: - """Update share permissions when a user is removed from a group.""" - # Find all share permissions where this user was shared via this group - share_permissions = UserSharePermission.objects.filter( - shared_with=user, - share_groups=share_group, - is_deleted=False, - is_enabled=True, - ) - - # For each permission, remove the group association and update enabled status - for permission in share_permissions: - permission.share_groups.remove(share_group) - permission.update_enabled_status() - - def _get_shared_assets_for_group_request( - self, request: HttpRequest - ) -> JsonResponse: - """Get shared assets for a group (for display in modal).""" - group_uuid = request.POST.get("group_uuid") - - if not group_uuid: - return JsonResponse({"error": "Group UUID is required"}, status=400) - - try: - share_group = request.user.owned_share_groups.get( - uuid=group_uuid, is_deleted=False - ) - except ShareGroup.DoesNotExist: - return JsonResponse({"error": "ShareGroup not found"}, status=404) - - shared_assets = self._get_shared_assets_for_group(share_group) - - return JsonResponse({"success": True, "shared_assets": shared_assets}) - - def _delete_share_group(self, request: HttpRequest) -> JsonResponse: - """Delete a ShareGroup (soft delete).""" - group_uuid = request.POST.get("group_uuid") - - if not group_uuid: - return JsonResponse({"error": "Group UUID is required"}, status=400) - - try: - share_group = request.user.owned_share_groups.get( - uuid=group_uuid, is_deleted=False - ) - except ShareGroup.DoesNotExist: - return JsonResponse({"error": "ShareGroup not found"}, status=404) - - try: - share_group.soft_delete() - return JsonResponse( - { - "success": True, - "message": f'ShareGroup "{share_group.name}" deleted successfully', - } - ) - except (ValueError, IntegrityError) as e: - return JsonResponse({"error": f"Failed to delete group: {e!s}"}, status=500) - - -user_share_group_list_view = ShareGroupListView.as_view() - - -class UploadCaptureView(Auth0LoginRequiredMixin, View): - def get(self, request, *args, **kwargs): - """Handle GET request to ensure CSRF token is available.""" - - return JsonResponse({"csrf_token": request.META.get("CSRF_COOKIE", "")}) - - def _process_file_uploads( - self, - request: HttpRequest, - upload_chunk_files: list[Any], - relative_paths: list[str], - ) -> tuple[int, list[str]]: - saved_files_count = 0 - file_errors = [] - skipped_files = [] - - # Validate that both lists have the same length before processing - if len(upload_chunk_files) != len(relative_paths): - log.error( - "upload_chunk_files and relative_paths have different lengths: " - f"{len(upload_chunk_files)} vs {len(relative_paths)}", - ) - file_errors.append( - "Internal error: mismatched file and path counts. " - "Please contact support." - ) - return 0, file_errors - - for f, rel_path in zip(upload_chunk_files, relative_paths, strict=True): - path = Path(rel_path) - directory = "/" + str(path.parent) if path.parent != Path() else "/" - filename = path.name - file_size = f.size - content_type = getattr(f, "content_type", "application/octet-stream") - - # Skip empty files (these are placeholders for skipped files) - if file_size == 0: - log.info( - f"Skipping empty file: {filename} " - "(likely a placeholder for skipped file)" - ) - continue - - file_data = { - "owner": request.user.pk, - "name": filename, - "directory": directory, - "file": f, - "size": file_size, - "media_type": content_type, - } - responses, upload_errors = upload_file_helper_simple(request, file_data) - - for response in responses: - if response.status_code in ( - status.HTTP_200_OK, - status.HTTP_201_CREATED, - ): - saved_files_count += 1 - else: - error_msg = f"Failed to upload {filename}: {response.data}" - file_errors.append(error_msg) - log.error(error_msg) - file_errors.extend(upload_errors) - file_errors.extend(skipped_files) - return saved_files_count, file_errors - - def _create_capture_with_endpoint_helper( - self, - request: HttpRequest, - capture_data: dict[str, Any], - all_relative_paths: list[str] | None = None, - ) -> tuple[str | None, str | None]: - """Create a single capture using prepared capture data. - - Returns (capture_uuid, error) where capture_uuid is the created capture - UUID or None if creation failed, and error is the error message or None. - """ - try: - # Set the index name based on capture type - capture_data["index_name"] = infer_index_name(capture_data["capture_type"]) - - # Use the helper function to create the capture - responses, capture_errors = create_capture_helper_simple( - request, capture_data - ) - except (ValueError, TypeError, AttributeError) as exc: - log.exception("Data validation error creating capture") - return None, f"Data validation error: {exc}" - except (ConnectionError, TimeoutError) as exc: - log.exception("Network error creating capture") - return None, f"Network error: {exc}" - else: - if responses: - # Capture created successfully - response = responses[0] - if hasattr(response, "data") and isinstance(response.data, dict): - capture_data = response.data - # Extract only the UUID since that's all we use - capture_uuid = capture_data.get("uuid") - return capture_uuid, None - log.warning(f"Unexpected response format: {response.data}") - return ( - None, - f"Unexpected response format: {response.data}", - ) - # Capture creation failed - error_msg = capture_errors[0] if capture_errors else "Unknown error" - log.error(f"Failed to create capture: {error_msg}") - return ( - None, - f"Failed to create capture: {error_msg}", - ) - - def _calculate_top_level_dir( - self, relative_paths: list[str], all_relative_paths: list[str] - ) -> str: - """Calculate the top level directory from relative paths.""" - if all_relative_paths and len(all_relative_paths) > 0: - # Use all_relative_paths when files are skipped - first_rel_path = all_relative_paths[0] - elif relative_paths and len(relative_paths) > 0: - # Use uploaded relative_paths for normal uploads - first_rel_path = relative_paths[0] - else: - first_rel_path = "" - - if first_rel_path and "/" in first_rel_path: - return "/" + first_rel_path.split("/")[0] - if first_rel_path: - return "/" - return "/" - - def check_rh_scan_group(self, scan_group: str) -> str | None: - """Check and validate RadioHound scan group. - - Args: - scan_group: The scan group string to validate - - Returns: - str: Error message if validation fails, None if valid - """ - if scan_group and scan_group.strip(): - # Validate UUID format if scan_group is provided - if not validate_uuid(scan_group.strip()): - return ( - f"Invalid scan group format. Must be a valid UUID, " - f"got: {scan_group}" - ) - return None - - def _create_captures_by_type( - self, - request: HttpRequest, - channels: list[str], - capture_data: dict[str, Any], - scan_group: str, - all_relative_paths: list[str], - ) -> tuple[list[str], list[str]]: - """Create captures based on capture type. - For RadioHound: Creates a single capture with scan_group - For DigitalRF: Creates multiple captures, one for each channel - """ - created_captures = [] - errors = [] - - if capture_data["capture_type"] == CaptureType.RadioHound: - # For RadioHound, create a single capture with scan_group - scan_group_error = self.check_rh_scan_group(scan_group) - if scan_group_error: - return [], [scan_group_error] - if scan_group and scan_group.strip(): - capture_data["scan_group"] = scan_group - - created_capture, error = self._create_capture_with_endpoint_helper( - request, capture_data, all_relative_paths - ) - if created_capture: - created_captures.append(created_capture) - if error: - errors.append(error) - else: - # For DigitalRF, create captures for each channel - for channel in channels: - # Add channel to capture data for this iteration - channel_capture_data = capture_data.copy() - channel_capture_data["channel"] = channel - created_capture, error = self._create_capture_with_endpoint_helper( - request, - channel_capture_data, - all_relative_paths, - ) - if created_capture: - created_captures.append(created_capture) - if error: - errors.append(error) - - return created_captures, errors - - def _parse_upload_request( - self, request: HttpRequest - ) -> tuple[list[Any], list[str], list[str], list[str], str, "CaptureType"]: - """Parse upload request parameters.""" - upload_chunk_files = request.FILES.getlist("files") - relative_paths = request.POST.getlist("relative_paths") - all_relative_paths = request.POST.getlist("all_relative_paths") - channels_str = request.POST.get("channels", "") - channels = [ch.strip() for ch in channels_str.split(",") if ch.strip()] - scan_group = request.POST.get("scan_group", "") - capture_type_str = request.POST.get( - "capture_type", CaptureType.DigitalRF.value - ) # Default to DigitalRF - # Convert string to CaptureType enum - capture_type = ( - CaptureType.RadioHound - if capture_type_str == CaptureType.RadioHound.value - else CaptureType.DigitalRF - ) - - return ( - upload_chunk_files, - relative_paths, - all_relative_paths, - channels, - scan_group, - capture_type, - ) - - def _check_required_fields( - self, capture_type: "CaptureType", channels: list[str], scan_group: str - ) -> bool: - """Check if required fields are provided for capture creation.""" - if capture_type == CaptureType.RadioHound: - # scan_group is optional for RadioHound captures - return True - return bool(channels) - - def file_upload_status_mux( - self, - saved_files_count: int, - upload_chunk_files: list[Any], - file_errors: list[str], - *, - all_files_empty: bool, - has_required_fields: bool, - ) -> str: - """Determine the response status based on upload and capture creation - results. - - Returns: - "success": All files successful OR All skipped + has required fields - "error": Some files successful OR All files failed OR All skipped + - missing fields - """ - - if all_files_empty: - # All files were skipped (empty) - return "success" if has_required_fields else "error" - - if ( - saved_files_count > 0 - and saved_files_count == len(upload_chunk_files) - and not file_errors - ): - # All files successful - return "success" - - # Some files successful OR All files failed - return "error" - - def _build_file_capture_response_data( - self, - file_upload_status: str, - saved_files_count: int, - created_captures: list[str], - file_errors: list[str], - capture_errors: list[str], - *, - all_files_empty: bool = False, - has_required_fields: bool = False, - ) -> dict[str, Any]: - """Build the response data dictionary.""" - response_data = { - "file_upload_status": file_upload_status, - "saved_files_count": saved_files_count, - "captures": created_captures, - } - - # Add custom message when all files are skipped (regardless of capture creation) - if all_files_empty and has_required_fields and not file_errors: - response_data["message"] = "Upload skipped. All files exist on server" - elif all_files_empty and not has_required_fields: - # All files were skipped but missing required fields - response_data["message"] = ( - "Upload skipped. All files exist on server, but missing required " - "fields for capture creation" - ) - elif all_files_empty and file_errors: - # All files were skipped but there were errors - response_data["message"] = ( - "Upload skipped. All files exist on server, but there were errors " - "during processing" - ) - elif file_upload_status == "success" and created_captures: - # Successful upload with capture creation - response_data["message"] = ( - f"Upload completed successfully! {saved_files_count} files uploaded " - f"and {len(created_captures)} capture(s) created." - ) - elif file_upload_status == "success": - # Successful upload without capture creation - response_data["message"] = ( - f"Upload completed successfully! {saved_files_count} files uploaded." - ) - - # Combine file upload errors and capture creation errors - all_errors = [] - if file_errors: - all_errors.extend(file_errors) - if capture_errors: - all_errors.extend(capture_errors) - if all_errors: - response_data["errors"] = all_errors - return response_data - - def _process_capture_creation( - self, - request: HttpRequest, - channels: list[str], - capture_type: "CaptureType", - scan_group: str, - all_relative_paths: list[str], - *, - has_required_fields: bool, - ) -> tuple[list[str], list[str]]: - """Handle capture creation logic.""" - capture_errors = [] - created_captures = [] - - if has_required_fields: - log.info( - f"Creating captures - has_required_fields: {has_required_fields}, " - f"capture_type: {capture_type}, channels: {channels}, " - f"scan_group: {scan_group}" - ) - - # Calculate top_level_dir from relative paths - top_level_dir = self._calculate_top_level_dir( - all_relative_paths, all_relative_paths - ) - - # Prepare base capture data - capture_data = { - "capture_type": capture_type, - "top_level_dir": str(top_level_dir), - } - - # Create captures based on type - created_captures, capture_errors = self._create_captures_by_type( - request, channels, capture_data, scan_group, all_relative_paths - ) - - if capture_errors: - log.error(f"Capture creation errors: {capture_errors}") - else: - created_captures = [] - capture_errors = [] - - return created_captures, capture_errors - - def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> JsonResponse: - try: - ( - upload_chunk_files, - relative_paths, - all_relative_paths, - channels, - scan_group, - capture_type, - ) = self._parse_upload_request(request) - - saved_files_count, file_errors = self._process_file_uploads( - request, upload_chunk_files, relative_paths - ) - - created_captures = [] - - # Check if all files were empty (skipped) - # If no files were sent (all skipped on frontend), consider them all empty - all_files_empty = ( - all(f.size == 0 for f in upload_chunk_files) - if upload_chunk_files - else True - ) - - # Additional check: if no files were sent but we have all_relative_paths, - # this indicates all files were skipped on the frontend - if not upload_chunk_files and all_relative_paths: - all_files_empty = True - - # Debug logging for request data - log.info( - "Upload request - files count: " - f"{len(upload_chunk_files) if upload_chunk_files else 0}, " - "all_relative_paths count: " - f"{len(all_relative_paths) if all_relative_paths else 0}, " - f"all_files_empty: {all_files_empty}, capture_type: {capture_type}, " - f"channels: {channels}, scan_group: {scan_group}" - ) - - # Create captures if: - # 1. All uploads succeeded, OR - # 2. We have required fields (regardless of file upload status) - capture_errors = [] - # Check if we have the required fields for capture creation - has_required_fields = self._check_required_fields( - capture_type, channels, scan_group - ) - - # Check if this is a chunked upload (skip capture creation for chunks) - is_chunk = request.POST.get("is_chunk", "false").lower() == "true" - chunk_number = request.POST.get("chunk_number", None) - total_chunks = request.POST.get("total_chunks", None) - - # Determine if this is the last chunk or not a chunked upload - is_last_chunk = ( - not is_chunk - or chunk_number is None - or total_chunks is None - or (int(chunk_number) == int(total_chunks)) - ) - should_create_captures = is_last_chunk - - created_captures = [] - capture_errors = [] - - # Only create captures if this is the last chunk AND there are no file - # upload errors - if should_create_captures and not file_errors: - # Handle capture creation - created_captures, capture_errors = self._process_capture_creation( - request, - channels, - capture_type, - scan_group, - all_relative_paths, - has_required_fields=has_required_fields, - ) - elif should_create_captures and file_errors: - log.info( - "Skipping capture creation due to " - f"file upload errors: {file_errors}" - ) - else: - log.info( - "Skipping capture creation for chunk " - f"{chunk_number} of {total_chunks}" - ) - - # Log file upload errors if they occurred - if file_errors and not all_files_empty: - log.error(f"File upload errors occurred. Errors: {file_errors}") - - # Determine file upload status for frontend display - file_upload_status = self.file_upload_status_mux( - saved_files_count, - upload_chunk_files, - file_errors, - all_files_empty=all_files_empty, - has_required_fields=has_required_fields, - ) - - file_capture_response_data = self._build_file_capture_response_data( - file_upload_status, - saved_files_count, - created_captures, - file_errors, - capture_errors, - all_files_empty=all_files_empty, - has_required_fields=has_required_fields, - ) - - return JsonResponse(file_capture_response_data) - - except (ValueError, TypeError, AttributeError) as e: - log.warning(f"Data validation error in UploadCaptureView.post: {e}") - return JsonResponse( - { - "success": False, - "error": "Invalid request data", - "error_code": "VALIDATION_ERROR", - "message": f"Data validation error: {e!s}", - }, - status=400, - ) - except (ConnectionError, TimeoutError) as e: - log.exception("Network error in UploadCaptureView.post") - return JsonResponse( - { - "success": False, - "error": "Network connection error", - "error_code": "NETWORK_ERROR", - "message": f"Network error: {e!s}", - }, - status=503, - ) - except Exception as e: # noqa: BLE001 - log.exception("Unexpected error in UploadCaptureView.post") - return JsonResponse( - { - "success": False, - "error": "Internal server error", - "error_code": "UNKNOWN_ERROR", - "message": f"{e!s}", - }, - status=500, - ) - - -user_upload_capture_view = UploadCaptureView.as_view() - - -class CheckFileExistsView(Auth0LoginRequiredMixin, View): - """View to check if a file exists based on path, name, and checksum.""" - - def get(self, request, *args, **kwargs): - """Handle GET request to ensure CSRF token is available.""" - - return JsonResponse({"csrf_token": request.META.get("CSRF_COOKIE", "")}) - - def post(self, request, *args, **kwargs): - """Check if a file exists using the provided path, name, and checksum.""" - try: - # Get data from request - data = json.loads(request.body) - directory = data.get("directory", "") - filename = data.get("filename", "") - checksum = data.get("checksum", "") - - # Validate required fields - if not all([directory, filename, checksum]): - return JsonResponse( - { - "error": ( - "Missing required fields: directory, filename, and " - "checksum are required" - ) - }, - status=400, - ) - - # Prepare data for check_file_contents_exist_helper - check_data = { - "directory": directory, - "name": filename, - "sum_blake3": checksum, - } - - # Call the helper function - response = check_file_contents_exist_helper(request, check_data) - - # Extract the response data - if hasattr(response, "data"): - response_data = response.data - else: - response_data = str(response) - - # Return the result - return JsonResponse( - { - "status_code": response.status_code, - "data": response_data, - } - ) - - except json.JSONDecodeError: - return JsonResponse({"error": "Invalid JSON in request body"}, status=400) - - -class FilesView(Auth0LoginRequiredMixin, View): - """Handle HTML requests for the files page.""" - - template_name = "users/files.html" - - def get(self, request, *args, **kwargs) -> HttpResponse: - """Handle HTML page requests for files page.""" - # Get the current directory from query params - current_dir = request.GET.get("dir", "/") - - # Debug logging - log.debug(f"FilesView: current_dir={current_dir}") - - # Initialize items list with proper typing - items: list[Item] = [] - - # Parse the current directory into a navigation context - nav_context = NavigationContext.from_path(current_dir) - - if nav_context.type == NavigationType.ROOT: - # Root directory - show captures and datasets as folders - items.extend(self._add_root_items(request)) - elif nav_context.type == NavigationType.CAPTURE: - # Inside a capture - show nested directories/files within the capture - if not nav_context.capture_uuid: - return HttpResponseRedirect("/users/files/") - items.extend( - add_capture_files( - request, nav_context.capture_uuid, subpath=nav_context.subpath - ) - ) - elif nav_context.type == NavigationType.DATASET: - # Inside a dataset - show nested directories/files within the dataset - # TODO: Implement dataset file browsing when needed - return HttpResponseRedirect("/users/files/") - elif nav_context.type == NavigationType.USER_FILES: - # Inside user file directory - show nested directories/files - items.extend(add_user_files(request, subpath=nav_context.subpath)) - else: - # Unknown directory - go back to root - return HttpResponseRedirect("/users/files/") - - # Build breadcrumb parts - breadcrumb_parts = build_breadcrumbs(nav_context.to_path(), request.user.email) - - # Debug logging - log.debug( - f"FilesView: context summary items={len(items)}", - ) - log.debug( - f"FilesView: first items preview={items[:3] if items else 'No items'}", - ) - - # Additional debugging for directory items - for i, item in enumerate(items): - if hasattr(item, "type") and item.type == "directory": - log.debug(f"FilesView: directory item {i} => {item}") - - # Convert Pydantic models to dictionaries for template - items_data = items_to_dicts(items) - - return render( - request, - self.template_name, - { - "items": items_data, - "current_dir": nav_context.to_path(), - "breadcrumb_parts": breadcrumb_parts, - "user_email": request.user.email, - }, - ) - - def _add_root_items(self, request) -> list[Item]: - """Add captures and datasets to the root directory.""" - items = add_root_items(request) - # Add shared items - items.extend(add_shared_items(request)) - return items - - -def files_view(request): - """Simple function-based view for files page.""" - # Check if user is authenticated - if not request.user.is_authenticated: - return redirect("users:redirect") - - return render( - request, - "users/files.html", - { - "items": [], - "current_dir": request.GET.get("dir", "/"), - "breadcrumb_parts": [], - "user_email": getattr(request.user, "email", ""), - }, - ) - - -user_check_file_exists_view = CheckFileExistsView.as_view() - - -class SPXDACDatasetAltView(Auth0LoginRequiredMixin, View): - """View for the SpectrumX Student Data Competition page.""" - - template_name = "pages/spx_dac_dataset_alt.html" - - def get(self, request, *args, **kwargs): - """Display the student data competition page and automatically share dataset.""" - dataset_id = settings.SPX_DAC_DATASET_ID - if not dataset_id: - log.warning("SPX_DAC_DATASET_ID not configured") - else: - try: - dataset_uuid = UUID(dataset_id) - # Get the dataset to find its owner - try: - dataset = Dataset.objects.get(uuid=dataset_uuid, is_deleted=False) - except Dataset.DoesNotExist: - log.warning(f"SpX-DAC dataset {dataset_id} not found") - dataset = None - - # Check if user is already the owner - if dataset and dataset.owner != request.user: - # Check if permission already exists - existing_permission = UserSharePermission.objects.filter( - owner=dataset.owner, - shared_with=request.user, - item_type=ItemType.DATASET, - item_uuid=dataset_uuid, - is_deleted=False, - ).first() - - if not existing_permission: - # Create share permission with VIEWER role - UserSharePermission.objects.create( - owner=dataset.owner, - shared_with=request.user, - item_type=ItemType.DATASET, - item_uuid=dataset_uuid, - message="Automatically shared for NSF SpectrumX " - "Data and Algorithm Competition (SpX-DAC)", - permission_level=PermissionLevel.VIEWER, - is_enabled=True, - ) - log.info( - "Automatically shared SpX-DAC dataset " - f"with user {request.user.email}" - ) - elif not existing_permission.is_enabled: - # Re-enable if it was previously disabled - existing_permission.is_enabled = True - existing_permission.save() - log.info( - "Re-enabled SpX-DAC dataset " - f"share for user {request.user.email}" - ) - except ValueError as e: - log.warning(f"Invalid SpX-DAC dataset ID format: {e}") - - context = { - "s3_bucket_url": settings.SPX_DAC_DATASET_S3_URL, - "dataset_id": dataset_id or "458c3f72-8d7e-49cc-9be3-ed0b0cd7e03d", - } - return render(request, self.template_name, context) - - def post(self, request, *args, **kwargs): - """Handle API key generation via AJAX.""" - # Check if user has reached the maximum number of active API keys - api_keys = UserAPIKey.objects.filter(user=request.user).exclude( - source=KeySources.SVIBackend - ) - active_api_key_count = get_active_api_key_count(api_keys) - if active_api_key_count >= MAX_API_KEY_COUNT: - return JsonResponse( - { - "success": False, - "error": "You have reached the maximum number of API keys " - f"({MAX_API_KEY_COUNT}). Please revoke an existing key before " - "creating a new one.", - }, - status=400, - ) - - # Get the name from the form (optional) - api_key_name = request.POST.get("api_key_name", "SpX-DAC Competition") - api_key_description = request.POST.get( - "api_key_description", - "Generated for NSF SpectrumX Data and Algorithm Competition (SpX-DAC)", - ) - - try: - # Create an API key for the user - _, raw_key = UserAPIKey.objects.create_key( - name=api_key_name, - description=api_key_description, - user=request.user, - source=KeySources.SDSWebUI, - expiry_date=None, - ) - return JsonResponse({"success": True, "api_key": raw_key}) - except Exception: # noqa: BLE001 - log.exception("Error generating API key for student competition") - return JsonResponse( - { - "success": False, - "error": "Failed to generate API key. Please try again.", - }, - status=500, - ) - - -spx_dac_dataset_alt_view = SPXDACDatasetAltView.as_view() From fedf84ebd5fc10c827bc0911680ac577e9bd4b6e Mon Sep 17 00:00:00 2001 From: klpoland Date: Thu, 30 Jul 2026 12:21:10 -0400 Subject: [PATCH 05/22] fix: import sds_opensearch_query by package name Use the installed package path, coerce OpenSearch client settings types, and align deptry ignores with the shared client move. Co-authored-by: Cursor --- common/sds_opensearch_query/pyproject.toml | 1 + federation/pyproject.toml | 2 +- federation/sds_federation/main.py | 15 ++++++++------- .../sds_federation/schemas/opensearch_indices.py | 10 +++++----- federation/uv.lock | 6 +++++- gateway/pyproject.toml | 1 + .../api_methods/utils/opensearch_client.py | 3 +-- gateway/uv.lock | 8 ++++++-- 8 files changed, 28 insertions(+), 18 deletions(-) diff --git a/common/sds_opensearch_query/pyproject.toml b/common/sds_opensearch_query/pyproject.toml index 960cdc00a..5236e8019 100644 --- a/common/sds_opensearch_query/pyproject.toml +++ b/common/sds_opensearch_query/pyproject.toml @@ -13,4 +13,5 @@ description = "Shared OpenSearch query builders for SDS gateway and federation" requires-python = ">=3.13,<3.15" dependencies = [ "opensearch-py>=2.7.1", + "requests>=2.32.0", ] diff --git a/federation/pyproject.toml b/federation/pyproject.toml index 805f8aca1..e7bd81564 100644 --- a/federation/pyproject.toml +++ b/federation/pyproject.toml @@ -32,7 +32,7 @@ dev = [ ] [tool.deptry] - extend_exclude = ["docs/"] + extend_exclude = ["docs/", "scripts/local_e2e/"] [tool.deptry.per_rule_ignores] DEP002 = [ diff --git a/federation/sds_federation/main.py b/federation/sds_federation/main.py index 68768df54..2e7e9ef32 100644 --- a/federation/sds_federation/main.py +++ b/federation/sds_federation/main.py @@ -7,8 +7,8 @@ from fastapi import FastAPI from loguru import logger +from sds_opensearch_query.client import build_opensearch_client -from common.sds_opensearch_query.client import build_opensearch_client from sds_federation.models import load_federation_config from sds_federation.routes.health import health_router from sds_federation.routes.webhooks import webhooks_router @@ -31,8 +31,9 @@ def _bootstrap_enabled() -> bool: ) -def get_setting(key: str) -> str: - return os.environ.get(key, "") +def get_setting(key: str, default: str = "") -> str: + return os.environ.get(key, default) + sync_app = FastAPI(title="SDS Federation Sync") sync_app.include_router(health_router) @@ -45,13 +46,13 @@ async def lifespan(app: FastAPI): http = build_gateway_http_client() os_client = build_opensearch_client( - host=get_setting("OPENSEARCH_HOST"), - port=get_setting("OPENSEARCH_PORT"), + host=get_setting("OPENSEARCH_HOST", "opensearch"), + port=int(get_setting("OPENSEARCH_PORT", "9200")), user=get_setting("OPENSEARCH_USER"), password=get_setting("OPENSEARCH_PASSWORD"), - use_ssl=get_setting("OPENSEARCH_USE_SSL"), + use_ssl=get_setting("OPENSEARCH_USE_SSL").lower() in ("1", "true", "yes"), verify_certs=get_setting("OPENSEARCH_VERIFY_CERTS") == "true", - ca_certs=get_setting("OPENSEARCH_CA_CERTS"), + ca_certs=get_setting("OPENSEARCH_CA_CERTS") or None, ) try: ensure_fed_indices(os_client) diff --git a/federation/sds_federation/schemas/opensearch_indices.py b/federation/sds_federation/schemas/opensearch_indices.py index 92b62577e..9e018fef4 100644 --- a/federation/sds_federation/schemas/opensearch_indices.py +++ b/federation/sds_federation/schemas/opensearch_indices.py @@ -1,4 +1,4 @@ -"""OpenSearch index definitions for federated metadata (RFC fed-datasets / fed-captures). +"""OpenSearch index definitions for federated metadata (RFC fed-* indices). Explicit ``properties`` match the RFC search-facing field lists; export-only and envelope fields (e.g. ``status``, ``capture_props``, ``is_deleted``) rely on @@ -9,11 +9,11 @@ from typing import Any -from sds_federation.schemas.webhooks import AssetTypeEnum -from common.sds_opensearch_query.mapping import fed_dataset_mappings -from common.sds_opensearch_query.mapping import fed_capture_mappings -from common.sds_opensearch_query.mapping import FED_INDEX_SETTINGS +from sds_opensearch_query.mapping import FED_INDEX_SETTINGS +from sds_opensearch_query.mapping import fed_capture_mappings +from sds_opensearch_query.mapping import fed_dataset_mappings +from sds_federation.schemas.webhooks import AssetTypeEnum def index_body_for_asset(asset_type: AssetTypeEnum) -> dict[str, Any]: diff --git a/federation/uv.lock b/federation/uv.lock index 16efa70f2..76bd7755d 100644 --- a/federation/uv.lock +++ b/federation/uv.lock @@ -591,10 +591,14 @@ version = "0.1.0" source = { directory = "../common/sds_opensearch_query" } dependencies = [ { name = "opensearch-py" }, + { name = "requests" }, ] [package.metadata] -requires-dist = [{ name = "opensearch-py", specifier = ">=2.7.1" }] +requires-dist = [ + { name = "opensearch-py", specifier = ">=2.7.1" }, + { name = "requests", specifier = ">=2.32.0" }, +] [[package]] name = "six" diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml index e0e0cf1d4..e468d86b7 100644 --- a/gateway/pyproject.toml +++ b/gateway/pyproject.toml @@ -181,6 +181,7 @@ "pytest-cov", # coverage plugin in tests "pytest-sugar", # better progress output for pytest "python-slugify", # provides the slugify import + "requests", # used by sds-opensearch-query OpenSearch client "ruff", # linter used in ci "uvicorn-worker", # uvicorn worker class for gunicorn "uvicorn", # asgi server used in dev diff --git a/gateway/sds_gateway/api_methods/utils/opensearch_client.py b/gateway/sds_gateway/api_methods/utils/opensearch_client.py index 272f27075..1199c241e 100644 --- a/gateway/sds_gateway/api_methods/utils/opensearch_client.py +++ b/gateway/sds_gateway/api_methods/utils/opensearch_client.py @@ -1,6 +1,5 @@ from django.conf import settings - -from common.sds_opensearch_query.client import build_opensearch_client +from sds_opensearch_query.client import build_opensearch_client def get_opensearch_client(): diff --git a/gateway/uv.lock b/gateway/uv.lock index addc5907f..4710c9172 100644 --- a/gateway/uv.lock +++ b/gateway/uv.lock @@ -3,7 +3,7 @@ revision = 3 requires-python = "==3.13.*" [options] -exclude-newer = "2026-06-25T12:23:34.503498Z" +exclude-newer = "2026-07-23T16:20:34.903318Z" exclude-newer-span = "P1W" [[package]] @@ -2238,10 +2238,14 @@ version = "0.1.0" source = { directory = "../common/sds_opensearch_query" } dependencies = [ { name = "opensearch-py" }, + { name = "requests" }, ] [package.metadata] -requires-dist = [{ name = "opensearch-py", specifier = ">=2.7.1" }] +requires-dist = [ + { name = "opensearch-py", specifier = ">=2.7.1" }, + { name = "requests", specifier = ">=2.32.0" }, +] [[package]] name = "sentry-sdk" From b7bdd3347cd78f8f06762d0e9657911667c900fa Mon Sep 17 00:00:00 2001 From: klpoland Date: Mon, 3 Aug 2026 16:47:44 -0400 Subject: [PATCH 06/22] troubleshooting bootstrap/fed init --- .dockerignore | 40 +++++++ .github/workflows/gwy-code-quality.yaml | 2 +- .../{sds_opensearch_query => }/pyproject.toml | 0 .../{sds_opensearch_query => }/__init__.py | 0 .../{sds_opensearch_query => }/client.py | 0 .../{sds_opensearch_query => }/filters.py | 0 .../{sds_opensearch_query => }/mapping.py | 0 .../{sds_opensearch_query => }/query.py | 0 federation/Dockerfile | 8 +- federation/compose.local.yaml | 1 - federation/compose.peer.local.yaml | 57 +++++++++ federation/compose.production.yaml | 1 - federation/justfile | 13 ++- federation/pyproject.toml | 2 +- federation/scripts/env-selection.sh | 10 +- .../local_e2e/download_drf_properties.py | 110 ++++++++++++++++++ federation/scripts/local_e2e/env.example | 2 +- federation/sds_federation/main.py | 4 +- .../schemas/opensearch_indices.py | 12 ++ .../sds_federation/services/local_events.py | 9 +- federation/tests/test_integration_webhooks.py | 4 +- federation/uv.lock | 4 +- gateway/.envs/example/django.env | 6 +- gateway/compose.ci.yaml | 16 +-- gateway/compose.local.yaml | 12 +- gateway/compose/local/django/Dockerfile | 26 +++-- gateway/compose/production/django/Dockerfile | 24 ++-- gateway/config/settings/base.py | 2 +- gateway/pyproject.toml | 2 +- gateway/sds_gateway/api_methods/apps.py | 11 +- .../serializers/dataset_serializers.py | 14 ++- gateway/uv.lock | 6 +- 32 files changed, 323 insertions(+), 75 deletions(-) create mode 100644 .dockerignore rename common/{sds_opensearch_query => }/pyproject.toml (100%) rename common/sds_opensearch_query/{sds_opensearch_query => }/__init__.py (100%) rename common/sds_opensearch_query/{sds_opensearch_query => }/client.py (100%) rename common/sds_opensearch_query/{sds_opensearch_query => }/filters.py (100%) rename common/sds_opensearch_query/{sds_opensearch_query => }/mapping.py (100%) rename common/sds_opensearch_query/{sds_opensearch_query => }/query.py (100%) create mode 100644 federation/compose.peer.local.yaml create mode 100644 federation/scripts/local_e2e/download_drf_properties.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..aa721049e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,40 @@ +# Monorepo-root build context (gateway + federation Dockerfiles). +# Keep gateway/ and federation/ and common/ available for COPY. + +**/.git +**/.venv +**/venv +**/node_modules +**/__pycache__ +**/.pytest_cache +**/.ruff_cache +**/dist +**/build +**/*.pyc + +# Secrets / local env +**/.envs/ +federation-shared.env + +# Unreadable / bulky OpenSearch host data +**/opensearch/data/ + +# Other monorepo trees not needed for gateway/federation images +sdk/ +jupyter/ +seaweedfs/ +docs/ +.cursor/ + +# Gateway local noise (mirrored from gateway/.dockerignore intent) +gateway/.editorconfig +gateway/.gitattributes +gateway/.github +gateway/.idea +gateway/.pre-commit-config.yaml +gateway/.readthedocs.yaml +gateway/.readthedocs.yml +gateway/.travis.yaml +gateway/.travis.yml +gateway/.gitlab-ci.yaml +gateway/.gitlab-ci.yml diff --git a/.github/workflows/gwy-code-quality.yaml b/.github/workflows/gwy-code-quality.yaml index 2bbf2b331..3cf525b66 100644 --- a/.github/workflows/gwy-code-quality.yaml +++ b/.github/workflows/gwy-code-quality.yaml @@ -144,7 +144,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v7 with: - context: gateway + context: . file: gateway/compose/production/django/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/common/sds_opensearch_query/pyproject.toml b/common/pyproject.toml similarity index 100% rename from common/sds_opensearch_query/pyproject.toml rename to common/pyproject.toml diff --git a/common/sds_opensearch_query/sds_opensearch_query/__init__.py b/common/sds_opensearch_query/__init__.py similarity index 100% rename from common/sds_opensearch_query/sds_opensearch_query/__init__.py rename to common/sds_opensearch_query/__init__.py diff --git a/common/sds_opensearch_query/sds_opensearch_query/client.py b/common/sds_opensearch_query/client.py similarity index 100% rename from common/sds_opensearch_query/sds_opensearch_query/client.py rename to common/sds_opensearch_query/client.py diff --git a/common/sds_opensearch_query/sds_opensearch_query/filters.py b/common/sds_opensearch_query/filters.py similarity index 100% rename from common/sds_opensearch_query/sds_opensearch_query/filters.py rename to common/sds_opensearch_query/filters.py diff --git a/common/sds_opensearch_query/sds_opensearch_query/mapping.py b/common/sds_opensearch_query/mapping.py similarity index 100% rename from common/sds_opensearch_query/sds_opensearch_query/mapping.py rename to common/sds_opensearch_query/mapping.py diff --git a/common/sds_opensearch_query/sds_opensearch_query/query.py b/common/sds_opensearch_query/query.py similarity index 100% rename from common/sds_opensearch_query/sds_opensearch_query/query.py rename to common/sds_opensearch_query/query.py diff --git a/federation/Dockerfile b/federation/Dockerfile index f0171082a..dacd1b9cf 100644 --- a/federation/Dockerfile +++ b/federation/Dockerfile @@ -6,11 +6,11 @@ WORKDIR /app/federation ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 -COPY ../common/sds_opensearch_query /app/common/sds_opensearch_query -COPY pyproject.toml uv.lock ./ -COPY sds_federation ./sds_federation +# Build context is repo root (compose: context: ..). +COPY common /app/common +COPY federation/pyproject.toml federation/uv.lock ./ +COPY federation/sds_federation ./sds_federation RUN uv sync --frozen --no-dev - EXPOSE 8000 CMD ["uv", "run", "uvicorn", "sds_federation.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/federation/compose.local.yaml b/federation/compose.local.yaml index 539f100e5..0c488d0ee 100644 --- a/federation/compose.local.yaml +++ b/federation/compose.local.yaml @@ -14,7 +14,6 @@ services: image: sds-federation-local-sync container_name: sds-federation-local-sync env_file: - - .envs/local/sync.env - ../gateway/.envs/local/opensearch.env - ../federation-shared.env environment: diff --git a/federation/compose.peer.local.yaml b/federation/compose.peer.local.yaml new file mode 100644 index 000000000..b7dac0c3b --- /dev/null +++ b/federation/compose.peer.local.yaml @@ -0,0 +1,57 @@ +networks: + sds-network-local: + external: true + name: sds-network-local + peer-internal: + driver: bridge + +services: + sds-federation-peer-opensearch: + image: opensearchproject/opensearch:2 + container_name: sds-federation-peer-opensearch + environment: + discovery.type: single-node + plugins.security.disabled: "true" + OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m + DISABLE_INSTALL_DEMO_CONFIG: "true" + ports: + - "9201:9200" + networks: [peer-internal] + + sds-federation-peer-redis: + image: redis:6 + container_name: sds-federation-peer-redis + ports: + - "6380:6379" + networks: [peer-internal] + + sds-federation-peer-sync: + build: + context: .. + dockerfile: federation/Dockerfile + image: sds-federation-local-sync + container_name: sds-federation-peer-sync + environment: + FEDERATION_CONFIG_PATH: /etc/sds/federation.toml + FEDERATION_SYNC_SERVICE_URL: http://sds-federation-peer-sync:8000/sync + # Dummy gateway — local export bootstrap will fail; that's OK + GATEWAY_INTERNAL_BASE_URL: http://127.0.0.1:9/api/v1 + REDIS_URL: redis://sds-federation-peer-redis:6379/0 + OPENSEARCH_HOST: sds-federation-peer-opensearch + OPENSEARCH_PORT: "9200" + OPENSEARCH_USER: "" + OPENSEARCH_PASSWORD: "" + OPENSEARCH_USE_SSL: "false" + OPENSEARCH_VERIFY_CERTS: "false" + # No mint: leave FEDERATION_SYNC_* unset + FEDERATION_SITE_NAME: peer + volumes: + - ./federation.peer.toml:/etc/sds/federation.toml:ro + ports: + - "8002:8000" + depends_on: + - sds-federation-peer-opensearch + - sds-federation-peer-redis + networks: + - sds-network-local + - peer-internal \ No newline at end of file diff --git a/federation/compose.production.yaml b/federation/compose.production.yaml index 7d377c7c7..5d6a5b22d 100644 --- a/federation/compose.production.yaml +++ b/federation/compose.production.yaml @@ -14,7 +14,6 @@ services: image: sds-federation-prod-sync container_name: sds-federation-prod-sync env_file: - - .envs/production/sync.env - ../gateway/.envs/production/opensearch.env - ../federation-shared.env environment: diff --git a/federation/justfile b/federation/justfile index 0cbd0c361..59de9b0de 100644 --- a/federation/justfile +++ b/federation/justfile @@ -9,9 +9,9 @@ env_selection_script := "./scripts/env-selection.sh" compose_file := shell(env_selection_script + ' $1', "compose_file") env := shell(env_selection_script + ' $1', "env") -env_file := shell(env_selection_script + ' $1', "env_file") sync_container := shell(env_selection_script + ' $1', "sync_container") -docker_compose := "COMPOSE_FILE=" + compose_file + " docker compose --env-file " + env_file +# Service env comes from compose env_file (gateway opensearch.env + federation-shared.env). +docker_compose := "COMPOSE_FILE=" + compose_file + " docker compose" alias run := up @@ -24,7 +24,6 @@ env: #!/usr/bin/env bash echo -e "\nSelected env:\n" echo -e "\tEnvironment: \e[34m '{{ env }}'\e[0m" - echo -e "\tEnvironment file: \e[34m '{{ env_file }}'\e[0m" echo -e "\tCompose file: \e[34m '{{ compose_file }}'\e[0m" echo -e "\tSync container: \e[34m '{{ sync_container }}'\e[0m" echo -e "\tDocker compose command: \e[34m '{{ docker_compose }}'\e[0m" @@ -33,9 +32,11 @@ env: echo -e "\n\e[31mError:\e[0m Compose file '{{ compose_file }}' does not exist." exit 1 fi - if ! [ -f "{{ env_file }}" ]; then - echo -e "\n\e[33mWarning:\e[0m Env file '{{ env_file }}' does not exist." - echo -e "Copy from .envs/example/sync.env if needed." + if ! [ -f "../federation-shared.env" ]; then + echo -e "\n\e[33mWarning:\e[0m ../federation-shared.env missing — run gateway generate-secrets." + fi + if ! [ -f "../gateway/.envs/{{ env }}/opensearch.env" ]; then + echo -e "\n\e[33mWarning:\e[0m ../gateway/.envs/{{ env }}/opensearch.env missing." fi # install runtime + dev deps (pytest) diff --git a/federation/pyproject.toml b/federation/pyproject.toml index e7bd81564..37fc8c546 100644 --- a/federation/pyproject.toml +++ b/federation/pyproject.toml @@ -168,4 +168,4 @@ markers = [ ] [tool.uv.sources] -sds-opensearch-query = { path = "../common/sds_opensearch_query" } +sds-opensearch-query = { path = "../common" } diff --git a/federation/scripts/env-selection.sh b/federation/scripts/env-selection.sh index 3cb89bc2a..ea1dc330e 100755 --- a/federation/scripts/env-selection.sh +++ b/federation/scripts/env-selection.sh @@ -49,14 +49,8 @@ function get_target_value() { local) value='sds-federation-local-sync' ;; esac ;; - env_file) - case "${env_type}" in - production) value='.envs/production/sync.env' ;; - local) value='.envs/local/sync.env' ;; - esac - ;; *) - printf 'unsupported target: %s (use env, compose_file, sync_container, or env_file)\n' "${target}" >&2 + printf 'unsupported target: %s (use env, compose_file, or sync_container)\n' "${target}" >&2 exit 1 ;; esac @@ -71,7 +65,7 @@ function get_target_value() { function main() { if [[ $# -ne 1 ]]; then - printf 'usage: %s \n' "${0}" >&2 + printf 'usage: %s \n' "${0}" >&2 exit 1 fi diff --git a/federation/scripts/local_e2e/download_drf_properties.py b/federation/scripts/local_e2e/download_drf_properties.py new file mode 100644 index 000000000..742c30012 --- /dev/null +++ b/federation/scripts/local_e2e/download_drf_properties.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Download only drf_properties.h5 files for a dataset (optionally one capture). + + cd sdk + set -a && source ../federation/scripts/local_e2e/.env && set +a + uv run python ../federation/scripts/local_e2e/download_drf_properties.py \\ + --dataset-uuid 50e979bd-8018-415c-8212-c08c3dc98654 \\ + --to ../federation/data/downloaded_dataset \\ + --top-level-dir '/files/rherban@nd.edu/CU_conference_NCAR_002_b08a_009f70' +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path +from uuid import UUID + +from spectrumx import Client +from spectrumx.errors import SDSError + +PROPS_NAME = "drf_properties.h5" + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--dataset-uuid", required=True) + p.add_argument( + "--to", + type=Path, + default=Path("../federation/data/downloaded_dataset"), + help="Local root (same as download_crc_dataset --to)", + ) + p.add_argument( + "--top-level-dir", + default="", + help="Optional capture top_level_dir from list_dataset_captures", + ) + p.add_argument("--capture-uuid", default="", help="Optional capture UUID filter") + p.add_argument("--list-only", action="store_true", help="List matches, no download") + p.add_argument("--overwrite", action="store_true") + return p.parse_args() + + +def main() -> int: + args = _parse_args() + host = os.environ.get("SDS_HOST", "sds.crc.nd.edu") + client = Client(host=host) + client.dry_run = False + client.authenticate() + + dataset_uuid = UUID(args.dataset_uuid) + top_level_dirs = [args.top_level_dir] if args.top_level_dir.strip() else None + capture_uuids = [UUID(args.capture_uuid)] if args.capture_uuid.strip() else None + + # Helpful: show captures if you need the exact top_level_dir + if not top_level_dirs and not capture_uuids: + caps = client.list_dataset_captures(dataset_uuid) + print(f"Captures in dataset ({len(caps)}):") + for c in caps: + print( + f" uuid={c.get('uuid')} " + f"top_level_dir={c.get('top_level_dir')} " + f"channels={c.get('channels') or c.get('channel')}" + ) + + files = client.datasets.get_files( + dataset_uuid, + capture_uuids=capture_uuids, + top_level_dirs=top_level_dirs, + ) + props = [f for f in files if f.name == PROPS_NAME] + + if not props: + print( + f"No {PROPS_NAME} in dataset manifest " + f"(filters: top_level_dir={top_level_dirs}, capture={capture_uuids}).\n" + "If empty, CRC never indexed that file for this capture — " + "recreate locally with digital_rf instead.", + file=sys.stderr, + ) + return 2 + + for f in props: + print(f" {f.uuid} {f.directory}/{f.name} size={f.size}") + + if args.list_only: + return 0 + + print(f"Downloading {len(props)} file(s) from {host} -> {args.to.resolve()}") + try: + results = client.download( + to_local_path=args.to, + files_to_download=props, + overwrite=args.overwrite, + verbose=True, + ) + except SDSError as exc: + print(f"Download failed: {exc}", file=sys.stderr) + return 1 + + ok = sum(1 for r in results if r) + fail = len(results) - ok + print(f"Done: {ok} ok, {fail} failed") + return 0 if fail == 0 else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/federation/scripts/local_e2e/env.example b/federation/scripts/local_e2e/env.example index 769b64bed..fa69224a3 100644 --- a/federation/scripts/local_e2e/env.example +++ b/federation/scripts/local_e2e/env.example @@ -4,7 +4,7 @@ SDS_HOST=sds.crc.nd.edu SDS_SECRET_TOKEN= # Optional: filter download to one capture directory (from list_dataset_captures) -# DATASET_UUID=50e979bd-8018-415c-8212-c08c3dc98654 +# DATASET_UUID= # TOP_LEVEL_DIR=/files/you@example.com/federation-fixture/... # Local gateway upload (after download): diff --git a/federation/sds_federation/main.py b/federation/sds_federation/main.py index 2e7e9ef32..8aa82374d 100644 --- a/federation/sds_federation/main.py +++ b/federation/sds_federation/main.py @@ -7,6 +7,7 @@ from fastapi import FastAPI from loguru import logger +from redis.exceptions import TimeoutError as RedisTimeoutError from sds_opensearch_query.client import build_opensearch_client from sds_federation.models import load_federation_config @@ -105,7 +106,8 @@ async def lifespan(app: FastAPI): stop.set() sub_task.cancel() - with suppress(asyncio.CancelledError): + # Pubsub listen may surface TimeoutError while cancelling the blocked read. + with suppress(asyncio.CancelledError, TimeoutError, RedisTimeoutError): await sub_task await http.aclose() diff --git a/federation/sds_federation/schemas/opensearch_indices.py b/federation/sds_federation/schemas/opensearch_indices.py index 9e018fef4..73000bc40 100644 --- a/federation/sds_federation/schemas/opensearch_indices.py +++ b/federation/sds_federation/schemas/opensearch_indices.py @@ -10,11 +10,23 @@ from typing import Any from sds_opensearch_query.mapping import FED_INDEX_SETTINGS +from sds_opensearch_query.mapping import RFC_FED_CAPTURE_PROPERTIES +from sds_opensearch_query.mapping import RFC_FED_DATASET_PROPERTIES from sds_opensearch_query.mapping import fed_capture_mappings from sds_opensearch_query.mapping import fed_dataset_mappings from sds_federation.schemas.webhooks import AssetTypeEnum +__all__ = [ + "FED_INDEX_SETTINGS", + "RFC_FED_CAPTURE_PROPERTIES", + "RFC_FED_DATASET_PROPERTIES", + "fed_capture_mappings", + "fed_dataset_mappings", + "index_body_for_asset", + "index_body_for_index_name", +] + def index_body_for_asset(asset_type: AssetTypeEnum) -> dict[str, Any]: mappings = ( diff --git a/federation/sds_federation/services/local_events.py b/federation/sds_federation/services/local_events.py index fc327df6a..cd7c6fbdc 100644 --- a/federation/sds_federation/services/local_events.py +++ b/federation/sds_federation/services/local_events.py @@ -166,9 +166,16 @@ async def run_federation_subscriber( config.site.name, gateway_site, ) - client = aioredis.from_url(redis_url) + # socket_timeout=None: idle pubsub.listen() must not raise TimeoutError and + # kill the subscriber task (that fails /sync/health redis_subscriber check). + client = aioredis.from_url( + redis_url, + socket_timeout=None, + socket_connect_timeout=5.0, + ) pubsub = client.pubsub() await pubsub.subscribe(resolved_channel) + log.info(f"Subscribed to federation Redis channel {resolved_channel}") try: async for message in pubsub.listen(): if stop.is_set(): diff --git a/federation/tests/test_integration_webhooks.py b/federation/tests/test_integration_webhooks.py index 2fd76a87b..bfdd21ae5 100644 --- a/federation/tests/test_integration_webhooks.py +++ b/federation/tests/test_integration_webhooks.py @@ -161,7 +161,7 @@ async def test_site_hello_backfills_peer_exports( recording_opensearch: RecordingOpenSearch, ) -> None: config = make_peer_config() - doc = sample_federated_dataset_doc(site_name="testsite") + doc = sample_federated_dataset_doc(site_name="localhost") export_hits = {"datasets": 0, "captures": 0} # Intentionally differs from federation.toml peer sync URL. hello_sync_url = "http://dynamic-testsite.test/sync" @@ -196,7 +196,7 @@ def handler(request: httpx.Request) -> httpx.Response: response = await client.post( f"{SYNC_API_PREFIX}/webhook/site-hello", json={ - "site_name": "testsite", + "site_name": "localhost", "fqdn": "localhost", "display_name": "Originating test site", "sync_service_url": hello_sync_url, diff --git a/federation/uv.lock b/federation/uv.lock index 76bd7755d..4a7f950ff 100644 --- a/federation/uv.lock +++ b/federation/uv.lock @@ -580,7 +580,7 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.25.0" }, { name = "redis", specifier = ">=5.2.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.0" }, - { name = "sds-opensearch-query", directory = "../common/sds_opensearch_query" }, + { name = "sds-opensearch-query", directory = "../common" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, ] provides-extras = ["dev"] @@ -588,7 +588,7 @@ provides-extras = ["dev"] [[package]] name = "sds-opensearch-query" version = "0.1.0" -source = { directory = "../common/sds_opensearch_query" } +source = { directory = "../common" } dependencies = [ { name = "opensearch-py" }, { name = "requests" }, diff --git a/gateway/.envs/example/django.env b/gateway/.envs/example/django.env index 500d77454..b56220cd6 100644 --- a/gateway/.envs/example/django.env +++ b/gateway/.envs/example/django.env @@ -18,10 +18,8 @@ API_KEY= # FEDERATION_ENABLED=true # RFC [site].name (short peer id); set SDS_SITE_FQDN separately for [site].fqdn. # FEDERATION_SITE_NAME=crc -# Redis pub/sub channel federation-sync subscribes to. -# FEDERATION_EVENTS_CHANNEL=federation:events -# Health probe target (federation-sync service). -# FEDERATION_SYNC_HEALTH_URL=http://federation-sync:8000/sync/health +# Health probe target (federation-sync service), replace {environment} with the environment you are deploying in. +FEDERATION_SYNC_HEALTH_URL=http://sds-federation-{environment}-sync:8000/sync/health # FEDERATION_SYNC_USER_EMAIL=federation-sync@internal.local # Comma-separated CIDRs allowed to call export (default: private Docker ranges). # FEDERATION_EXPORT_ALLOWED_CIDRS= diff --git a/gateway/compose.ci.yaml b/gateway/compose.ci.yaml index 2a02b5e35..eaeb98825 100644 --- a/gateway/compose.ci.yaml +++ b/gateway/compose.ci.yaml @@ -33,8 +33,8 @@ networks: services: sds-gateway-ci-app: build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile image: sds-gateway-ci-app container_name: sds-gateway-ci-app tty: true # colored logs @@ -252,8 +252,8 @@ services: # Celery services for background tasks celery-worker: build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile image: sds-gateway-ci-app container_name: sds-gateway-ci-celery-worker tty: true @@ -310,8 +310,8 @@ services: celery-beat: # Celery Beat scheduler for periodic tasks build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile image: sds-gateway-ci-app container_name: sds-gateway-ci-celery-beat tty: true @@ -368,8 +368,8 @@ services: celery-flower: # Celery monitoring and administration tool build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile image: sds-gateway-ci-app container_name: sds-gateway-ci-celery-flower tty: true diff --git a/gateway/compose.local.yaml b/gateway/compose.local.yaml index c9d4d25aa..935bb15db 100644 --- a/gateway/compose.local.yaml +++ b/gateway/compose.local.yaml @@ -39,8 +39,8 @@ networks: services: sds-gateway-local-app: build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile args: GIT_COMMIT: ${GIT_COMMIT:-none} image: sds-gateway-local-app @@ -262,8 +262,8 @@ services: # Celery services for background tasks celery-worker: build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile args: GIT_COMMIT: ${GIT_COMMIT:-none} image: sds-gateway-local-app @@ -323,8 +323,8 @@ services: celery-beat: # Celery Beat scheduler for periodic tasks build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile args: GIT_COMMIT: ${GIT_COMMIT:-none} image: sds-gateway-local-app diff --git a/gateway/compose/local/django/Dockerfile b/gateway/compose/local/django/Dockerfile index 2692b8f90..c85a6542c 100644 --- a/gateway/compose/local/django/Dockerfile +++ b/gateway/compose/local/django/Dockerfile @@ -6,11 +6,12 @@ RUN pnpm runtime set node 24 -g && cp /pnpm/bin/node /usr/local/bin/node && rm / ARG APP_HOME=/app WORKDIR ${APP_HOME} -COPY ./package.json ${APP_HOME} -COPY ./pnpm-lock.yaml ${APP_HOME} -COPY ./pnpm-workspace.yaml ${APP_HOME} +# Build context is the monorepo root (compose: context: ..). +COPY gateway/package.json ${APP_HOME} +COPY gateway/pnpm-lock.yaml ${APP_HOME} +COPY gateway/pnpm-workspace.yaml ${APP_HOME} RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile -COPY . ${APP_HOME} +COPY gateway/ ${APP_HOME} RUN pnpm run build # https://hub.docker.com/_/python/tags?name=3.13 @@ -64,22 +65,25 @@ ENV UV_PROJECT_ENVIRONMENT=/opt/uv-venv/ RUN mkdir -p /opt/uv-cache/ /opt/uv-venv/ /opt/uv-venv-seed/ +# Path dep ../common from /app → /common +COPY common /common + # Bake third-party dependencies, then the project, into a seed virtualenv. -COPY ./pyproject.toml ./uv.lock ${APP_HOME}/ +COPY gateway/pyproject.toml gateway/uv.lock ${APP_HOME}/ RUN UV_PROJECT_ENVIRONMENT=${UV_PROJECT_SEED} uv sync --frozen --no-install-project --extra ${BUILD_ENVIRONMENT} # "entrypoint" only has a production version -COPY ./compose/production/django/entrypoint /entrypoint +COPY gateway/compose/production/django/entrypoint /entrypoint # copy command scripts and set exec flag -COPY ./compose/local/django/start /start -COPY ./compose/local/django/celery/flower-start /flower-start -COPY ./compose/local/django/celery/worker-start /worker-start -COPY ./compose/local/django/celery/beat-start /beat-start +COPY gateway/compose/local/django/start /start +COPY gateway/compose/local/django/celery/flower-start /flower-start +COPY gateway/compose/local/django/celery/worker-start /worker-start +COPY gateway/compose/local/django/celery/beat-start /beat-start RUN chmod +x /entrypoint /start /worker-start /flower-start /beat-start # copy application code to WORKDIR -COPY . ${APP_HOME} +COPY gateway/ ${APP_HOME} RUN UV_PROJECT_ENVIRONMENT=${UV_PROJECT_SEED} uv sync --frozen --extra ${BUILD_ENVIRONMENT} # Generate version.json with baked-in commit hash (passed as build ARG) diff --git a/gateway/compose/production/django/Dockerfile b/gateway/compose/production/django/Dockerfile index d1657be64..46abc171f 100644 --- a/gateway/compose/production/django/Dockerfile +++ b/gateway/compose/production/django/Dockerfile @@ -6,11 +6,12 @@ RUN pnpm runtime set node 24 -g && cp /pnpm/bin/node /usr/local/bin/node && rm / ARG APP_HOME=/app WORKDIR ${APP_HOME} -COPY ./package.json ${APP_HOME} -COPY ./pnpm-lock.yaml ${APP_HOME} -COPY ./pnpm-workspace.yaml ${APP_HOME} +# Build context is the monorepo root (compose/CI: context: .. / .). +COPY gateway/package.json ${APP_HOME} +COPY gateway/pnpm-lock.yaml ${APP_HOME} +COPY gateway/pnpm-workspace.yaml ${APP_HOME} RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile -COPY . ${APP_HOME} +COPY gateway/ ${APP_HOME} RUN pnpm run build # https://hub.docker.com/_/python/tags?name=3.13 @@ -67,16 +68,19 @@ RUN mkdir -p /opt/uv-cache/ /opt/uv-venv/ /opt/uv-venv-seed/ RUN addgroup --system django \ && adduser --system --ingroup django django +# Path dep ../common from /app → /common +COPY common /common + # Bake third-party dependencies first; install the project after app code is copied. -COPY ./pyproject.toml ./uv.lock ${APP_HOME}/ +COPY gateway/pyproject.toml gateway/uv.lock ${APP_HOME}/ RUN UV_PROJECT_ENVIRONMENT=${UV_PROJECT_SEED} uv sync --frozen --no-install-project --extra ${BUILD_ENVIRONMENT} # copy command scripts and set exec flag -COPY --chown=django:django ./compose/production/django/entrypoint /entrypoint -COPY --chown=django:django ./compose/production/django/start /start -COPY --chown=django:django ./compose/production/django/celery/worker-start /worker-start -COPY --chown=django:django ./compose/production/django/celery/flower-start /flower-start -COPY --chown=django:django ./compose/production/django/celery/beat-start /beat-start +COPY --chown=django:django gateway/compose/production/django/entrypoint /entrypoint +COPY --chown=django:django gateway/compose/production/django/start /start +COPY --chown=django:django gateway/compose/production/django/celery/worker-start /worker-start +COPY --chown=django:django gateway/compose/production/django/celery/flower-start /flower-start +COPY --chown=django:django gateway/compose/production/django/celery/beat-start /beat-start RUN chmod +x /entrypoint /start /worker-start /flower-start /beat-start diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index 40f8b5eb5..b1cbc5fc6 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -775,7 +775,7 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: FEDERATION_SYNC_DRF_TOKEN: str = env.str("FEDERATION_SYNC_DRF_TOKEN", default="") FEDERATION_SYNC_HEALTH_URL: str = env.str( "FEDERATION_SYNC_HEALTH_URL", - default="http://federation-sync:8000/sync/health", + default="http://federation-local-sync:8000/sync/health", ) FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT: float = env.float( "FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT", diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml index e468d86b7..841ea7ca4 100644 --- a/gateway/pyproject.toml +++ b/gateway/pyproject.toml @@ -666,4 +666,4 @@ exclude-newer = "1 week" [tool.uv.sources] -sds-opensearch-query = { path = "../common/sds_opensearch_query" } +sds-opensearch-query = { path = "../common" } diff --git a/gateway/sds_gateway/api_methods/apps.py b/gateway/sds_gateway/api_methods/apps.py index db8856579..62cf738b8 100644 --- a/gateway/sds_gateway/api_methods/apps.py +++ b/gateway/sds_gateway/api_methods/apps.py @@ -1,3 +1,4 @@ +import asyncio import logging import sys @@ -19,6 +20,14 @@ def _skip_federation_init_in_ready() -> bool: } +def _in_async_context() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + class ApiMethodsConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "sds_gateway.api_methods" @@ -41,7 +50,7 @@ def _init_federation_after_migrate(sender, **kwargs) -> None: dispatch_uid="api_methods_federation_operational_init", ) - if not _skip_federation_init_in_ready(): + if not _skip_federation_init_in_ready() and not _in_async_context(): initialize_federation_operational_state() silence_unwanted_logs() diff --git a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py index a8deb619c..81b8c0c64 100644 --- a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py +++ b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py @@ -289,6 +289,7 @@ class DatasetPublicSerializer(serializers.ModelSerializer[Dataset]): authors = serializers.SerializerMethodField() keywords = serializers.SerializerMethodField() + institutions = serializers.SerializerMethodField() created_at = serializers.DateTimeField( format=READABLE_ISO_DATE_TIME, read_only=True ) @@ -303,6 +304,13 @@ def get_keywords(self, obj): """Return a list of keyword names for the dataset.""" return [kw.name for kw in obj.keywords.filter(is_deleted=False)] + def get_institutions(self, obj): + if not obj.institutions: + return [] + if isinstance(obj.institutions, list): + return obj.institutions + return [] + def get_owner_name(self, obj): """Get the owner's display name.""" return obj.owner.name if obj.owner else "Owner" @@ -338,8 +346,12 @@ class DatasetFederationSerializer(DatasetPublicSerializer): """Serializer for dataset data for federation export.""" site_name = serializers.SerializerMethodField() + created_at = serializers.DateTimeField( + format="iso-8601", # ISO 8601 format for OpenSearch schema + read_only=True, + ) updated_at = serializers.DateTimeField( - format=READABLE_ISO_DATE_TIME, + format="iso-8601", # ISO 8601 format for OpenSearch schema read_only=True, ) size = serializers.SerializerMethodField() diff --git a/gateway/uv.lock b/gateway/uv.lock index 4710c9172..4a13c3b3e 100644 --- a/gateway/uv.lock +++ b/gateway/uv.lock @@ -3,7 +3,7 @@ revision = 3 requires-python = "==3.13.*" [options] -exclude-newer = "2026-07-23T16:20:34.903318Z" +exclude-newer = "2026-07-23T18:25:15.89876Z" exclude-newer-span = "P1W" [[package]] @@ -2222,7 +2222,7 @@ requires-dist = [ { name = "rich", marker = "extra == 'local'", specifier = ">=13.9.3" }, { name = "ruff", marker = "extra == 'local'", specifier = ">=0.7" }, { name = "scipy", specifier = ">=1.16.1" }, - { name = "sds-opensearch-query", directory = "../common/sds_opensearch_query" }, + { name = "sds-opensearch-query", directory = "../common" }, { name = "sentry-sdk", extras = ["django"], specifier = ">=2.25.1" }, { name = "uvicorn", specifier = ">=0.32.0" }, { name = "uvicorn-worker", specifier = ">=0.2.0" }, @@ -2235,7 +2235,7 @@ provides-extras = ["local", "production"] [[package]] name = "sds-opensearch-query" version = "0.1.0" -source = { directory = "../common/sds_opensearch_query" } +source = { directory = "../common" } dependencies = [ { name = "opensearch-py" }, { name = "requests" }, From bb6525cafba877183aa8092e6d1db848356af648 Mon Sep 17 00:00:00 2001 From: klpoland Date: Tue, 4 Aug 2026 08:24:03 -0400 Subject: [PATCH 07/22] remove local_e2e scripts from tree --- federation/scripts/local_e2e/README.md | 89 -------------- .../scripts/local_e2e/download_crc_dataset.py | 82 ------------- .../local_e2e/download_drf_properties.py | 110 ------------------ federation/scripts/local_e2e/env.example | 19 --- .../local_e2e/upload_capture_to_local.py | 91 --------------- .../local_e2e/verify_federation_live.py | 79 ------------- 6 files changed, 470 deletions(-) delete mode 100644 federation/scripts/local_e2e/README.md delete mode 100644 federation/scripts/local_e2e/download_crc_dataset.py delete mode 100644 federation/scripts/local_e2e/download_drf_properties.py delete mode 100644 federation/scripts/local_e2e/env.example delete mode 100644 federation/scripts/local_e2e/upload_capture_to_local.py delete mode 100644 federation/scripts/local_e2e/verify_federation_live.py diff --git a/federation/scripts/local_e2e/README.md b/federation/scripts/local_e2e/README.md deleted file mode 100644 index 9f63f7d20..000000000 --- a/federation/scripts/local_e2e/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Local federation live test (gateway + sync + OpenSearch) - -End-to-end flow: pull real RF data from CRC → load into **local** gateway → publish for federation → index in `fed-*` → query sync search. - -**Never commit API tokens.** Use `env.example` → copy to `.env` in this directory. - -## Prerequisites - -1. Gateway stack up (`gateway` compose local): app, Redis, OpenSearch, MinIO. -2. Federation sync up (`federation/compose.local.yaml`) on port **8001**. -3. Gateway env (e.g. `django.env`): - - `FEDERATION_ENABLED=true` - - `FEDERATION_SITE_NAME=crc` (short id) - - `SDS_SITE_FQDN=localhost` (must match `federation.toml` `[site].fqdn` for local) - - `FEDERATION_EXPORT_ALLOWED_CIDRS` includes your sync container / dev machine -4. Federation sync env: `FEDERATION_GATEWAY_API_KEY` from: - ```bash - docker compose exec sds-gateway-local-app python manage.py create_federation_sync_api_key - ``` -5. OpenSearch capture indices: `python manage.py init_indices` (gateway). - -## 1. Download from CRC (read-only) - -```bash -cd sdk -cp ../federation/scripts/local_e2e/env.example ../federation/scripts/local_e2e/.env -# Edit .env: SDS_SECRET_TOKEN=... (your user token; rotate if exposed) - -set -a && source ../federation/scripts/local_e2e/.env && set +a -uv run python ../federation/scripts/local_e2e/download_crc_dataset.py \ - --dataset-uuid 50e979bd-8018-415c-8212-c08c3dc98654 \ - --to ../federation/data/downloaded_dataset -``` - -To mimic a Haystack-style folder (one time slice), discover `top_level_dir` on CRC with the SDK (`list_dataset_captures`) then pass `--top-level-dir`. - -**Size warning:** full datasets can be huge; prefer `--top-level-dir` or `--skip-contents` for metadata-only federation tests. - -## 2. Upload to local gateway - -Use a **local** user token (`LOCAL_SDS_SECRET_TOKEN` in `.env`). - -```bash -uv run python ../federation/scripts/local_e2e/upload_capture_to_local.py \ - --local-path ../federation/data/downloaded_dataset \ - --sds-path federation-fixture/starlink-sample -``` - -Note the printed **capture UUID**. - -## 3. Create / publish dataset - -Create a dataset in the UI or API, attach the capture, then: - -```bash -docker compose exec sds-gateway-local-app python manage.py publish_for_federation \ - --dataset-uuid \ - --capture-uuids -``` - -This sets `FINAL` + `is_public` and marks captures public (fires `federation:events` when enabled). - -## 4. Re-index federation - -Restart sync (bootstrap on start) or trigger an update: - -```bash -cd federation -just simulate-redis --event-type updated --item-type dataset --uuid -``` - -## 5. Verify - -```bash -cd federation -uv run python scripts/local_e2e/verify_federation_live.py --q starlink -``` - -Or: - -```bash -curl -s 'http://localhost:8001/api/v1/search/datasets?site=localhost&q=fixture' | jq . -curl -s 'http://localhost:8000/api/v1/federation/export/datasets/' \ - -H "Authorization: Api-Key $FEDERATION_GATEWAY_API_KEY" | jq . -``` - -## Haystack public HTTP (optional) - -Apollo directory listings (e.g. `rf@*.h5` under `Vpol_11.325GHz/...`) are the same kind of DigitalRF files CRC stores. You can `wget`/`curl` a **small** subset into `federation/data/apollo_sample/` and upload with the same `upload_capture_to_local.py` script instead of using `download_dataset`. diff --git a/federation/scripts/local_e2e/download_crc_dataset.py b/federation/scripts/local_e2e/download_crc_dataset.py deleted file mode 100644 index 4f810bf82..000000000 --- a/federation/scripts/local_e2e/download_crc_dataset.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -"""Download a public dataset from CRC (or SDS_HOST) for local federation fixtures. - -Run from the SDK environment (spectrumx is not a federation dependency): - - cd sdk - cp ../federation/scripts/local_e2e/env.example ../federation/scripts/local_e2e/.env - # Edit .env: SDS_SECRET_TOKEN=... - - set -a && source ../federation/scripts/local_e2e/.env && set +a - uv run python ../federation/scripts/local_e2e/download_crc_dataset.py \\ - --dataset-uuid 50e979bd-8018-415c-8212-c08c3dc98654 \\ - --to ./../federation/data/downloaded_dataset - -Or from ``federation/``: ``just local-e2e-env`` then ``just download-crc-dataset ``. - -Use ``--top-level-dir`` to limit to one capture tree (Haystack-style DRF folders). -""" - -from __future__ import annotations - -import argparse -import os -import sys -from pathlib import Path -from uuid import UUID - -from spectrumx import Client -from spectrumx.errors import SDSError - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--dataset-uuid", required=True) - parser.add_argument( - "--to", - type=Path, - default=Path("downloaded_dataset"), - help="Local download root", - ) - parser.add_argument( - "--top-level-dir", - default="", - help="Optional SDS top_level_dir filter (one capture)", - ) - parser.add_argument("--overwrite", action="store_true") - parser.add_argument("--skip-contents", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = _parse_args() - host = os.environ.get("SDS_HOST", "sds.crc.nd.edu") - client = Client(host=host) - client.dry_run = False - client.authenticate() - - dataset_uuid = UUID(args.dataset_uuid) - top_level_dirs = [args.top_level_dir] if args.top_level_dir.strip() else None - - print(f"Downloading {dataset_uuid} from {host} -> {args.to.resolve()}") - try: - results = client.download_dataset( - dataset_uuid=dataset_uuid, - to_local_path=args.to, - top_level_dirs=top_level_dirs, - skip_contents=args.skip_contents, - overwrite=args.overwrite, - verbose=True, - ) - except SDSError as exc: - print(f"Download failed: {exc}", file=sys.stderr) - return 1 - - ok = sum(1 for r in results if r) - fail = len(results) - ok - print(f"Done: {ok} ok, {fail} failed") - return 0 if fail == 0 else 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/federation/scripts/local_e2e/download_drf_properties.py b/federation/scripts/local_e2e/download_drf_properties.py deleted file mode 100644 index 742c30012..000000000 --- a/federation/scripts/local_e2e/download_drf_properties.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python3 -"""Download only drf_properties.h5 files for a dataset (optionally one capture). - - cd sdk - set -a && source ../federation/scripts/local_e2e/.env && set +a - uv run python ../federation/scripts/local_e2e/download_drf_properties.py \\ - --dataset-uuid 50e979bd-8018-415c-8212-c08c3dc98654 \\ - --to ../federation/data/downloaded_dataset \\ - --top-level-dir '/files/rherban@nd.edu/CU_conference_NCAR_002_b08a_009f70' -""" - -from __future__ import annotations - -import argparse -import os -import sys -from pathlib import Path -from uuid import UUID - -from spectrumx import Client -from spectrumx.errors import SDSError - -PROPS_NAME = "drf_properties.h5" - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--dataset-uuid", required=True) - p.add_argument( - "--to", - type=Path, - default=Path("../federation/data/downloaded_dataset"), - help="Local root (same as download_crc_dataset --to)", - ) - p.add_argument( - "--top-level-dir", - default="", - help="Optional capture top_level_dir from list_dataset_captures", - ) - p.add_argument("--capture-uuid", default="", help="Optional capture UUID filter") - p.add_argument("--list-only", action="store_true", help="List matches, no download") - p.add_argument("--overwrite", action="store_true") - return p.parse_args() - - -def main() -> int: - args = _parse_args() - host = os.environ.get("SDS_HOST", "sds.crc.nd.edu") - client = Client(host=host) - client.dry_run = False - client.authenticate() - - dataset_uuid = UUID(args.dataset_uuid) - top_level_dirs = [args.top_level_dir] if args.top_level_dir.strip() else None - capture_uuids = [UUID(args.capture_uuid)] if args.capture_uuid.strip() else None - - # Helpful: show captures if you need the exact top_level_dir - if not top_level_dirs and not capture_uuids: - caps = client.list_dataset_captures(dataset_uuid) - print(f"Captures in dataset ({len(caps)}):") - for c in caps: - print( - f" uuid={c.get('uuid')} " - f"top_level_dir={c.get('top_level_dir')} " - f"channels={c.get('channels') or c.get('channel')}" - ) - - files = client.datasets.get_files( - dataset_uuid, - capture_uuids=capture_uuids, - top_level_dirs=top_level_dirs, - ) - props = [f for f in files if f.name == PROPS_NAME] - - if not props: - print( - f"No {PROPS_NAME} in dataset manifest " - f"(filters: top_level_dir={top_level_dirs}, capture={capture_uuids}).\n" - "If empty, CRC never indexed that file for this capture — " - "recreate locally with digital_rf instead.", - file=sys.stderr, - ) - return 2 - - for f in props: - print(f" {f.uuid} {f.directory}/{f.name} size={f.size}") - - if args.list_only: - return 0 - - print(f"Downloading {len(props)} file(s) from {host} -> {args.to.resolve()}") - try: - results = client.download( - to_local_path=args.to, - files_to_download=props, - overwrite=args.overwrite, - verbose=True, - ) - except SDSError as exc: - print(f"Download failed: {exc}", file=sys.stderr) - return 1 - - ok = sum(1 for r in results if r) - fail = len(results) - ok - print(f"Done: {ok} ok, {fail} failed") - return 0 if fail == 0 else 2 - - -if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file diff --git a/federation/scripts/local_e2e/env.example b/federation/scripts/local_e2e/env.example deleted file mode 100644 index fa69224a3..000000000 --- a/federation/scripts/local_e2e/env.example +++ /dev/null @@ -1,19 +0,0 @@ -# Copy to federation/scripts/local_e2e/.env (never commit secrets). -# CRC download (production read-only): -SDS_HOST=sds.crc.nd.edu -SDS_SECRET_TOKEN= - -# Optional: filter download to one capture directory (from list_dataset_captures) -# DATASET_UUID= -# TOP_LEVEL_DIR=/files/you@example.com/federation-fixture/... - -# Local gateway upload (after download): -LOCAL_SDS_HOST=localhost:8000 -LOCAL_SDS_SECRET_TOKEN= - -# Federation sync (gateway export for bootstrap): -FEDERATION_GATEWAY_API_KEY= - -# Live checks -FEDERATION_SYNC_URL=http://localhost:8001 -LOCAL_SITE_FQDN=localhost diff --git a/federation/scripts/local_e2e/upload_capture_to_local.py b/federation/scripts/local_e2e/upload_capture_to_local.py deleted file mode 100644 index 4fc3f9d5d..000000000 --- a/federation/scripts/local_e2e/upload_capture_to_local.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -"""Upload a local DigitalRF directory to your local gateway as a new capture. - - cd sdk - cp ../federation/scripts/local_e2e/env.example ../federation/scripts/local_e2e/.env - # Edit .env: LOCAL_SDS_SECRET_TOKEN=... - - set -a && source ../federation/scripts/local_e2e/.env && set +a - uv run python ../federation/scripts/local_e2e/upload_capture_to_local.py \\ - --local-path ../federation/data/downloaded_dataset \\ - --sds-path federation-fixture/starlink-sample - -Prints the new capture UUID for publish_for_federation / dataset linking. -""" - -from __future__ import annotations - -import argparse -import os -import sys -from pathlib import Path -from pathlib import PurePosixPath - -from spectrumx import Client -from spectrumx.errors import SDSError -from spectrumx.models.captures import CaptureType - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--local-path", - type=Path, - required=True, - help="Directory containing DRF/HDF5 files (downloaded tree)", - ) - parser.add_argument( - "--sds-path", - default="federation-fixture/sample", - help="Virtual path under your SDS user root", - ) - parser.add_argument("--channel", default="0") - parser.add_argument("--name", default="Federation local fixture capture") - return parser.parse_args() - - -def main() -> int: - args = _parse_args() - host = os.environ.get("LOCAL_SDS_HOST", "localhost:8000") - host = host.removeprefix("https://").removeprefix("http://") - env_config: dict[str, str] = {} - token = os.environ.get("LOCAL_SDS_SECRET_TOKEN") or os.environ.get( - "SDS_SECRET_TOKEN", - "", - ) - if token: - env_config["SDS_SECRET_TOKEN"] = token - - client = Client(host=host, env_config=env_config or None) - client.dry_run = False - client.authenticate() - - if not args.local_path.is_dir(): - print(f"Not a directory: {args.local_path}", file=sys.stderr) - return 1 - - print(f"Uploading {args.local_path} -> {args.sds_path} on {client.host}") - try: - capture = client.upload_capture( - local_path=args.local_path, - sds_path=PurePosixPath(args.sds_path), - capture_type=CaptureType.DigitalRF, - channel=args.channel, - name=args.name, - verbose=True, - ) - except SDSError as exc: - print(f"Upload failed: {exc}", file=sys.stderr) - return 1 - - if capture is None: - print("Upload returned no capture", file=sys.stderr) - return 2 - - print(f"Capture UUID: {capture.uuid}") - print(f"top_level_dir: {capture.top_level_dir}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/federation/scripts/local_e2e/verify_federation_live.py b/federation/scripts/local_e2e/verify_federation_live.py deleted file mode 100644 index a269ad6ba..000000000 --- a/federation/scripts/local_e2e/verify_federation_live.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -"""Smoke-test federation export + search against a running local sync service.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys - -import httpx - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--sync-base", - default=os.environ.get("FEDERATION_SYNC_URL", "http://localhost:8001"), - ) - parser.add_argument( - "--site-fqdn", - default=os.environ.get("LOCAL_SITE_FQDN", "localhost"), - ) - parser.add_argument("--q", default="") - parser.add_argument( - "--gateway-export", - default=os.environ.get( - "GATEWAY_EXPORT_URL", - "http://localhost:8000/api/v1/federation/export/datasets/", - ), - ) - parser.add_argument( - "--api-key", - default=os.environ.get("FEDERATION_GATEWAY_API_KEY", ""), - ) - return parser.parse_args() - - -def main() -> int: - args = _parse_args() - base = args.sync_base.rstrip("/") - - with httpx.Client(timeout=30.0) as client: - health = client.get(f"{base}/health") - print(f"health {health.status_code}: {health.text[:200]}") - if health.status_code != 200: - return 1 - - headers = {} - if args.api_key: - headers["Authorization"] = f"Api-Key: {args.api_key}" - export = client.get(args.gateway_export, headers=headers) - print(f"gateway export {export.status_code}") - if export.status_code == 200: - data = export.json() - print(f" export datasets: {len(data)}") - if data: - print(f" first site_name: {data[0].get('site_name')}") - - params = {"site": args.site_fqdn} - if args.q: - params["q"] = args.q - search = client.get( - f"{base}/api/v1/search/datasets", - params=params, - ) - print(f"search datasets {search.status_code}") - if search.status_code == 200: - body = search.json() - print(json.dumps(body, indent=2)[:1500]) - else: - print(search.text, file=sys.stderr) - return 2 - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From b721a44805cba0596059a3f7cdf8b30074895481 Mon Sep 17 00:00:00 2001 From: klpoland Date: Tue, 4 Aug 2026 12:51:44 -0400 Subject: [PATCH 08/22] gh actions and p2p setup optimization --- .github/workflows/fed-code-quality.yaml | 103 ++++++++++++++ federation/.gitignore | 5 +- federation/compose.peer.local.yaml | 14 +- federation/justfile | 41 +----- federation/pyproject.toml | 2 + .../scripts/deploy-local-peer-2-peer.sh | 126 ++++++++++++++++++ federation/scripts/verify_federation_live.py | 79 +++++++++++ federation/sds_federation/routes/webhooks.py | 48 ++++--- .../sds_federation/services/bootstrap.py | 77 +++++++++-- .../sds_federation/services/fed_index.py | 12 +- .../sds_federation/services/peer_sync.py | 8 +- federation/tests/test_integration_mesh.py | 4 +- federation/tests/test_integration_webhooks.py | 2 +- .../tests/test_regression_fed_indices.py | 4 +- federation/tests/test_regression_fed_list.py | 2 +- .../tests/test_regression_peer_list_retry.py | 91 +++++++++++++ federation/tests/test_regression_peer_sync.py | 2 +- federation/uv.lock | 26 ++++ gateway/config/settings/base.py | 2 +- .../api_methods/federation/availability.py | 4 + .../federation/compile_federated_data.py | 6 +- .../tests/test_federation_events.py | 3 +- .../tests/test_federation_hardening.py | 21 +++ .../tests/test_federation_signals.py | 4 + 24 files changed, 600 insertions(+), 86 deletions(-) create mode 100644 .github/workflows/fed-code-quality.yaml create mode 100755 federation/scripts/deploy-local-peer-2-peer.sh create mode 100644 federation/scripts/verify_federation_live.py create mode 100644 federation/tests/test_regression_peer_list_retry.py diff --git a/.github/workflows/fed-code-quality.yaml b/.github/workflows/fed-code-quality.yaml new file mode 100644 index 000000000..d7e3dcd20 --- /dev/null +++ b/.github/workflows/fed-code-quality.yaml @@ -0,0 +1,103 @@ +# Workflow to run pre-commit checks and pytest for the Federation app +# GitHub Action Workflow validator: https://rhysd.github.io/actionlint/ +name: federation-checks + +on: + workflow_dispatch: + # To manually trigger the workflow + # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_dispatch + push: + paths: + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet + - federation/** + - .pre-commit-config.yaml + - .github/workflows/fed-code-quality.yaml + branches: + - main + - master + pull_request: + paths: + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet + - federation/** + - .pre-commit-config.yaml + - .github/workflows/fed-code-quality.yaml + branches: + - main + - master + types: + # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request + - ready_for_review + - synchronize + +env: + DOCKER_BUILDKIT: 1 + COMPOSE_DOCKER_CLI_BUILD: 1 + +jobs: + # Federation hooks live in the repo-root .pre-commit-config.yaml; + # prek is a federation --extra dev dependency. + fed-pre-commit: + runs-on: ubuntu-latest + env: + UV_LINK_MODE: copy + steps: + - uses: actions/checkout@v7 + + - name: Install uv + uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + # https://github.com/marketplace/actions/astral-sh-setup-uv + + - name: Cache prek hooks + id: cache-prek + uses: actions/cache@v6 + # https://github.com/actions/cache/blob/main/examples.md#python---pip + with: + key: prek-federation-${{ hashFiles('.pre-commit-config.yaml') }} + path: ~/.cache/prek/ + + - name: Sync federation dependencies + working-directory: ./federation + # pyrefly-federation runs: cd federation && uv run --extra dev pyrefly check + run: uv sync --extra dev + + - name: Install hooks + working-directory: ./federation + run: uv run --extra dev prek install --install-hooks + + - name: Run federation prek hooks + working-directory: ./federation + # Only federation-scoped hooks (no biome / djLint / JS). + # prek walks up to the repo-root .pre-commit-config.yaml. + run: | + uv run --extra dev prek run ruff-check-federation --all-files + uv run --extra dev prek run ruff-format-federation --all-files + uv run --extra dev prek run pyrefly-federation --all-files + + # Run federation pytest (mocked deps — no Docker stack required) + fed-tests: + runs-on: ubuntu-latest + strategy: + matrix: + # uv will take care of installing other python versions, + # so we don't need a python-version matrix here. + platform: [ubuntu-latest] + steps: + - uses: actions/checkout@v7 + + - name: Install uv + uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + # https://github.com/marketplace/actions/astral-sh-setup-uv + + - name: Install just on ubuntu + if: matrix.platform == 'ubuntu-latest' + working-directory: ./federation + run: | + npm install -g rust-just + + - name: Sync federation dependencies + working-directory: ./federation + run: uv sync --extra dev + + - name: Run tests + working-directory: ./federation + run: just test diff --git a/federation/.gitignore b/federation/.gitignore index acbbb2d06..f633817ff 100644 --- a/federation/.gitignore +++ b/federation/.gitignore @@ -3,7 +3,6 @@ __pycache__/ federation.toml +federation.peer.toml -.envs/* -!/.envs/example -scripts/local_e2e/.env +scripts/local_e2e/ diff --git a/federation/compose.peer.local.yaml b/federation/compose.peer.local.yaml index b7dac0c3b..d5d1c2695 100644 --- a/federation/compose.peer.local.yaml +++ b/federation/compose.peer.local.yaml @@ -17,6 +17,12 @@ services: ports: - "9201:9200" networks: [peer-internal] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9200/_cluster/health || exit 1"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 30s sds-federation-peer-redis: image: redis:6 @@ -50,8 +56,10 @@ services: ports: - "8002:8000" depends_on: - - sds-federation-peer-opensearch - - sds-federation-peer-redis + sds-federation-peer-opensearch: + condition: service_healthy + sds-federation-peer-redis: + condition: service_started networks: - sds-network-local - - peer-internal \ No newline at end of file + - peer-internal diff --git a/federation/justfile b/federation/justfile index 59de9b0de..8263c7f8e 100644 --- a/federation/justfile +++ b/federation/justfile @@ -60,12 +60,12 @@ test +args='': test-q: uv run pytest -q -# Run federation pre-commit hooks from repo root (requires gateway dev deps / pre-commit install) +# Run federation pre-commit hooks (prek walks up to repo-root .pre-commit-config.yaml) [group('qa')] pre-commit +args='': - cd .. && uv run --directory gateway --extra local pre-commit run ruff-check-federation --all-files {{ args }} - cd .. && uv run --directory gateway --extra local pre-commit run ruff-format-federation --all-files {{ args }} - cd .. && uv run --directory gateway --extra local pre-commit run pyrefly-federation --all-files {{ args }} + uv run --extra dev prek run ruff-check-federation --all-files {{ args }} + uv run --extra dev prek run ruff-format-federation --all-files {{ args }} + uv run --extra dev prek run pyrefly-federation --all-files {{ args }} # dev PKI for mTLS experiments (see docs/mtls-certificates.md) [group('setup')] @@ -85,37 +85,10 @@ simulate-redis +args='': verify-federation-live +args='': uv run python scripts/local_e2e/verify_federation_live.py {{ args }} +#deploy local and peer federation stacks [group('dev')] -local-e2e-env: - @if [ ! -f scripts/local_e2e/.env ]; then cp scripts/local_e2e/env.example scripts/local_e2e/.env; fi - @echo "Edit federation/scripts/local_e2e/.env (SDS_SECRET_TOKEN, LOCAL_SDS_SECRET_TOKEN, …)" - -# Download from CRC (SDK env); run `just local-e2e-env` first -[group('dev')] -download-crc-dataset dataset_uuid to='../federation/data/downloaded_dataset' +args='': - #!/usr/bin/env bash - set -eu -o pipefail - env_file="scripts/local_e2e/.env" - if [[ ! -f "$env_file" ]]; then - echo "Missing $env_file — run: just local-e2e-env" >&2 - exit 1 - fi - set -a && source "$env_file" && set +a - cd ../sdk && uv run python ../federation/scripts/local_e2e/download_crc_dataset.py \ - --dataset-uuid {{ dataset_uuid }} --to {{ to }} {{ args }} - -[group('dev')] -upload-capture-local local_path sds_path='federation-fixture/sample' +args='': - #!/usr/bin/env bash - set -eu -o pipefail - env_file="scripts/local_e2e/.env" - if [[ ! -f "$env_file" ]]; then - echo "Missing $env_file — run: just local-e2e-env" >&2 - exit 1 - fi - set -a && source "$env_file" && set +a - cd ../sdk && uv run python ../federation/scripts/local_e2e/upload_capture_to_local.py \ - --local-path {{ local_path }} --sds-path {{ sds_path }} {{ args }} +deploy-local-peer-2-peer: + ./scripts/deploy-local-peer-2-peer.sh [group('docker')] build +args='': diff --git a/federation/pyproject.toml b/federation/pyproject.toml index 37fc8c546..85c4eff79 100644 --- a/federation/pyproject.toml +++ b/federation/pyproject.toml @@ -25,6 +25,7 @@ module-root = "" [project.optional-dependencies] dev = [ "deptry>=0.24.0", + "prek>=0.3.4", "pyrefly>=0.42.1", "pytest>=8.3.0", "pytest-asyncio>=0.25.0", @@ -37,6 +38,7 @@ dev = [ [tool.deptry.per_rule_ignores] DEP002 = [ "deptry", + "prek", "pyrefly", "pytest", "pytest-asyncio", diff --git a/federation/scripts/deploy-local-peer-2-peer.sh b/federation/scripts/deploy-local-peer-2-peer.sh new file mode 100755 index 000000000..c5397b5d3 --- /dev/null +++ b/federation/scripts/deploy-local-peer-2-peer.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Bring up local + peer federation sync stacks for p2p testing. +# Prereq: gateway local stack can be started from ../gateway. +set -euo pipefail + +FEDERATION_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GATEWAY_ROOT="${FEDERATION_ROOT}/../gateway" +LOCAL_COMPOSE="${FEDERATION_ROOT}/compose.local.yaml" +PEER_COMPOSE="${FEDERATION_ROOT}/compose.peer.local.yaml" +GATEWAY_COMPOSE="${GATEWAY_ROOT}/compose.local.yaml" + +GATEWAY_URL="${GATEWAY_URL:-http://localhost:8000}" +GATEWAY_HEALTH_PATH="${GATEWAY_HEALTH_PATH:-/}" +WAIT_SECS="${WAIT_SECS:-180}" +LOCAL_SYNC_URL="${LOCAL_SYNC_URL:-http://localhost:8001/sync/health}" +PEER_SYNC_URL="${PEER_SYNC_URL:-http://localhost:8002/sync/health}" +PEER_OS_URL="${PEER_OS_URL:-http://localhost:9201/_cluster/health}" + +cd "${FEDERATION_ROOT}" + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +info() { printf '==> %s\n' "$*"; } + +require_file() { + [[ -f "$1" ]] || die "missing $1" +} + +wait_http_ok() { + local url=$1 label=$2 + local deadline=$((SECONDS + WAIT_SECS)) + info "Waiting for ${label} (${url})" + while ((SECONDS < deadline)); do + if curl -fsS -o /dev/null --max-time 3 "${url}"; then + info "${label} is up" + return 0 + fi + sleep 2 + done + die "${label} not healthy after ${WAIT_SECS}s: ${url}" +} + +# Require /sync/health JSON status == "ok" (not merely HTTP 200/503). +wait_sync_operational() { + local url=$1 label=$2 + local deadline=$((SECONDS + WAIT_SECS)) + info "Waiting for ${label} operational (${url})" + while ((SECONDS < deadline)); do + body="$(curl -fsS --max-time 3 "${url}" 2>/dev/null || true)" + if [[ -n "${body}" ]] && printf '%s' "${body}" | grep -q '"status"[[:space:]]*:[[:space:]]*"ok"'; then + info "${label} operational" + return 0 + fi + sleep 2 + done + die "${label} not operational after ${WAIT_SECS}s: ${url}" +} + +restart_and_wait_sync() { + local container=$1 url=$2 label=$3 + info "Restarting ${container} so site-hello/bootstrap see a live peer" + docker restart "${container}" >/dev/null + wait_sync_operational "${url}" "${label}" +} + +# --- configs --- +info "Checking federation configs" +require_file "${LOCAL_COMPOSE}" +require_file "${PEER_COMPOSE}" +require_file "${FEDERATION_ROOT}/federation.toml" +require_file "${FEDERATION_ROOT}/federation.peer.toml" +require_file "${FEDERATION_ROOT}/../federation-shared.env" +require_file "${GATEWAY_ROOT}/.envs/local/opensearch.env" + +# Optional sanity: peer sync URL must be Docker DNS, not localhost +if ! grep -q 'sds-federation-peer-sync' "${FEDERATION_ROOT}/federation.toml"; then + die "federation.toml [[peers]] should use sync_service_url → sds-federation-peer-sync" +fi +if ! grep -q 'sds-federation-local-sync' "${FEDERATION_ROOT}/federation.peer.toml"; then + die "federation.peer.toml [[peers]] should use sync_service_url → sds-federation-local-sync" +fi + +# --- gateway --- +info "Ensuring gateway local stack is running" +require_file "${GATEWAY_COMPOSE}" +( + cd "${GATEWAY_ROOT}" + # Start if missing; no-op if already up + COMPOSE_FILE=compose.local.yaml docker compose \ + --env-file .envs/local/opensearch.env \ + --env-file .envs/local/storage.env \ + up -d --remove-orphans +) + +# External nets required by federation compose +docker network inspect sds-network-local >/dev/null 2>&1 \ + || die "network sds-network-local missing (gateway compose should create it)" +docker network inspect sds-gateway-local-opensearch-net >/dev/null 2>&1 \ + || die "network sds-gateway-local-opensearch-net missing" + +wait_http_ok "${GATEWAY_URL}${GATEWAY_HEALTH_PATH}" "gateway" +docker exec sds-gateway-local-redis redis-cli ping | grep -q PONG \ + || die "gateway redis not responding" + +# --- federation images + up --- +info "Building federation sync image" +docker compose -f "${LOCAL_COMPOSE}" -f "${PEER_COMPOSE}" build + +info "Starting local + peer federation stacks" +docker compose -f "${LOCAL_COMPOSE}" -f "${PEER_COMPOSE}" up -d --remove-orphans + +# Peer OS is slow; assert host :9201 before trusting peer sync bootstrap +wait_http_ok "${PEER_OS_URL}" "peer opensearch" +wait_sync_operational "${LOCAL_SYNC_URL}" "local sync" +wait_sync_operational "${PEER_SYNC_URL}" "peer sync" + +# Local often site-hellos while peer is still starting; peer then pulls empty/missed +# lists. Restart both once peers are listening so mutual registration succeeds. +restart_and_wait_sync sds-federation-local-sync "${LOCAL_SYNC_URL}" "local sync" +restart_and_wait_sync sds-federation-peer-sync "${PEER_SYNC_URL}" "peer sync" + +info "Final health" +curl -sS "${LOCAL_SYNC_URL}" +echo +curl -sS "${PEER_SYNC_URL}" +echo +info "Done. Seed peer OpenSearch if testing peer-owned docs; publish local assets for crc→peer." diff --git a/federation/scripts/verify_federation_live.py b/federation/scripts/verify_federation_live.py new file mode 100644 index 000000000..a269ad6ba --- /dev/null +++ b/federation/scripts/verify_federation_live.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Smoke-test federation export + search against a running local sync service.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys + +import httpx + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--sync-base", + default=os.environ.get("FEDERATION_SYNC_URL", "http://localhost:8001"), + ) + parser.add_argument( + "--site-fqdn", + default=os.environ.get("LOCAL_SITE_FQDN", "localhost"), + ) + parser.add_argument("--q", default="") + parser.add_argument( + "--gateway-export", + default=os.environ.get( + "GATEWAY_EXPORT_URL", + "http://localhost:8000/api/v1/federation/export/datasets/", + ), + ) + parser.add_argument( + "--api-key", + default=os.environ.get("FEDERATION_GATEWAY_API_KEY", ""), + ) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + base = args.sync_base.rstrip("/") + + with httpx.Client(timeout=30.0) as client: + health = client.get(f"{base}/health") + print(f"health {health.status_code}: {health.text[:200]}") + if health.status_code != 200: + return 1 + + headers = {} + if args.api_key: + headers["Authorization"] = f"Api-Key: {args.api_key}" + export = client.get(args.gateway_export, headers=headers) + print(f"gateway export {export.status_code}") + if export.status_code == 200: + data = export.json() + print(f" export datasets: {len(data)}") + if data: + print(f" first site_name: {data[0].get('site_name')}") + + params = {"site": args.site_fqdn} + if args.q: + params["q"] = args.q + search = client.get( + f"{base}/api/v1/search/datasets", + params=params, + ) + print(f"search datasets {search.status_code}") + if search.status_code == 200: + body = search.json() + print(json.dumps(body, indent=2)[:1500]) + else: + print(search.text, file=sys.stderr) + return 2 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/federation/sds_federation/routes/webhooks.py b/federation/sds_federation/routes/webhooks.py index ebe995a1d..f929720fa 100644 --- a/federation/sds_federation/routes/webhooks.py +++ b/federation/sds_federation/routes/webhooks.py @@ -4,10 +4,12 @@ import httpx from fastapi import APIRouter +from fastapi import BackgroundTasks from fastapi import HTTPException from fastapi import Request from loguru import logger +from sds_federation.models import PeerInfo from sds_federation.models import allowed_federated_origin_fqdns from sds_federation.models import site_name_for_federation from sds_federation.schemas.webhooks import AssetTypeEnum @@ -144,8 +146,30 @@ async def list_captures(request: Request) -> list[dict]: return [doc.model_dump(mode="json") for doc in docs] +async def _run_site_hello_backfill( + http: httpx.AsyncClient, + peer: PeerInfo, + indexer: FederatedAssetIndexer, +) -> None: + """Background: pull registering peer's list-* exports (with retries).""" + try: + indexed = await backfill_peer_on_hello(http, peer, indexer) + logger.info( + "site-hello backfill indexed {} document(s) from {} ({})", + indexed, + peer.name, + peer.sync_service_url, + ) + except Exception as exc: # noqa: BLE001 + logger.error("site-hello backfill failed for {}: {}", peer.name, exc) + + @webhooks_router.post("/webhook/site-hello") -async def site_hello(payload: SiteHelloWebhook, request: Request) -> dict: +async def site_hello( + payload: SiteHelloWebhook, + request: Request, + background_tasks: BackgroundTasks, +) -> dict: config = request.app.state.config if payload.site_name == site_name_for_federation(config.site): raise HTTPException( @@ -176,19 +200,13 @@ async def site_hello(payload: SiteHelloWebhook, request: Request) -> dict: ) else: outbound = peer_for_outbound(peer, _peer_registry(request)) - try: - indexed = await backfill_peer_on_hello( - http, - outbound, - _indexer(request), - ) - logger.info( - "site-hello backfill indexed {} document(s) from {} ({})", - indexed, - outbound.name, - outbound.sync_service_url, - ) - except Exception as exc: # noqa: BLE001 - logger.error("site-hello backfill failed for {}: {}", outbound.name, exc) + # Return registered immediately; backfill retries while the peer finishes + # binding so mutual startup races do not fail the hello handshake. + background_tasks.add_task( + _run_site_hello_backfill, + http, + outbound, + _indexer(request), + ) return {"status": "registered", "site_name": hello.site_name} diff --git a/federation/sds_federation/services/bootstrap.py b/federation/sds_federation/services/bootstrap.py index bd6aad2de..6062bbec2 100644 --- a/federation/sds_federation/services/bootstrap.py +++ b/federation/sds_federation/services/bootstrap.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import os from datetime import UTC from datetime import datetime @@ -25,6 +26,10 @@ SITE_HELLO_PATH = "/webhook/site-hello" _MINT_PATH = "/users/get-federation-sync-api-key/" +# Peer sync often receives site-hello while the peer process is still binding. +_PEER_LIST_ATTEMPTS = 5 +_PEER_LIST_BACKOFF_SECS = 0.5 +_HTTP_INTERNAL_ERROR = 500 def _export_list_url(peer: PeerInfo, asset_type: AssetTypeEnum) -> str: @@ -164,24 +169,70 @@ async def fetch_gateway_export_list( return [doc_class.model_validate(item) for item in data] +def _is_retryable_peer_list_error(exc: BaseException) -> bool: + """True for transient connect/read failures while a peer sync is starting.""" + if isinstance( + exc, + ( + httpx.ConnectError, + httpx.ConnectTimeout, + httpx.ReadTimeout, + httpx.WriteTimeout, + httpx.PoolTimeout, + httpx.RemoteProtocolError, + ), + ): + return True + if isinstance(exc, httpx.HTTPStatusError): + return exc.response.status_code >= _HTTP_INTERNAL_ERROR + return False + + async def fetch_peer_sync_list( http: httpx.AsyncClient, peer: PeerInfo, asset_type: AssetTypeEnum, + *, + attempts: int = _PEER_LIST_ATTEMPTS, + backoff_secs: float = _PEER_LIST_BACKOFF_SECS, ) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: - """Pull peer-owned docs from the peer sync service (fed-* OpenSearch export).""" + """Pull peer-owned docs from the peer sync service (fed-* OpenSearch export). + + Retries transient connection/5xx errors so site-hello backfill survives peer + startup races. + """ url = _webhook_list_url(peer, asset_type) - data = await _get_json( - http, - url, - api_key="", - verify=peer.ca_cert_path or True, - ) - if not isinstance(data, list): - msg = f"expected list from {url}, got {type(data).__name__}" - raise TypeError(msg) - doc_class = asset_doc_class(asset_type) - return [doc_class.model_validate(item) for item in data] + last_exc: httpx.HTTPError | None = None + tries = max(1, attempts) + for attempt in range(1, tries + 1): + try: + data = await _get_json( + http, + url, + api_key="", + verify=peer.ca_cert_path or True, + ) + if not isinstance(data, list): + msg = f"expected list from {url}, got {type(data).__name__}" + raise TypeError(msg) + doc_class = asset_doc_class(asset_type) + return [doc_class.model_validate(item) for item in data] + except httpx.HTTPError as exc: + last_exc = exc + if not _is_retryable_peer_list_error(exc) or attempt >= tries: + raise + logger.warning( + "peer sync list {} {} attempt {}/{} failed ({}); retrying in {:.1f}s", + peer.name, + asset_type.value, + attempt, + tries, + exc, + backoff_secs * attempt, + ) + await asyncio.sleep(backoff_secs * attempt) + assert last_exc is not None + raise last_exc def _parse_doc_event_at( @@ -355,7 +406,7 @@ async def bootstrap_all_peers( def peer_by_name(config: FederationConfig, site_name: str) -> PeerInfo | None: for peer in config.peers: - if peer.name == site_name or peer.fqdn == site_name: + if site_name in (peer.name, peer.fqdn): return peer return None diff --git a/federation/sds_federation/services/fed_index.py b/federation/sds_federation/services/fed_index.py index 2b57053fa..5aeef471c 100644 --- a/federation/sds_federation/services/fed_index.py +++ b/federation/sds_federation/services/fed_index.py @@ -1,18 +1,21 @@ from __future__ import annotations import asyncio -from typing import Any from datetime import datetime -from uuid import UUID +from typing import TYPE_CHECKING +from typing import Any from opensearchpy import OpenSearch from opensearchpy.exceptions import NotFoundError +from sds_federation.schemas.opensearch_indices import index_body_for_asset from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import FederatedCaptureDoc from sds_federation.schemas.webhooks import FederatedDatasetDoc from sds_federation.schemas.webhooks import asset_doc_class -from sds_federation.schemas.opensearch_indices import index_body_for_asset + +if TYPE_CHECKING: + from uuid import UUID def ensure_fed_indices(client: OpenSearch) -> None: @@ -47,7 +50,6 @@ def _strip_federation_meta(source: dict) -> dict: } - _LIST_PAGE_SIZE = 1000 @@ -104,8 +106,6 @@ async def aload_federated_asset( ) - - def _site_owned_query(site_name: str) -> dict[str, Any]: return { "bool": { diff --git a/federation/sds_federation/services/peer_sync.py b/federation/sds_federation/services/peer_sync.py index 940b22517..dde05f7a1 100644 --- a/federation/sds_federation/services/peer_sync.py +++ b/federation/sds_federation/services/peer_sync.py @@ -21,10 +21,14 @@ def peer_for_outbound( peer: PeerInfo, registry: PeerRegistry | None, ) -> PeerInfo: - """Return peer with sync_service_url overlaid from site-hello when present.""" + """Return peer with sync_service_url overlaid from site-hello when present. + + ``site-hello`` registers under RFC ``site_name`` (FQDN). Look up by fqdn first, + then short ``peer.name``, so toml name/fqdn mismatches still resolve. + """ if registry is None: return peer - hello = registry.get(peer.name) + hello = registry.get(peer.fqdn) or registry.get(peer.name) if hello is None: return peer return peer.model_copy(update={"sync_service_url": hello.sync_service_url}) diff --git a/federation/tests/test_integration_mesh.py b/federation/tests/test_integration_mesh.py index cf736fece..d14aff9ce 100644 --- a/federation/tests/test_integration_mesh.py +++ b/federation/tests/test_integration_mesh.py @@ -16,11 +16,11 @@ from sds_federation.services.fed_index import FederatedAssetIndexer from sds_federation.services.fed_index import doc_id from sds_federation.services.local_events import dispatch_federation_redis_payload -from sds_federation.tests.support.federation_mesh import PEER_ONE_FQDN -from sds_federation.tests.support.federation_mesh import TESTSITE_FQDN from sds_federation.testing.sample_data import TEST_DATASET_UUID from sds_federation.testing.sample_data import sample_federated_dataset_doc from sds_federation.testing.sample_data import simulated_dataset_redis_payload +from sds_federation.tests.support.federation_mesh import PEER_ONE_FQDN +from sds_federation.tests.support.federation_mesh import TESTSITE_FQDN if TYPE_CHECKING: from tests.support.federation_mesh import FederationMesh diff --git a/federation/tests/test_integration_webhooks.py b/federation/tests/test_integration_webhooks.py index bfdd21ae5..177fc39ff 100644 --- a/federation/tests/test_integration_webhooks.py +++ b/federation/tests/test_integration_webhooks.py @@ -207,7 +207,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert export_hits == {"datasets": 1, "captures": 1} assert len(recording_opensearch.index_calls) == 1 assert recording_opensearch.index_calls[0]["id"] == doc_id( - "testsite", + "localhost", TEST_DATASET_UUID, ) diff --git a/federation/tests/test_regression_fed_indices.py b/federation/tests/test_regression_fed_indices.py index 12dac6ac8..6ab2e2fb1 100644 --- a/federation/tests/test_regression_fed_indices.py +++ b/federation/tests/test_regression_fed_indices.py @@ -14,10 +14,10 @@ def __init__(self) -> None: self.existing: set[str] = set() self.create_calls: list[dict[str, Any]] = [] - def exists(self, *, index: str, **kwargs: Any) -> bool: # noqa: ARG002 + def exists(self, *, index: str, **kwargs: Any) -> bool: return index in self.existing - def create(self, *, index: str, body: dict[str, Any], **kwargs: Any) -> dict: # noqa: ARG002 + def create(self, *, index: str, body: dict[str, Any], **kwargs: Any) -> dict: self.create_calls.append({"index": index, "body": body}) self.existing.add(index) return {"acknowledged": True} diff --git a/federation/tests/test_regression_fed_list.py b/federation/tests/test_regression_fed_list.py index 8c6df2a4b..30e66a8de 100644 --- a/federation/tests/test_regression_fed_list.py +++ b/federation/tests/test_regression_fed_list.py @@ -12,8 +12,8 @@ from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.services.bootstrap import _index_export_docs from sds_federation.services.bootstrap import _parse_doc_event_at -from sds_federation.services.fed_index import FederatedAssetIndexer from sds_federation.services.fed_index import _LIST_PAGE_SIZE +from sds_federation.services.fed_index import FederatedAssetIndexer from sds_federation.services.fed_index import list_federated_assets_for_site from sds_federation.testing.sample_data import sample_federated_dataset_doc diff --git a/federation/tests/test_regression_peer_list_retry.py b/federation/tests/test_regression_peer_list_retry.py new file mode 100644 index 000000000..0c31f84a3 --- /dev/null +++ b/federation/tests/test_regression_peer_list_retry.py @@ -0,0 +1,91 @@ +"""Regression: peer sync list fetch retries transient connection failures.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock +from unittest.mock import patch + +import httpx +import pytest +from sds_federation.models import PeerInfo +from sds_federation.schemas.webhooks import AssetTypeEnum +from sds_federation.services.bootstrap import _is_retryable_peer_list_error +from sds_federation.services.bootstrap import fetch_peer_sync_list +from sds_federation.testing.sample_data import sample_federated_dataset_doc + + +def _peer() -> PeerInfo: + return PeerInfo( + name="peer", + fqdn="peer.local", + display_name="Peer", + gateway_api_base="http://unused:8000/api/v1", + sync_service_url="http://sds-federation-peer-sync:8000/sync", + ) + + +def test_connect_error_is_retryable() -> None: + assert _is_retryable_peer_list_error(httpx.ConnectError("boom")) is True + assert _is_retryable_peer_list_error(httpx.ReadTimeout("slow")) is True + req = httpx.Request("GET", "http://example/sync/api/v1/webhook/list-datasets/") + resp = httpx.Response(503, request=req) + assert _is_retryable_peer_list_error( + httpx.HTTPStatusError("x", request=req, response=resp) + ) + resp_400 = httpx.Response(400, request=req) + assert ( + _is_retryable_peer_list_error( + httpx.HTTPStatusError("x", request=req, response=resp_400), + ) + is False + ) + + +@pytest.mark.asyncio +async def test_fetch_peer_sync_list_retries_then_succeeds() -> None: + doc = sample_federated_dataset_doc(site_name="peer.local") + calls = {"n": 0} + + async def flaky_get_json(*_args, **_kwargs): + calls["n"] += 1 + if calls["n"] < 3: + raise httpx.ConnectError("All connection attempts failed") + return [doc.model_dump(mode="json")] + + with ( + patch( + "sds_federation.services.bootstrap._get_json", + new=AsyncMock(side_effect=flaky_get_json), + ), + patch("sds_federation.services.bootstrap.asyncio.sleep", new=AsyncMock()), + ): + docs = await fetch_peer_sync_list( + httpx.AsyncClient(), + _peer(), + AssetTypeEnum.DATASET, + attempts=5, + backoff_secs=0.01, + ) + + assert calls["n"] == 3 + assert len(docs) == 1 + assert docs[0].site_name == "peer.local" + + +@pytest.mark.asyncio +async def test_fetch_peer_sync_list_exhausts_retries() -> None: + with ( + patch( + "sds_federation.services.bootstrap._get_json", + new=AsyncMock(side_effect=httpx.ConnectError("down")), + ), + patch("sds_federation.services.bootstrap.asyncio.sleep", new=AsyncMock()), + ): + with pytest.raises(httpx.ConnectError): + await fetch_peer_sync_list( + httpx.AsyncClient(), + _peer(), + AssetTypeEnum.DATASET, + attempts=3, + backoff_secs=0.01, + ) diff --git a/federation/tests/test_regression_peer_sync.py b/federation/tests/test_regression_peer_sync.py index fd26107e1..1e21d261b 100644 --- a/federation/tests/test_regression_peer_sync.py +++ b/federation/tests/test_regression_peer_sync.py @@ -39,7 +39,7 @@ def test_peer_for_outbound_overlays_site_hello_url() -> None: registry = PeerRegistry() registry.register( SiteHelloWebhook( - site_name="peer-one", + site_name="peer.test", # RFC: FQDN fqdn="peer.test", display_name="Peer", sync_service_url="http://live-sync.test/sync", diff --git a/federation/uv.lock b/federation/uv.lock index 4a7f950ff..8e139db15 100644 --- a/federation/uv.lock +++ b/federation/uv.lock @@ -308,6 +308,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "prek" +version = "0.4.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/5c/cb6e63f7e5a58a5313ddb70409174f4dc004e4b0910b8a8d3f59b2225a95/prek-0.4.12.tar.gz", hash = "sha256:04beeba7f40437cd2f36804b84101bd7f3c9fb40b52da46a25604642ab2bfb09", size = 519080, upload-time = "2026-08-03T11:28:33.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/23/5811a3161e072e5f93e4da01af611ee30c32922507b8ab4d9873df6affd3/prek-0.4.12-py3-none-linux_armv6l.whl", hash = "sha256:cd92000b051e433f26340821cf1cc8e6e3960f1275f3d516ca01f05905abba64", size = 5793226, upload-time = "2026-08-03T11:28:09.534Z" }, + { url = "https://files.pythonhosted.org/packages/a3/88/8607845d94eb1482e1bd335dadf098618f077a15775f7e98de99669052b4/prek-0.4.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5904fe6c6ab26e7d8792a3c7f1e3fc8d94fcfb63ad33b247c35f004b62cb6275", size = 6132269, upload-time = "2026-08-03T11:28:11.147Z" }, + { url = "https://files.pythonhosted.org/packages/ac/28/571d79ba457fbd9ecf40ae879c91952e12f5fa475306218c91139b86db7a/prek-0.4.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:df3eff1db9c24dc293010a07bc7a0ae0c541d55af828f5586405dedc28c4920d", size = 5614964, upload-time = "2026-08-03T11:28:12.983Z" }, + { url = "https://files.pythonhosted.org/packages/b0/a9/3f5cb79a73c764a8ac38d5bcd51e0df57239856eca7949b09bdac4338bf3/prek-0.4.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:c7733b44ca772ea32ec6a8bee669d0358bdf45873e79767afed196065084f31c", size = 5941047, upload-time = "2026-08-03T11:28:14.45Z" }, + { url = "https://files.pythonhosted.org/packages/8c/00/1dfed0ef8af10c5c32aa903486dccd33d2df171f3d945a037c5692f10760/prek-0.4.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:87f170cf1ffd6e3a196f947b83dff1f6c2cd68635f8d49740278bebe7b682262", size = 5707994, upload-time = "2026-08-03T11:28:15.914Z" }, + { url = "https://files.pythonhosted.org/packages/c0/bd/5f388f6cbdc0445b850e7c1a160d0be67fcef8bf221e3c8141a1feccef17/prek-0.4.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57dad513831f060cf73808df8edec29d46ec311435aa69f21c80edebf23dc5e1", size = 6133784, upload-time = "2026-08-03T11:28:17.184Z" }, + { url = "https://files.pythonhosted.org/packages/ba/47/342091a987bf68a74acec6d226a40ce7d51faf0019aa4126cc7bc952f8a7/prek-0.4.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b204844abc7ded983471f576ae8dc13b99e9b8d022e4d4b46176c6654769c9d8", size = 6901589, upload-time = "2026-08-03T11:28:18.545Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8a/3ef7bdc3c3441649ebc040b9e164a13163e1e5fabae23e7bbb901992f3de/prek-0.4.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43b0a5a9d3f2f77871fdcb7893bfc5c8fe7e44f4e603ce6e4712bfec96b2d6f2", size = 6342189, upload-time = "2026-08-03T11:28:20Z" }, + { url = "https://files.pythonhosted.org/packages/c4/da/6277908442301b1b92a2879f6b04aaa03accb900f80e42776fc28b8197ef/prek-0.4.12-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:0d188e572c306cc44b96e1bae5647e25b7bd311113f3f3f4a67320c257ee64a3", size = 5951250, upload-time = "2026-08-03T11:28:21.339Z" }, + { url = "https://files.pythonhosted.org/packages/a3/68/bff51a7332837edb1ecbe017325adb7fafd69b9c7828ddc81a1334b884af/prek-0.4.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:986f52d104b7066190f0f32aebe3467710356de265e9bfd892101ba99371db4d", size = 5804147, upload-time = "2026-08-03T11:28:22.656Z" }, + { url = "https://files.pythonhosted.org/packages/aa/de/b7f544971072ed7814125145dfeb1f7c15cce6b78ccea65a96298ff37838/prek-0.4.12-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:13e34d9e09bafcbf1f25a01cf86985e2c5e486591d3f45b2786ba3de82e5153a", size = 5680104, upload-time = "2026-08-03T11:28:24.271Z" }, + { url = "https://files.pythonhosted.org/packages/68/94/95942bcc20a6a91ec2989aa30fdeb00ad095be736ec48b4bbcf0376166b1/prek-0.4.12-py3-none-musllinux_1_1_i686.whl", hash = "sha256:3d0208370da73e8b5bc97f2492dc3975f8dd2c22f4bf6e1f2cf3342503764b52", size = 5975030, upload-time = "2026-08-03T11:28:25.683Z" }, + { url = "https://files.pythonhosted.org/packages/ef/6d/26e6497198d81cf9aa82495400aef46adea8df3e4a4efc5f00e3b6ab3292/prek-0.4.12-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b1005f42920111bec1403c25e8f2f12ec7af0be06686cc3b8dcf85429af908a8", size = 6458532, upload-time = "2026-08-03T11:28:27.121Z" }, + { url = "https://files.pythonhosted.org/packages/44/02/ee140c2eb4701bd194db429d84630733492be94897d5f72b61d6f11e6619/prek-0.4.12-py3-none-win32.whl", hash = "sha256:afee229488dcceaea282288e4d7096a93da5a8b85649d9ef506dbdbcd78f38a7", size = 5502213, upload-time = "2026-08-03T11:28:28.691Z" }, + { url = "https://files.pythonhosted.org/packages/e5/7b/744cff84def48c1ce38c0b4f643a3553c66976c5bb7869ab7317044870e4/prek-0.4.12-py3-none-win_amd64.whl", hash = "sha256:fdd27bad8adafea8fe77606950ca09200d59296a47ab131cfb88718d460949d7", size = 5868065, upload-time = "2026-08-03T11:28:30.377Z" }, + { url = "https://files.pythonhosted.org/packages/46/1d/e2c0fc222904ef73df1739b11a83edc29e38bc4bc61259f2ca6d2f15abb0/prek-0.4.12-py3-none-win_arm64.whl", hash = "sha256:45e34a24fba4a4e4568682477158591698efc2375b8d1d418ae424691c4bd01b", size = 5632819, upload-time = "2026-08-03T11:28:31.743Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -561,6 +585,7 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "deptry" }, + { name = "prek" }, { name = "pyrefly" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -574,6 +599,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.28.1" }, { name = "loguru", specifier = ">=0.7.2" }, { name = "opensearch-py", specifier = ">=2.8.0" }, + { name = "prek", marker = "extra == 'dev'", specifier = ">=0.3.4" }, { name = "pydantic", specifier = ">=2.11.0" }, { name = "pyrefly", marker = "extra == 'dev'", specifier = ">=0.42.1" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index b1cbc5fc6..0b6c3be69 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -753,7 +753,7 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: SDS_SITE_FQDN: str = env.str("SDS_SITE_FQDN", default="localhost") # Federation peer short name (RFC [site].name, e.g. crc, haystack); -# not SDS_PROGRAMMATIC_SITE_NAME. +# used for Redis channel ``federation:events:{name}``, not OpenSearch site_name. # Export/OpenSearch ``site_name`` uses SDS_SITE_FQDN (matches federation.toml [site].fqdn). FEDERATION_SITE_NAME: str = env.str("FEDERATION_SITE_NAME", default="").strip() # Master switch: when False, federation export and Redis events are inactive. diff --git a/gateway/sds_gateway/api_methods/federation/availability.py b/gateway/sds_gateway/api_methods/federation/availability.py index dcca8aada..4971b9653 100644 --- a/gateway/sds_gateway/api_methods/federation/availability.py +++ b/gateway/sds_gateway/api_methods/federation/availability.py @@ -142,6 +142,10 @@ def evaluate_federation_operational() -> tuple[bool, str]: if not site_name: return False, "FEDERATION_SITE_NAME must be set when federation is enabled" + site_fqdn = (_setting("SDS_SITE_FQDN", default="") or "").strip() + if not site_fqdn: + return False, "SDS_SITE_FQDN must be set when federation is enabled" + for check in (_sync_api_key_present, _redis_ok, _sync_health_ok): ok, reason = check() if not ok: diff --git a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py index c49d46b59..046c71b81 100644 --- a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py +++ b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py @@ -31,7 +31,11 @@ def federation_site_name() -> str: - return getattr(settings, "FEDERATION_SITE_NAME", "").strip() + """FQDN written to export/OpenSearch ``site_name`` (RFC §6 / toml ``[site].fqdn``). + + Uses ``SDS_SITE_FQDN`. + """ + return str(getattr(settings, "SDS_SITE_FQDN", "") or "").strip() def capture_in_published_dataset(capture: Capture) -> bool: diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_events.py b/gateway/sds_gateway/api_methods/tests/test_federation_events.py index 363b1f199..aed52aeba 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_events.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_events.py @@ -99,6 +99,7 @@ class TestFederationSignals: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, FEDERATION_EVENTS_CHANNEL="federation:events:crc", ) @@ -127,7 +128,7 @@ def test_dataset_post_save_indexes_when_published( mock_indexer.apply_local_event.assert_called_once() call = mock_indexer.apply_local_event.call_args.kwargs - assert call["site_name"] == "crc" + assert call["site_name"] == "sds.localhost" assert call["item_type"] == ItemType.DATASET assert call["uuid"] == dataset.uuid mock_publish.assert_called_once_with( diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py index f512459ab..f535018ba 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -52,6 +52,7 @@ def test_disabled_when_master_switch_off(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, @@ -63,6 +64,7 @@ def test_operational_when_probes_skipped(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, ) @@ -74,7 +76,9 @@ def test_fails_without_sync_api_key(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=False, FEDERATION_SYNC_HEALTH_URL="http://sync.test/health", FEDERATION_SKIP_REDIS_PROBE=True, ) @@ -93,7 +97,9 @@ def test_health_probe_success(self, mock_urlopen: MagicMock) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=False, FEDERATION_SYNC_HEALTH_URL="http://sync.test/health", FEDERATION_SKIP_REDIS_PROBE=True, ) @@ -134,6 +140,7 @@ def test_empty_cidr_override_falls_back_to_private_defaults(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, @@ -146,6 +153,20 @@ def test_fails_without_site_name(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="", + FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, + FEDERATION_SKIP_REDIS_PROBE=True, + ) + def test_fails_without_site_fqdn(self) -> None: + ok, reason = evaluate_federation_operational() + assert ok is False + assert "SDS_SITE_FQDN" in reason + + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py index dc1935fe1..5544b352e 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py @@ -37,6 +37,7 @@ def _federation_on_commit(): @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestFederationDatasetSignals(TestCase): @@ -149,6 +150,7 @@ def test_deleted_dataset_skips_when_fed_doc_already_deleted( @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestFederationCaptureSignals(TestCase): @@ -230,6 +232,7 @@ def test_deleted_capture_reindexes_with_is_deleted( @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestDatasetDisconnectReindex(TestCase): @@ -293,6 +296,7 @@ def test_dataset_soft_delete_reindexes_orphan_capture( @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestFederationReindexOnCommit(TestCase): From 15fbc02b3661f9b1c7a267acef11bd18ab4f0025 Mon Sep 17 00:00:00 2001 From: klpoland Date: Tue, 4 Aug 2026 13:53:54 -0400 Subject: [PATCH 09/22] p2p remote test config, update docs --- federation/.gitignore | 3 +- federation/README.md | 211 ++++++++++++++++++ federation/compose.peer.remote.yaml | 74 ++++++ federation/docs/peer-remote-test.md | 116 ++++++++++ federation/env.peer.example | 11 + .../federation.peer.remote.example.toml | 16 ++ federation/justfile | 14 +- federation/pyproject.toml | 2 +- .../scripts/deploy-local-peer-2-peer.sh | 2 +- federation/scripts/local_e2e/README.md | 103 +++++++++ federation/scripts/local_e2e/env.example | 16 ++ federation/scripts/seed_peer_opensearch.py | 147 ++++++++++++ 12 files changed, 710 insertions(+), 5 deletions(-) create mode 100644 federation/README.md create mode 100644 federation/compose.peer.remote.yaml create mode 100644 federation/docs/peer-remote-test.md create mode 100644 federation/env.peer.example create mode 100644 federation/federation.peer.remote.example.toml create mode 100644 federation/scripts/local_e2e/README.md create mode 100644 federation/scripts/local_e2e/env.example create mode 100644 federation/scripts/seed_peer_opensearch.py diff --git a/federation/.gitignore b/federation/.gitignore index f633817ff..712f81586 100644 --- a/federation/.gitignore +++ b/federation/.gitignore @@ -5,4 +5,5 @@ __pycache__/ federation.toml federation.peer.toml -scripts/local_e2e/ +# Local live-test secrets only (keep README + env.example tracked) +scripts/local_e2e/.env diff --git a/federation/README.md b/federation/README.md new file mode 100644 index 000000000..339224be4 --- /dev/null +++ b/federation/README.md @@ -0,0 +1,211 @@ +# SDS Federation Sync + +FastAPI service that indexes federated dataset/capture metadata into OpenSearch +(`fed-datasets`, `fed-captures`), receives peer webhooks, and bootstraps from the +local gateway export API plus peer sync list endpoints. + +On startup (unless `FEDERATION_BOOTSTRAP_ON_START=false`): + +1. Ensure `fed-*` indices exist +2. Mint or reuse a FederationSync export Api-Key +3. Pull local gateway `/api/v1/federation/export/{datasets,captures}/` +4. Pull each peer’s `/sync/api/v1/webhook/list-{datasets,captures}/` +5. Register with peers via `POST .../webhook/site-hello` +6. Subscribe to Redis `federation:events:{site}` for incremental updates + +Local HTTP: sync on **host port 8001** → container `:8000`. +Health: `http://localhost:8001/sync/health`. + +See also: + +- [scripts/local_e2e/README.md](scripts/local_e2e/README.md) — local live test (publish / seed / verify) +- [docs/peer-remote-test.md](docs/peer-remote-test.md) — two-server sync-only peer +- Repo-root `rfc_federation.md` — architecture RFC + +--- + +## Site identity (critical) + +| Setting | Matches | Used for | +|---------|---------|----------| +| `FEDERATION_SITE_NAME` | toml `[site].name` | Short peer id; Redis `federation:events:{name}` | +| `SDS_SITE_FQDN` | toml `[site].fqdn` | Export / OpenSearch **`site_name`**; sync `list-*` filters | + +Do **not** put the short name into OpenSearch docs. Gateway writes FQDN via `SDS_SITE_FQDN`. + +Local lab defaults: `name=crc`, `fqdn=sds.localhost`. Peer lab: `name=peer`, `fqdn=peer.local`. + +--- + +## Prerequisites + +- Docker / Compose, `just`, OpenSSL (for secrets) +- Gateway local stack (`sds-network-local`, app, Redis, OpenSearch) +- Python tooling for federation tests (`uv` in this directory) + +--- + +## Local initialization + +### 1. Generate shared secrets + +From the gateway directory (fills `gateway/.envs/local/*` and repo-root +`federation-shared.env`): + +```bash +cd gateway +./scripts/generate-secrets.sh local +``` + +Confirm the DRF token (40 characters). Leave `FEDERATION_SYNC_SERVER_API_KEY` empty +so sync mints an export key on start. + +### 2. Gateway federation env + +`gateway/.envs/local/django.env`: + +```env +FEDERATION_ENABLED=true +FEDERATION_SITE_NAME=crc +SDS_SITE_FQDN=sds.localhost +FEDERATION_SYNC_HEALTH_URL=http://sds-federation-local-sync:8000/sync/health +FEDERATION_SYNC_USER_EMAIL=federation-sync@internal.local +# Leave FEDERATION_EVENTS_CHANNEL unset → federation:events:crc +``` + +Optional first boot: `FEDERATION_SKIP_SYNC_HEALTH_PROBE=true` until sync is healthy. + +### 3. Federation config + +```bash +cd federation +cp federation.example.toml federation.toml +# [site] name/fqdn must match gateway FEDERATION_SITE_NAME / SDS_SITE_FQDN +``` + +Compose loads OpenSearch from `gateway/.envs//opensearch.env` and auth from +`federation-shared.env`. + +### 4. Start gateway + federation token + +```bash +cd gateway +just up +docker compose -f compose.local.yaml exec sds-gateway-local-app \ + python manage.py init_federation_sync_token +# or: prepare_gateway +``` + +### 5. Start sync + +**Solo (this site only):** + +```bash +cd federation +just build && just up +curl -s http://localhost:8001/sync/health | jq . +``` + +**Local peer mesh (recommended for p2p):** + +```bash +cd federation +just deploy-local-peer-2-peer # gateway check + build + local+peer up +just seed-peer # dummy peer.local docs on :9201 +docker restart sds-federation-local-sync # pull peer list / site-hello +``` + +Tear down both stacks: + +```bash +docker compose -f compose.local.yaml -f compose.peer.local.yaml down +# just down # only stops the primary compose file +``` + +### 6. Export Api-Key (manual checks) + +```bash +TOKEN=$(grep '^FEDERATION_SYNC_DRF_TOKEN=' ../federation-shared.env | cut -d= -f2-) +curl -sS http://localhost:8000/users/get-federation-sync-api-key/ \ + -H "Authorization: Token $TOKEN" | jq -r .api_key +``` + +Use header: `Authorization: Api-Key: ` (note the colon). + +--- + +## Auth model + +| Secret | Where | Role | +|--------|--------|------| +| `FEDERATION_SYNC_DRF_TOKEN` | `federation-shared.env` → DB via `init_federation_sync_token` | Mint endpoint auth | +| `FEDERATION_SYNC_SERVER_API_KEY` | Minted by sync (or set in shared env) | Gateway `/federation/export/*` | + +There is **no** `create_federation_sync_api_key` command. + +--- + +## Commands + +```bash +cd federation +just # list recipes +just deploy-local-peer-2-peer # local + peer stacks +just seed-peer # seed peer OpenSearch (:9201, site peer.local) +just verify-federation-live --q fixture +just simulate-redis --event-type updated --item-type dataset --uuid +just test / just test-regression / just test-integration +``` + +--- + +## Publishing local data + +```bash +cd gateway +docker compose -f compose.local.yaml exec sds-gateway-local-app \ + python manage.py publish_for_federation \ + --dataset-uuid --capture-uuids +``` + +Confirm docs use the FQDN: + +```bash +curl -s 'http://localhost:9200/fed-datasets/_search' | jq '.hits.hits[]._source.site_name' +curl -s http://localhost:8001/sync/api/v1/webhook/list-datasets/ | jq 'length' +``` + +Live-test steps: [scripts/local_e2e/README.md](scripts/local_e2e/README.md). + +--- + +## Peer sync testing + +### Same machine + +`compose.peer.local.yaml` + `federation.peer.toml` (mutual `[[peers]]` with **container DNS** +URLs). Orchestrated by `just deploy-local-peer-2-peer`. + +- Peer has **no gateway** → `GATEWAY_INTERNAL_BASE_URL=http://127.0.0.1:9/...` failures in + peer logs are expected. +- Peer-owned docs: `just seed-peer` (writes `site_name=peer.local`). +- Then restart local sync so site-hello / list pull ingest peer docs. + +### Two servers (sync-only peer) + +See **[docs/peer-remote-test.md](docs/peer-remote-test.md)** and +`compose.peer.remote.yaml` / `federation.peer.remote.example.toml`. + +--- + +## Troubleshooting + +| Symptom | Check | +|---------|--------| +| Export `503` | Sync health URL, Api-Key, Redis, `FEDERATION_ENABLED` | +| Bootstrap mint fail | DRF token length 40; `init_federation_sync_token`; gateway URL | +| `list-*` empty but OS has docs | Doc `site_name` must be **FQDN** (`SDS_SITE_FQDN`), not short name | +| site-hello connect race | Retries are built in; ensure peer healthy before relying on backfill | +| Redis events missing | Leave `FEDERATION_EVENTS_CHANNEL` unset; channel = `federation:events:{FEDERATION_SITE_NAME}` | +| Gateway code changes not live | App image is not fully bind-mounted — rebuild gateway image | +| Docker build / `sds-opensearch-query` | Build context repo root; `COPY common/` | diff --git a/federation/compose.peer.remote.yaml b/federation/compose.peer.remote.yaml new file mode 100644 index 000000000..59a9b538f --- /dev/null +++ b/federation/compose.peer.remote.yaml @@ -0,0 +1,74 @@ +# Sync-only peer on a remote host (no gateway / Postgres). +# Pair with federation.peer.remote.toml and docs/peer-remote-test.md. +# +# On the peer machine: +# cp federation.peer.remote.example.toml federation.peer.toml +# cp env.peer.example .env.peer # optional overrides +# docker compose -f compose.peer.remote.yaml --env-file .env.peer up -d --build +# +# Main site must list this peer's public sync URL in federation.toml [[peers]]. + +networks: + peer-internal: + driver: bridge + +services: + sds-federation-peer-opensearch: + image: opensearchproject/opensearch:2 + container_name: sds-federation-peer-opensearch + environment: + discovery.type: single-node + plugins.security.disabled: "true" + OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m + DISABLE_INSTALL_DEMO_CONFIG: "true" + ports: + - "${PEER_OPENSEARCH_PORT:-9200}:9200" + networks: [peer-internal] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9200/_cluster/health || exit 1"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 40s + + sds-federation-peer-redis: + image: redis:6 + container_name: sds-federation-peer-redis + ports: + - "${PEER_REDIS_PORT:-6379}:6379" + networks: [peer-internal] + + sds-federation-peer-sync: + build: + context: .. + dockerfile: federation/Dockerfile + image: sds-federation-peer-sync + container_name: sds-federation-peer-sync + environment: + FEDERATION_CONFIG_PATH: /etc/sds/federation.toml + # Must be reachable by the main site (set in toml sync_service_url too). + FEDERATION_SYNC_SERVICE_URL: ${PEER_SYNC_PUBLIC_URL:-http://0.0.0.0:8000/sync} + # Intentional: sync-only peer has no gateway export. + GATEWAY_INTERNAL_BASE_URL: http://127.0.0.1:9/api/v1 + REDIS_URL: redis://sds-federation-peer-redis:6379/0 + OPENSEARCH_HOST: sds-federation-peer-opensearch + OPENSEARCH_PORT: "9200" + OPENSEARCH_USER: "" + OPENSEARCH_PASSWORD: "" + OPENSEARCH_USE_SSL: "false" + OPENSEARCH_VERIFY_CERTS: "false" + FEDERATION_SITE_NAME: ${PEER_SITE_NAME:-peer} + FEDERATION_BOOTSTRAP_ON_START: ${FEDERATION_BOOTSTRAP_ON_START:-true} + volumes: + - ./federation.peer.toml:/etc/sds/federation.toml:ro + - ./certs:/etc/sds/certs:ro + ports: + - "${PEER_SYNC_PORT:-8000}:8000" + depends_on: + sds-federation-peer-opensearch: + condition: service_healthy + sds-federation-peer-redis: + condition: service_started + networks: + - peer-internal + # Attach an extra external network / reverse-proxy in front for TLS in real deploys. diff --git a/federation/docs/peer-remote-test.md b/federation/docs/peer-remote-test.md new file mode 100644 index 000000000..e98357b45 --- /dev/null +++ b/federation/docs/peer-remote-test.md @@ -0,0 +1,116 @@ +# Remote peer sync test (two servers) + +Test federation **without** a second full gateway: the peer host runs OpenSearch, +Redis, and the sync service only, then seeds dummy `fed-*` docs. The main host +runs the normal gateway + sync stack. + +```text +[Main] [Peer] + gateway + Redis + OpenSearch + sync OpenSearch + Redis + sync + publishes real/public FINAL datasets seed_peer_opensearch.py + federation.toml [[peers]] → peer URL federation.peer.toml [[peers]] → main +``` + +## Identity + +| Field | Main | Peer | +|-------|------|------| +| toml `[site].name` | e.g. `crc` | e.g. `peer` | +| toml `[site].fqdn` / OpenSearch `site_name` | e.g. `sds.example.com` | e.g. `peer.example.com` | +| Redis channel | `federation:events:{name}` | `federation:events:peer` | + +`list-*` and search filter by **FQDN**, not the short name. + +## Peer host setup + +1. Checkout this repo (or deploy the federation image + compose files). +2. Config: + + ```bash + cd federation + cp federation.peer.remote.example.toml federation.peer.toml + # Edit FQDNs + main sync_service_url (reachable from peer) + cp env.peer.example .env.peer + # Edit PEER_SYNC_PUBLIC_URL to this host's public sync URL + ``` + +3. Start: + + ```bash + docker compose -f compose.peer.remote.yaml --env-file .env.peer up -d --build + curl -sS http://localhost:8000/sync/health | jq . + ``` + + Local gateway bootstrap errors (`127.0.0.1:9`) are **expected** on a sync-only peer. +4. Seed peer-owned docs (FQDN must match toml `[site].fqdn`): + + ```bash + uv sync --extra dev + uv run python scripts/seed_peer_opensearch.py \ + --opensearch-url http://localhost:9200 \ + --site-name peer.example.com + ``` + +5. Open firewall / TLS so the **main** site can reach `https://peer.example.com/sync` + (and peer can reach main's sync URL). + +## Main host setup + +1. Normal gateway + federation sync (`compose.local.yaml` or production compose). +2. In `federation.toml` add the peer with a **public** URL (not Docker DNS): + + ```toml + [[peers]] + name = "peer" + fqdn = "peer.example.com" + display_name = "Remote Peer" + gateway_api_base = "http://peer-gateway-unused:8000/api/v1" + sync_service_url = "https://peer.example.com/sync" + ``` + +3. Ensure `SDS_SITE_FQDN` matches main `[site].fqdn` (export/OpenSearch `site_name`). +4. Restart main sync after toml changes so bootstrap + site-hello run. + +## Bring-up order + +1. Main gateway healthy + federation operational. +2. Peer stack healthy + seeded. +3. Restart **main** sync (pulls peer `list-*` / site-hello backfill). +4. Optionally restart **peer** sync (pulls main `list-*`). + +Site-hello backfill retries transient connect errors; still prefer peer healthy before main restart. + +## Verify + +### Peer → main + +```bash +# peer +curl -sS https://peer.example.com/sync/api/v1/webhook/list-datasets/ | jq '.[].site_name' +# → ["peer.example.com"] + +# main OpenSearch +curl -sS 'http://localhost:9200/fed-datasets/_search?q=site_name:peer.example.com' \ + | jq '.hits.total' +curl -sS 'http://localhost:8001/api/v1/search/datasets?site=peer.example.com' | jq . +``` + +### Main → peer + +```bash +# main: publish a FINAL public dataset, then +curl -sS 'https://peer.example.com/sync/…' # or query peer OS +curl -sS 'http://PEER_OS:9200/fed-datasets/_search?q=site_name:sds.example.com' \ + | jq '.hits.total' +``` + +## Safety + +- Use a **dedicated** peer OpenSearch; never point peer sync at production `fed-*` by mistake. +- Seed data is synthetic; do not load real user PII onto the peer host. +- Prefer TLS + CA paths in toml (`ca_cert_path`) for non-lab networks. + +## Local analogue + +Same idea on one machine: `just deploy-local-peer-2-peer` then `just seed-peer` +(see [../README.md](../README.md) and [../scripts/local_e2e/README.md](../scripts/local_e2e/README.md)). diff --git a/federation/env.peer.example b/federation/env.peer.example new file mode 100644 index 000000000..ae1eabc17 --- /dev/null +++ b/federation/env.peer.example @@ -0,0 +1,11 @@ +# Optional env for compose.peer.remote.yaml (copy to .env.peer; do not commit secrets). + +PEER_SITE_NAME=peer +PEER_SYNC_PORT=8000 +PEER_OPENSEARCH_PORT=9200 +PEER_REDIS_PORT=6379 + +# Public URL other sites use in their federation.toml [[peers]].sync_service_url +PEER_SYNC_PUBLIC_URL=https://peer.example.com/sync + +FEDERATION_BOOTSTRAP_ON_START=true diff --git a/federation/federation.peer.remote.example.toml b/federation/federation.peer.remote.example.toml new file mode 100644 index 000000000..a547d5b2c --- /dev/null +++ b/federation/federation.peer.remote.example.toml @@ -0,0 +1,16 @@ +# Copy to federation.peer.toml on the peer host (gitignored if using that name). +# Replace hostnames with values reachable from the main gateway/sync server. + +[site] +name = "peer" +fqdn = "peer.example.com" +display_name = "Remote Peer (sync-only)" + +[[peers]] +name = "crc" +fqdn = "sds.example.com" +display_name = "Main SDS" +gateway_api_base = "https://sds.example.com/api/v1" +# Main federation sync base (must end with /sync) +sync_service_url = "https://sds.example.com/sync" +# ca_cert_path = "/etc/sds/certs/main-ca.pem" diff --git a/federation/justfile b/federation/justfile index 8263c7f8e..404b6509f 100644 --- a/federation/justfile +++ b/federation/justfile @@ -83,13 +83,23 @@ simulate-redis +args='': [group('dev')] verify-federation-live +args='': - uv run python scripts/local_e2e/verify_federation_live.py {{ args }} + uv run python scripts/verify_federation_live.py {{ args }} -#deploy local and peer federation stacks +# Deploy local + peer federation stacks (checks gateway, builds, waits for health) [group('dev')] deploy-local-peer-2-peer: ./scripts/deploy-local-peer-2-peer.sh +# Seed dummy peer.local docs into peer OpenSearch (:9201) +[group('dev')] +seed-peer +args='': + uv run python scripts/seed_peer_opensearch.py --opensearch-url http://localhost:9201 --site-name peer.local {{ args }} + +# Tear down local + peer compose stacks together +[group('dev')] +down-peer-2-peer +args='': + docker compose -f compose.local.yaml -f compose.peer.local.yaml down {{ args }} + [group('docker')] build +args='': {{ docker_compose }} build {{ args }} diff --git a/federation/pyproject.toml b/federation/pyproject.toml index 85c4eff79..0a52056b3 100644 --- a/federation/pyproject.toml +++ b/federation/pyproject.toml @@ -33,7 +33,7 @@ dev = [ ] [tool.deptry] - extend_exclude = ["docs/", "scripts/local_e2e/"] + extend_exclude = ["docs/"] [tool.deptry.per_rule_ignores] DEP002 = [ diff --git a/federation/scripts/deploy-local-peer-2-peer.sh b/federation/scripts/deploy-local-peer-2-peer.sh index c5397b5d3..863ad4f25 100755 --- a/federation/scripts/deploy-local-peer-2-peer.sh +++ b/federation/scripts/deploy-local-peer-2-peer.sh @@ -123,4 +123,4 @@ curl -sS "${LOCAL_SYNC_URL}" echo curl -sS "${PEER_SYNC_URL}" echo -info "Done. Seed peer OpenSearch if testing peer-owned docs; publish local assets for crc→peer." +info "Done. Next: just seed-peer (peer-owned docs), then docker restart sds-federation-local-sync; publish local assets for crc→peer." diff --git a/federation/scripts/local_e2e/README.md b/federation/scripts/local_e2e/README.md new file mode 100644 index 000000000..f064dc8f4 --- /dev/null +++ b/federation/scripts/local_e2e/README.md @@ -0,0 +1,103 @@ +# Local federation live test + +Exercise gateway → sync → OpenSearch without depending on any remote production site. + +**Never commit API tokens.** Copy `env.example` → `.env` in this directory (gitignored). + +## Identity (do not mix these up) + +| Setting | Role | +|---------|------| +| `FEDERATION_SITE_NAME` / toml `[site].name` | Short peer id; Redis channel `federation:events:{name}` | +| `SDS_SITE_FQDN` / toml `[site].fqdn` | OpenSearch + export `site_name`; sync `list-*` filters | + +Local compose defaults (see `federation.toml`): name `crc`, fqdn `sds.localhost`. + +## Prerequisites + +1. Gateway stack up (`gateway` compose local): app, Redis, OpenSearch, object store. +2. Federation sync up — either: + - Solo: `cd federation && just up` (port **8001**) + - Peer mesh: `just deploy-local-peer-2-peer` (local **8001**, peer **8002**) +3. Gateway `django.env`: + - `FEDERATION_ENABLED=true` + - `FEDERATION_SITE_NAME` / `SDS_SITE_FQDN` match `federation.toml` `[site]` +4. Mint export Api-Key (no `create_federation_sync_api_key` command): + + ```bash + TOKEN=$(grep '^FEDERATION_SYNC_DRF_TOKEN=' ../../../federation-shared.env | cut -d= -f2-) + curl -sS http://localhost:8000/users/get-federation-sync-api-key/ \ + -H "Authorization: Token $TOKEN" | jq -r .api_key + # save as FEDERATION_GATEWAY_API_KEY in .env + ``` + + Header form: `Authorization: Api-Key: ` (colon after `Api-Key`). +5. Gateway capture indices: `python manage.py init_indices` (inside app container). + +## Recommended: peer seed (no RF files) + +For peer→local backfill / `list-*` without a second gateway: + +```bash +cd federation +just seed-peer +docker restart sds-federation-local-sync +curl -s 'http://localhost:9200/fed-datasets/_search' \ + | jq '.hits.hits[]._source | {site: .site_name, name}' +``` + +## Local-owned data: publish any capture + +Create a public FINAL dataset in the UI or API (attach any local capture), then: + +```bash +cd gateway +docker compose -f compose.local.yaml exec sds-gateway-local-app \ + python manage.py publish_for_federation \ + --dataset-uuid \ + --capture-uuids +``` + +Optional Redis fan-out: + +```bash +cd federation +just simulate-redis --event-type updated --item-type dataset --uuid +``` + +Confirm OpenSearch `site_name` is the FQDN: + +```bash +curl -s 'http://localhost:9200/fed-datasets/_search' \ + | jq '.hits.hits[]._source.site_name' +# → matches SDS_SITE_FQDN / [site].fqdn +``` + +## Verify + +```bash +cd federation +set -a && source scripts/local_e2e/.env && set +a +just verify-federation-live --q +``` + +Or: + +```bash +curl -s "http://localhost:8001/api/v1/search/datasets?site=${LOCAL_SITE_FQDN}&q=" | jq . +curl -s http://localhost:8000/api/v1/federation/export/datasets/ \ + -H "Authorization: Api-Key: $FEDERATION_GATEWAY_API_KEY" | jq 'length' +curl -s http://localhost:8001/sync/api/v1/webhook/list-datasets/ | jq 'length' +``` + +## Optional: import a capture via SDK + +Federation does **not** ship remote download/upload helpers. Use the SpectrumX SDK from +`sdk/` against whatever host you choose (`SDS_HOST`, `SDS_SECRET_TOKEN`), then upload +to local gateway (`LOCAL_SDS_HOST` / `LOCAL_SDS_SECRET_TOKEN`). See `sdk/docs` for +`download_dataset` / `upload_capture`. Ensure channel layout and `drf_properties.h5` +match what the local gateway expects before `publish_for_federation`. + +## Remote peer (two servers) + +See [../../docs/peer-remote-test.md](../../docs/peer-remote-test.md). diff --git a/federation/scripts/local_e2e/env.example b/federation/scripts/local_e2e/env.example new file mode 100644 index 000000000..3ea78a539 --- /dev/null +++ b/federation/scripts/local_e2e/env.example @@ -0,0 +1,16 @@ +# Copy to federation/scripts/local_e2e/.env (never commit secrets). + +# Optional: remote SDS host for SDK imports (no default — set explicitly) +# SDS_HOST= +# SDS_SECRET_TOKEN= +# DATASET_UUID= +# TOP_LEVEL_DIR= + +# Local gateway (publish / upload target) +LOCAL_SDS_HOST=localhost:8000 +LOCAL_SDS_SECRET_TOKEN= + +# Federation sync live checks +FEDERATION_GATEWAY_API_KEY= +FEDERATION_SYNC_URL=http://localhost:8001 +LOCAL_SITE_FQDN=sds.localhost diff --git a/federation/scripts/seed_peer_opensearch.py b/federation/scripts/seed_peer_opensearch.py new file mode 100644 index 000000000..c6ea546ba --- /dev/null +++ b/federation/scripts/seed_peer_opensearch.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Seed dummy federated docs into a peer OpenSearch (no gateway). + +Use for peer→main backfill / list-* tests. ``site_name`` must be the peer FQDN +(matches federation.toml ``[site].fqdn``). + +Examples:: + + # Local peer stack (OpenSearch on host :9201) + uv run python scripts/seed_peer_opensearch.py \\ + --opensearch-url http://localhost:9201 \\ + --site-name peer.local + + # Remote peer after port-forward / public OS URL + uv run python scripts/seed_peer_opensearch.py \\ + --opensearch-url https://peer-os.example:9200 \\ + --site-name peer.example.com \\ + --user admin --password secret +""" + +from __future__ import annotations + +import argparse +import sys +from datetime import UTC +from datetime import datetime +from uuid import UUID + +from opensearchpy import OpenSearch +from sds_federation.schemas.webhooks import AssetTypeEnum +from sds_federation.services.fed_index import FederatedAssetIndexer +from sds_federation.services.fed_index import doc_id +from sds_federation.services.fed_index import ensure_fed_indices +from sds_federation.testing.sample_data import TEST_CAPTURE_UUID +from sds_federation.testing.sample_data import TEST_DATASET_UUID +from sds_federation.testing.sample_data import sample_federated_capture_doc +from sds_federation.testing.sample_data import sample_federated_dataset_doc + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--opensearch-url", + default="http://localhost:9201", + help="Peer OpenSearch base URL (local peer compose publishes :9201)", + ) + p.add_argument( + "--site-name", + default="peer.local", + help="Peer FQDN written into docs (must match toml [site].fqdn)", + ) + p.add_argument("--dataset-uuid", default=str(TEST_DATASET_UUID)) + p.add_argument("--capture-uuid", default=str(TEST_CAPTURE_UUID)) + p.add_argument("--user", default="", help="Optional basic-auth user") + p.add_argument("--password", default="", help="Optional basic-auth password") + p.add_argument( + "--use-ssl", + action="store_true", + help="Force SSL client (also inferred from https:// URL)", + ) + return p.parse_args() + + +def _client_from_url(args: argparse.Namespace) -> OpenSearch: + url = args.opensearch_url.rstrip("/") + use_ssl = args.use_ssl or url.startswith("https://") + # OpenSearch client wants host/port; parse simply. + without_scheme = url.split("://", 1)[-1] + host_port, _, _path = without_scheme.partition("/") + if ":" in host_port: + host, port_s = host_port.rsplit(":", 1) + port = int(port_s) + else: + host = host_port + port = 443 if use_ssl else 9200 + + kwargs: dict = { + "hosts": [{"host": host, "port": port}], + "use_ssl": use_ssl, + "verify_certs": False, + "ssl_show_warn": False, + } + if args.user: + kwargs["http_auth"] = (args.user, args.password) + return OpenSearch(**kwargs) + + +def main() -> int: + args = _parse_args() + site = args.site_name.strip() + if not site: + print("ERROR: --site-name is required", file=sys.stderr) + return 1 + + dataset_uuid = UUID(args.dataset_uuid) + capture_uuid = UUID(args.capture_uuid) + client = _client_from_url(args) + ensure_fed_indices(client) + indexer = FederatedAssetIndexer(client) + event_at = datetime.now(UTC) + + dataset = sample_federated_dataset_doc(uuid=dataset_uuid, site_name=site) + dataset = dataset.model_copy( + update={ + "name": f"Peer seed dataset ({site})", + "is_public": True, + "status": "final", + "status_display": "Final", + "created_at": event_at.isoformat(), + "updated_at": event_at.isoformat(), + }, + ) + capture = sample_federated_capture_doc(uuid=capture_uuid, site_name=site) + capture = capture.model_copy( + update={ + "name": f"Peer seed capture ({site})", + "channel": "chA", + "public_dataset_ids": [str(dataset_uuid)], + "created_at": event_at.isoformat(), + "updated_at": event_at.isoformat(), + }, + ) + + indexer.apply_asset_event( + event_at=event_at, + site_name=site, + asset=dataset, + asset_type=AssetTypeEnum.DATASET, + ) + indexer.apply_asset_event( + event_at=event_at, + site_name=site, + asset=capture, + asset_type=AssetTypeEnum.CAPTURE, + ) + + print(f"Seeded {AssetTypeEnum.DATASET.index_name} id={doc_id(site, dataset_uuid)}") + print(f"Seeded {AssetTypeEnum.CAPTURE.index_name} id={doc_id(site, capture_uuid)}") + print(f"site_name={site!r} opensearch={args.opensearch_url}") + print( + "Restart the *other* site's sync (or wait for site-hello) to pull these docs." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From b3eff63e1fb14f8ea371dd61bd14815c9afbccc4 Mon Sep 17 00:00:00 2001 From: klpoland Date: Tue, 4 Aug 2026 14:46:23 -0400 Subject: [PATCH 10/22] gateway linting fixes --- common/sds_opensearch_query/mapping.py | 1 - gateway/config/settings/base.py | 3 +- .../api_methods/federation/search_helpers.py | 8 ++- .../api_methods/helpers/list_helpers.py | 23 ++++----- .../commands/publish_for_federation.py | 4 +- .../serializers/dataset_serializers.py | 4 +- gateway/sds_gateway/templates/pages/home.html | 6 ++- .../users/partials/dataset_search_form.html | 11 ++++- .../search_published_datasets_tab.html | 10 ++-- gateway/sds_gateway/users/forms.py | 12 +++++ gateway/sds_gateway/users/views/datasets.py | 49 ++++++++++++++++--- .../sds_gateway/users/views/special_pages.py | 32 +++--------- 12 files changed, 103 insertions(+), 60 deletions(-) diff --git a/common/sds_opensearch_query/mapping.py b/common/sds_opensearch_query/mapping.py index 00f9b0a00..227767415 100644 --- a/common/sds_opensearch_query/mapping.py +++ b/common/sds_opensearch_query/mapping.py @@ -3,7 +3,6 @@ from collections.abc import Mapping from typing import Any - FED_INDEX_SETTINGS: dict[str, Any] = { "index": { "number_of_shards": 1, diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index 0b6c3be69..ae1d5608d 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -754,7 +754,8 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: # Federation peer short name (RFC [site].name, e.g. crc, haystack); # used for Redis channel ``federation:events:{name}``, not OpenSearch site_name. -# Export/OpenSearch ``site_name`` uses SDS_SITE_FQDN (matches federation.toml [site].fqdn). +# Export/OpenSearch ``site_name`` uses SDS_SITE_FQDN +# (matches federation.toml [site].fqdn). FEDERATION_SITE_NAME: str = env.str("FEDERATION_SITE_NAME", default="").strip() # Master switch: when False, federation export and Redis events are inactive. FEDERATION_ENABLED: bool = env.bool("FEDERATION_ENABLED", default=False) diff --git a/gateway/sds_gateway/api_methods/federation/search_helpers.py b/gateway/sds_gateway/api_methods/federation/search_helpers.py index 7da68e766..c48ff8a48 100644 --- a/gateway/sds_gateway/api_methods/federation/search_helpers.py +++ b/gateway/sds_gateway/api_methods/federation/search_helpers.py @@ -2,10 +2,9 @@ from __future__ import annotations -from collections.abc import Mapping +from typing import TYPE_CHECKING from typing import Any -from opensearchpy import OpenSearch from sds_opensearch_query import bool_must_search_body from sds_opensearch_query import build_metadata_filter_clauses from sds_opensearch_query import federation_not_deleted_clause @@ -20,6 +19,11 @@ from sds_gateway.api_methods.federation.fed_index import FED_CAPTURES_INDEX from sds_gateway.api_methods.federation.fed_index import FED_DATASETS_INDEX +if TYPE_CHECKING: + from collections.abc import Mapping + + from opensearchpy import OpenSearch + FED_DATASET_TEXT_FIELDS = [ "name^2", "description", diff --git a/gateway/sds_gateway/api_methods/helpers/list_helpers.py b/gateway/sds_gateway/api_methods/helpers/list_helpers.py index fe23529a4..f91280231 100644 --- a/gateway/sds_gateway/api_methods/helpers/list_helpers.py +++ b/gateway/sds_gateway/api_methods/helpers/list_helpers.py @@ -34,17 +34,16 @@ def _parse_datetime(value: Any) -> datetime | None: if value is None or value == "": return None if isinstance(value, datetime): - if timezone.is_naive(value): - return timezone.make_aware(value, timezone.get_current_timezone()) - return value - if isinstance(value, str): + parsed: datetime | None = value + elif isinstance(value, str): parsed = dateparse.parse_datetime(value) - if parsed is None: - return None - if timezone.is_naive(parsed): - return timezone.make_aware(parsed, timezone.get_current_timezone()) - return parsed - return None + else: + return None + if parsed is None: + return None + if timezone.is_naive(parsed): + return timezone.make_aware(parsed, timezone.get_current_timezone()) + return parsed def dataset_list_dropdown_menu_items(row: dict[str, Any]) -> list[dict[str, Any]]: @@ -157,9 +156,7 @@ def dataset_list_dropdown_menu_items(row: dict[str, Any]) -> list[dict[str, Any] def _keyword_names(dataset: Dataset) -> list[str]: return [ - kw.name - for kw in dataset.keywords.all() - if not getattr(kw, "is_deleted", False) + kw.name for kw in dataset.keywords.all() if not getattr(kw, "is_deleted", False) ] diff --git a/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py b/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py index 27c8280e0..8aee53f13 100644 --- a/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py +++ b/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py @@ -44,9 +44,7 @@ def handle(self, *args, **options) -> None: uuid__in=[UUID(str(u)) for u in capture_uuids], is_deleted=False, ) - missing = set(capture_uuids) - { - str(c.uuid) for c in captures - } + missing = set(capture_uuids) - {str(c.uuid) for c in captures} if missing: msg = f"Captures not found: {sorted(missing)}" raise CommandError(msg) diff --git a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py index 81b8c0c64..ce9c79438 100644 --- a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py +++ b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py @@ -347,11 +347,11 @@ class DatasetFederationSerializer(DatasetPublicSerializer): site_name = serializers.SerializerMethodField() created_at = serializers.DateTimeField( - format="iso-8601", # ISO 8601 format for OpenSearch schema + format="iso-8601", # ISO 8601 format for OpenSearch schema read_only=True, ) updated_at = serializers.DateTimeField( - format="iso-8601", # ISO 8601 format for OpenSearch schema + format="iso-8601", # ISO 8601 format for OpenSearch schema read_only=True, ) size = serializers.SerializerMethodField() diff --git a/gateway/sds_gateway/templates/pages/home.html b/gateway/sds_gateway/templates/pages/home.html index 01e338e49..b179d2022 100644 --- a/gateway/sds_gateway/templates/pages/home.html +++ b/gateway/sds_gateway/templates/pages/home.html @@ -64,7 +64,11 @@
data-item-uuid="{{ dataset.uuid }}" data-item-type="dataset" aria-label="View details for {{ dataset.name }}">{{ dataset.name }} - {% if dataset.is_public %}{% endif %} + {% if dataset.is_federated %} + {{ dataset.site_name }} + {% elif dataset.is_public %} + + {% endif %}
diff --git a/gateway/sds_gateway/templates/users/partials/dataset_search_form.html b/gateway/sds_gateway/templates/users/partials/dataset_search_form.html index 64a0bfd6b..2b90dbc5c 100644 --- a/gateway/sds_gateway/templates/users/partials/dataset_search_form.html +++ b/gateway/sds_gateway/templates/users/partials/dataset_search_form.html @@ -9,14 +9,21 @@

Search Published Datasets

action="{% url 'users:search_datasets' %}" class="flex-grow-1 d-flex flex-column mt-5">
-
+
{{ search_form.query }} {% if search_form.query.help_text %} {{ search_form.query.help_text }} {% endif %}
-
+
+ + {{ search_form.site_name }} + {% if search_form.site_name.help_text %} + {{ search_form.site_name.help_text }} + {% endif %} +
+
diff --git a/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html b/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html index a0a8f9eef..5270393a9 100644 --- a/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html +++ b/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html @@ -19,7 +19,7 @@