diff --git a/docs/user_guide.md b/docs/user_guide.md index 15b927b0..e724037a 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -321,6 +321,52 @@ and you can see all the stored metadata against a remote simulation using: simdb remote info ``` +To filter simulations and return selected quantities from their metadata, use +`data-query`. Each quantity has a response name and a metadata path: + +```bash +simdb remote data-query \ + 'summary.global_quantities.ip.value=gt:-5' \ + 'summary.global_quantities.ip.value=lt:10' \ + --quantity 'ip=summary.global_quantities.ip.value' \ + --quantity 'q95=summary.global_quantities.q_95.value' +``` + +The corresponding REST endpoint accepts the same operation as a structured request: + +```text +POST /v1.3/simulations/data/query +``` + +```json +{ + "filters": [ + { + "field": "summary.global_quantities.ip.value", + "operator": "gt", + "value": -5 + }, + { + "field": "summary.global_quantities.ip.value", + "operator": "lt", + "value": 10 + } + ], + "quantities": [ + { + "name": "ip", + "path": "summary.global_quantities.ip.value", + "source": "metadata" + } + ], + "page": 1, + "limit": 100 +} +``` + +For range metadata, `gt`/`ge` compare the minimum and `lt`/`le` compare the maximum. +Scalar numeric metadata keeps the existing comparison behavior. + ## Accessing Simulation Metadata via the SimDB Dashboard You can view a simulation's metadata directly in the SimDB dashboard using its UUID. diff --git a/src/simdb/cli/commands/remote.py b/src/simdb/cli/commands/remote.py index 4401464b..8bc515f5 100644 --- a/src/simdb/cli/commands/remote.py +++ b/src/simdb/cli/commands/remote.py @@ -1,3 +1,4 @@ +import json import re import shutil import sys @@ -12,6 +13,7 @@ from simdb.cli.remote_api import RemoteAPI from simdb.database.models.simulation import Simulation from simdb.notifications import Notification +from simdb.query import QueryType, parse_query_arg from . import check_meta_args, pass_config from .utils import print_simulations, print_trace @@ -534,6 +536,81 @@ def remote_query( ) +@remote.command("data-query", cls=remote_command_cls()) +@pass_api +@click.argument("constraints", nargs=-1) +@click.option( + "-q", + "--quantity", + "quantities", + multiple=True, + required=True, + metavar="NAME=METADATA_PATH", + help="Stored metadata quantity to return. May be provided multiple times.", +) +@click.option( + "-l", + "--limit", + default=100, + show_default=True, + callback=validate_positive, + help="Maximum simulations returned per page.", +) +@click.option( + "--page", + default=1, + show_default=True, + callback=validate_positive, + help="One-indexed result page.", +) +def remote_data_query( + api: RemoteAPI, + constraints: Tuple[str, ...], + quantities: Tuple[str, ...], + limit: int, + page: int, +): + """Filter simulations and return quantities from remote database metadata. + + CONSTRAINTS use the same ``NAME=[operator:]VALUE`` syntax as ``remote query``. + """ + filters = [] + for constraint in constraints: + if "=" not in constraint: + raise click.ClickException(f"Invalid constraint {constraint}.") + field, expression = constraint.split("=", 1) + value, operator = parse_query_arg(expression) + if operator == QueryType.NONE: + raise click.ClickException( + f"Invalid constraint {constraint}; a value or operator is required." + ) + filters.append( + { + "field": field, + "operator": operator.name.lower(), + "value": None if operator == QueryType.EXIST else value, + } + ) + + requested_quantities = [] + for quantity in quantities: + if "=" not in quantity: + raise click.ClickException( + f"Invalid quantity {quantity}; expected NAME=METADATA_PATH." + ) + name, path = quantity.split("=", 1) + if not name or not path: + raise click.ClickException( + f"Invalid quantity {quantity}; expected NAME=METADATA_PATH." + ) + requested_quantities.append({"name": name, "path": path, "source": "metadata"}) + + result = api.query_simulation_data( + filters, requested_quantities, limit=limit, page=page + ) + click.echo(json.dumps(result, indent=2)) + + @remote.group(cls=RemoteSubGroup) def token(): """Manage user authentication tokens.""" diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 99f87363..05990dbd 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -712,7 +712,28 @@ def query_simulations( data = res.json(cls=CustomDecoder) return [Simulation.from_data(sim) for sim in data["results"]] - @versioned_method("v1.2", "v1.3") + @versioned_method("v1.3") + @try_request + def query_simulation_data( + self, + filters: List[Dict[str, Any]], + quantities: List[Dict[str, str]], + limit: int = 100, + page: int = 1, + ) -> Dict[str, Any]: + """Filter remote simulations and select stored metadata quantities.""" + response = self.post( + "simulations/data/query", + { + "filters": filters, + "quantities": quantities, + "limit": limit, + "page": page, + }, + ) + return response.json() + + @versioned_method("v1.2") @try_request def delete_simulation(self, sim_id: str) -> Dict: res = self.delete("simulation/" + sim_id, {}) diff --git a/src/simdb/database/database.py b/src/simdb/database/database.py index 356bb73e..a067fa17 100644 --- a/src/simdb/database/database.py +++ b/src/simdb/database/database.py @@ -486,10 +486,10 @@ def _number_cmp(cmp_op): cmp_op(), ) - def _num_with_op(cmp_op): + def _scalar_or_range_cmp(scalar_cmp_op, range_cmp_op): if cmp_float is None: return None - return _number_cmp(cmp_op) + return sql_or(_number_cmp(scalar_cmp_op), range_cmp_op()) if query_type == QueryType.EQ: return _string_cmp(lambda a, b: a == b) @@ -500,13 +500,25 @@ def _num_with_op(cmp_op): elif query_type == QueryType.NI: return _string_cmp(lambda a, b: a.notilike(f"%{b}%")) elif query_type == QueryType.GT: - return _num_with_op(lambda: sql_cast(json_access, Float) > cmp_float) + return _scalar_or_range_cmp( + lambda: sql_cast(json_access, Float) > cmp_float, + lambda: sql_cast(json_min, Float) > cmp_float, + ) elif query_type == QueryType.GE: - return _num_with_op(lambda: sql_cast(json_access, Float) >= cmp_float) + return _scalar_or_range_cmp( + lambda: sql_cast(json_access, Float) >= cmp_float, + lambda: sql_cast(json_min, Float) >= cmp_float, + ) elif query_type == QueryType.LT: - return _num_with_op(lambda: sql_cast(json_access, Float) < cmp_float) + return _scalar_or_range_cmp( + lambda: sql_cast(json_access, Float) < cmp_float, + lambda: sql_cast(json_max, Float) < cmp_float, + ) elif query_type == QueryType.LE: - return _num_with_op(lambda: sql_cast(json_access, Float) <= cmp_float) + return _scalar_or_range_cmp( + lambda: sql_cast(json_access, Float) <= cmp_float, + lambda: sql_cast(json_max, Float) <= cmp_float, + ) elif query_type == QueryType.AGT: if cmp_float is not None: return sql_cast(json_max, Float) > cmp_float diff --git a/src/simdb/remote/apis/simulation_data_query.py b/src/simdb/remote/apis/simulation_data_query.py new file mode 100644 index 00000000..ff6f7aa0 --- /dev/null +++ b/src/simdb/remote/apis/simulation_data_query.py @@ -0,0 +1,88 @@ +"""Database-backed simulation filtering and quantity selection endpoint.""" + +from typing import Annotated + +from flask_restx import Namespace, Resource + +from simdb.query import QueryType +from simdb.remote.core.auth import User, requires_auth +from simdb.remote.core.pydantic_utils import Body, pydantic_validate +from simdb.remote.core.typing import current_app +from simdb.remote.models import ( + PaginatedResponse, + SimulationDataQuantityResult, + SimulationDataQueryItem, + SimulationDataQueryRequest, +) + +api = Namespace("simulation-data-query", path="/") + + +def _filter_value(value) -> str: + if isinstance(value, bool): + return str(value).lower() + return "" if value is None else str(value) + + +@api.route("/simulations/data/query") +class SimulationDataQuery(Resource): + @requires_auth() + @pydantic_validate(api) + def post( + self, + user: User, + body: Annotated[SimulationDataQueryRequest, Body()], + ) -> PaginatedResponse[SimulationDataQueryItem]: + """Filter simulations and retrieve quantities from stored metadata.""" + constraints = [ + ( + item.field, + _filter_value(item.value), + QueryType[item.operator.upper()], + ) + for item in body.filters + ] + metadata_paths = list(dict.fromkeys(item.path for item in body.quantities)) + + count, rows = current_app.db.query_meta_data( + constraints, + metadata_paths, + limit=body.limit, + page=body.page, + sort_by=body.sort_by, + sort_asc=body.sort_asc, + ) + + results = [] + for row in rows: + metadata = { + item["element"]: item["value"] for item in row.get("metadata", []) + } + quantities = {} + missing = [] + for requested in body.quantities: + if requested.path not in metadata: + missing.append(requested.name) + continue + quantities[requested.name] = SimulationDataQuantityResult( + source=requested.source, + path=requested.path, + value=metadata[requested.path], + ) + + results.append( + SimulationDataQueryItem( + uuid=row["uuid"], + alias=row["alias"], + datetime=row["datetime"], + quantities=quantities, + missing=missing, + ) + ) + + return PaginatedResponse[SimulationDataQueryItem]( + count=count, + page=body.page, + limit=body.limit, + results=results, + ) diff --git a/src/simdb/remote/apis/v1_3/__init__.py b/src/simdb/remote/apis/v1_3/__init__.py index eab5e0dc..8782296c 100644 --- a/src/simdb/remote/apis/v1_3/__init__.py +++ b/src/simdb/remote/apis/v1_3/__init__.py @@ -2,6 +2,7 @@ from simdb.remote.apis.files import api as file_ns from simdb.remote.apis.metadata import api as metadata_ns +from simdb.remote.apis.simulation_data_query import api as simulation_data_query_ns from simdb.remote.apis.v1_2 import StagingDirectory from simdb.remote.apis.v1_2 import api as api_v1_2 from simdb.remote.apis.v1_2.simulations import api as sim_ns @@ -28,7 +29,14 @@ doc="/docs", ) -namespaces = [metadata_ns, watcher_ns, file_ns, sim_ns, data_ns] +namespaces = [ + metadata_ns, + watcher_ns, + file_ns, + sim_ns, + data_ns, + simulation_data_query_ns, +] api.route("/staging_dir", defaults={"sim_hex": None})(StagingDirectory) api.route("/staging_dir/")(StagingDirectory) diff --git a/src/simdb/remote/models.py b/src/simdb/remote/models.py index eb3827ae..ba6aa3e6 100644 --- a/src/simdb/remote/models.py +++ b/src/simdb/remote/models.py @@ -390,6 +390,158 @@ class SimulationListItem(BaseModel): """Simulation metadata.""" +MetadataQueryOperator = Literal[ + "eq", + "ne", + "in", + "ni", + "gt", + "ge", + "lt", + "le", + "agt", + "age", + "alt", + "ale", + "exist", +] +"""Operators supported by database metadata queries.""" + + +class SimulationDataFilter(BaseModel): + """A filter applied to stored simulation metadata.""" + + field: str = Field(min_length=1) + """Metadata key to filter on.""" + operator: MetadataQueryOperator = "eq" + """Comparison operator.""" + value: Optional[Union[str, int, float, bool]] = None + """Comparison value. Omit only when using the ``exist`` operator.""" + + @model_validator(mode="after") + def validate_filter(self): + self.field = self.field.strip() + if not self.field: + raise ValueError("field must not be blank") + + if self.operator != "exist" and self.value is None: + raise ValueError("value is required unless operator is 'exist'") + + scalar_operators = {"eq", "ne", "in", "ni", "gt", "ge", "lt", "le"} + allowed_reserved_operators = { + "alias": {"eq", "ne", "in", "ni", "exist"}, + "uuid": {"eq", "ne", "in", "ni", "exist"}, + "creation_date": {"eq", "ne", "gt", "ge", "lt", "le"}, + } + allowed = allowed_reserved_operators.get(self.field) + if allowed is not None and self.operator not in allowed: + raise ValueError( + f"operator '{self.operator}' is not supported for field '{self.field}'" + ) + + if isinstance(self.value, bool) and self.operator not in {"eq", "ne"}: + raise ValueError("Boolean filter values support only 'eq' and 'ne'") + + numeric_operators = { + "gt", + "ge", + "lt", + "le", + "agt", + "age", + "alt", + "ale", + } + if self.operator in numeric_operators and self.field != "creation_date": + assert self.value is not None + try: + float(self.value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"operator '{self.operator}' requires a numeric value" + ) from exc + + if self.field == "uuid" and self.operator in {"eq", "ne"}: + try: + UUID(str(self.value)) + except ValueError as exc: + raise ValueError("invalid UUID filter value") from exc + + if ( + self.field == "creation_date" + and self.operator in scalar_operators + and self.operator not in {"in", "ni"} + ): + try: + dt.strptime(str(self.value).replace("_", ":"), "%Y-%m-%d %H:%M:%S") + except ValueError as exc: + raise ValueError("creation_date must use YYYY-MM-DD HH_MM_SS") from exc + return self + + +class SimulationDataQuantity(BaseModel): + """A quantity requested for every matching simulation.""" + + name: str = Field(min_length=1) + """Name used for this quantity in the response.""" + path: str = Field(min_length=1) + """Stored metadata key. May later identify another supported data source.""" + source: Literal["metadata"] = "metadata" + """Quantity provider. Only database metadata is currently supported.""" + + +class SimulationDataQueryRequest(BaseModel): + """Filter simulations and select quantities from stored metadata.""" + + filters: List[SimulationDataFilter] = Field(default_factory=list, max_length=50) + """Metadata constraints. All filters are combined with logical AND.""" + quantities: List[SimulationDataQuantity] = Field(min_length=1, max_length=50) + """Quantities returned for each matching simulation.""" + page: int = Field(1, ge=1) + """One-indexed result page.""" + limit: int = Field(100, ge=1, le=1000) + """Maximum simulations returned on this page.""" + sort_by: Literal["", "alias", "uuid", "datetime"] = "" + """Optional simulation field used for sorting.""" + sort_asc: bool = False + """Sort ascending when true, descending otherwise.""" + + @model_validator(mode="after") + def require_unique_quantity_names(self): + names = [quantity.name for quantity in self.quantities] + if len(names) != len(set(names)): + raise ValueError("quantity names must be unique") + return self + + +class SimulationDataQuantityResult(BaseModel): + """Value returned by a quantity provider.""" + + source: Literal["metadata"] + """Provider used to retrieve the quantity.""" + path: str + """Provider-specific quantity path.""" + value: MetadataValue + """Retrieved value.""" + + +class SimulationDataQueryItem(BaseModel): + """Selected quantities for one matching simulation.""" + + uuid: CustomUUID + """Simulation UUID.""" + alias: Optional[str] = None + """Simulation alias.""" + datetime: str + """Simulation creation timestamp.""" + quantities: Dict[str, SimulationDataQuantityResult] = Field(default_factory=dict) + """Retrieved quantities, keyed by their requested names.""" + missing: List[str] = Field(default_factory=list) + """Requested quantity names not present in this simulation's metadata.""" + errors: Dict[str, str] = Field(default_factory=dict) + """Provider errors keyed by requested quantity name.""" + + T = TypeVar("T") """Type variable for generic paginated responses.""" diff --git a/tests/cli/test_cli_remote_command.py b/tests/cli/test_cli_remote_command.py index 0e626042..6166f4f5 100644 --- a/tests/cli/test_cli_remote_command.py +++ b/tests/cli/test_cli_remote_command.py @@ -1,8 +1,10 @@ from unittest import mock +import pytest from click.testing import CliRunner from utils import config_test_file +from simdb.cli.remote_api import RemoteAPI, RemoteError from simdb.cli.simdb import cli from simdb.notifications import Notification @@ -335,3 +337,87 @@ def test_remote_query_command_with_verbose( assert args == (constraints, (), 100) assert kwargs == {} assert get_api_version.called + + +@mock.patch("simdb.cli.remote_api.RemoteAPI.get_server_authentication") +@mock.patch("simdb.cli.remote_api.RemoteAPI.get_endpoints") +@mock.patch("simdb.cli.remote_api.RemoteAPI.get_api_version") +@mock.patch("simdb.cli.remote_api.RemoteAPI.get_server_version") +@mock.patch("simdb.cli.remote_api.RemoteAPI.query_simulation_data") +def test_remote_data_query_command( + query_simulation_data, + get_server_version, + get_api_version, + get_endpoints, + get_server_authentication, +): + get_endpoints.return_value = ["v1", "v1.1", "v1.1.1", "v1.2", "v1.3"] + get_api_version.return_value = "1.3" + get_server_version.return_value = "0.11" + get_server_authentication.return_value = "None" + query_simulation_data.return_value = { + "count": 1, + "page": 1, + "limit": 25, + "results": [{"alias": "inside-range", "quantities": {"ip": 2.0}}], + } + + config_file = config_test_file() + runner = CliRunner() + result = runner.invoke( + cli, + [ + f"--config-file={config_file}", + "remote", + "data-query", + "--limit", + "25", + "--quantity", + "ip=summary.global_quantities.ip.value", + "summary.global_quantities.ip.value=gt:-5", + "summary.global_quantities.ip.value=lt:10", + "summary.valid=eq:true", + ], + ) + + assert result.exception is None + assert '"inside-range"' in result.output + query_simulation_data.assert_called_once_with( + [ + { + "field": "summary.global_quantities.ip.value", + "operator": "gt", + "value": "-5", + }, + { + "field": "summary.global_quantities.ip.value", + "operator": "lt", + "value": "10", + }, + { + "field": "summary.valid", + "operator": "eq", + "value": "true", + }, + ], + [ + { + "name": "ip", + "path": "summary.global_quantities.ip.value", + "source": "metadata", + } + ], + limit=25, + page=1, + ) + + +def test_remote_data_query_rejects_v1_2(): + remote = mock.Mock(spec=RemoteAPI) + remote._api_version = "v1.2" + + with pytest.raises( + RemoteError, + match=r"'query_simulation_data' is not supported.*version 'v1.2'", + ): + RemoteAPI.query_simulation_data(remote, [], []) diff --git a/tests/database/test_metadata_queries.py b/tests/database/test_metadata_queries.py index f7630055..95fd5709 100644 --- a/tests/database/test_metadata_queries.py +++ b/tests/database/test_metadata_queries.py @@ -320,3 +320,18 @@ def test_query_meta_data_with_constraint(self, db): assert count == 1 assert len(results) == 1 assert results[0]["metadata"] == [{"element": "type", "value": "A"}] + + def test_query_meta_data_with_range_bounds(self, db): + db.insert_simulation( + create_simulation( + alias="inside", + metadata={"range": {"min": -4.0, "max": 9.0}}, + ) + ) + db.session.commit() + constraints = [("range", "-5", QueryType.GT)] + + count, results = db.query_meta_data(constraints, []) + + assert count == 1 + assert results[0]["alias"] == "inside"