diff --git a/README.md b/README.md index c36b805..0bb36a6 100644 --- a/README.md +++ b/README.md @@ -109,12 +109,29 @@ arquivos de configuração de ambiente que contenham credenciais. ## Cobertura parcial nas extensões -Ranking e distribuição usam `coverage_policy=allow_partial` por padrão: -contam dias confirmados no mensal e os totais disponíveis no anual de país/idioma. -JSON, CSV e XLSX preservam a cobertura e os avisos, inclusive sem resultados. -Use `coverage_policy=strict` para exigir cobertura completa, como antes. -País/idioma exigem intervalos de anos inteiros; seus valores podem ser atualizados -com novas cargas. A cobertura anual é informativa, não um filtro dos totais. +Ranking e distribuição aceitam cortes diários e o mês corrente. Use datas +`YYYY-MM-DD` ou meses `YYYY-MM`; o mês corrente como fim termina em hoje. +Datas diárias futuras são rejeitadas. `coverage_policy=allow_partial` conta +somente dias confirmados dentro do intervalo; meses completos usam os totais +mensais. A cobertura distingue dias solicitados, confirmados, ausentes e futuros. +Use `coverage_policy=strict` para exigir os meses inteiros processados, mesmo +quando a consulta corta apenas alguns dias. + +País/idioma continuam anuais: selecione janeiro–dezembro, inclusive do ano +corrente, para obter os totais disponíveis. A cobertura é informativa, não um +filtro desses totais; métricas `Unique` não são suportadas nesses recortes. +A descoberta de disponibilidade das extensões inclui meses com dias confirmados. + +Rankings com `group_by` paginam os grupos: `group_limit=5` por padrão (máximo 20), +com `limit=100` itens por grupo (máximo 1000). O produto dos limites não pode +exceder 10000. Continue com `pagination.next_cursor` e os mesmos filtros/período, +mesmo se a página estiver vazia; `null` encerra a navegação. Formato e limites +podem mudar entre páginas. Distribuições retornam até 10000 grupos; consultas +maiores falham explicitamente, sem truncamento. + +JSON, CSV e XLSX preservam cobertura, avisos e cursor, inclusive sem resultados. +No XLSX, `Coverage` contém o resumo e `Coverage months` detalha um mês por linha, +com a indicação `Included`, evitando o limite de texto por célula do Excel. Consultas mensais sem dias confirmados e marcadores inconsistentes retornam erro. Relatórios COUNTER padrão não mudam. Consulte o OpenAPI para o contrato completo. diff --git a/VERSION b/VERSION index 24ba9a3..860487c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.7.0 +2.7.1 diff --git a/counter_api/availability.py b/counter_api/availability.py index defa5de..d544ffb 100644 --- a/counter_api/availability.py +++ b/counter_api/availability.py @@ -12,11 +12,26 @@ PAGE_SIZE = 1000 -def available_months(platform): +def available_months(platform, complete_only=True): index_name = generate_month_status_index_name(settings.OPENSEARCH_INDEX_NAME) client = OpenSearchUsageClient().client deadline = monotonic() + settings.COUNTER_QUERY_BUDGET_SECONDS current_month = date.today().strftime("%Y-%m") + filters = [{"term": {"collection": platform.acron3}}] + if complete_only: + filters.extend( + [ + {"term": {"complete": True}}, + {"range": {"month": {"lt": current_month}}}, + ] + ) + else: + filters.extend( + [ + {"range": {"day_mask": {"gt": 0}}}, + {"range": {"month": {"lte": current_month}}}, + ] + ) months = [] after = None @@ -24,15 +39,7 @@ def available_months(platform): body = { "size": 0, "track_total_hits": False, - "query": { - "bool": { - "filter": [ - {"term": {"collection": platform.acron3}}, - {"term": {"complete": True}}, - {"range": {"month": {"lt": current_month}}}, - ] - } - }, + "query": {"bool": {"filter": filters}}, "aggs": { "months": { "composite": { diff --git a/counter_api/constants.py b/counter_api/constants.py index 4d1a7bb..22aeace 100644 --- a/counter_api/constants.py +++ b/counter_api/constants.py @@ -3,6 +3,7 @@ COMPOSITE_PAGE_SIZE = 1000 METADATA_BATCH_SIZE = 1000 EXCEL_MAX_ROWS = 1048576 +EXCEL_MAX_CELL_LENGTH = 32767 SPOOL_SIZE = 8 * 1024 * 1024 METRICS = { "Total_Item_Investigations": "total_investigations", diff --git a/counter_api/dates.py b/counter_api/dates.py index 062b58d..51d5c8b 100644 --- a/counter_api/dates.py +++ b/counter_api/dates.py @@ -4,7 +4,7 @@ from counter_api.exceptions import insufficient_information, invalid_dates -def parse_period(params): +def parse_period_inputs(params): begin_value = params.get("begin_date") end_value = params.get("end_date") if not begin_value or not end_value: @@ -14,6 +14,11 @@ def parse_period(params): end = _parse_date(end_value, first=False) if begin > end: raise invalid_dates("end_date must not be earlier than begin_date") + return begin, end + + +def parse_period(params): + begin, end = parse_period_inputs(params) if begin.day != 1: raise invalid_dates("begin_date must be the first day of a month") if end.day != calendar.monthrange(end.year, end.month)[1]: diff --git a/counter_api/extensions/contracts.py b/counter_api/extensions/contracts.py index bfaed56..d20d94d 100644 --- a/counter_api/extensions/contracts.py +++ b/counter_api/extensions/contracts.py @@ -1,3 +1,4 @@ +MAX_RESULTS = 10000 DOWNLOAD_FORMATS = ("json", "csv", "xlsx") SEGMENT_DIMENSIONS = {"country", "language"} RANKING_GROUPS = {"", "country", "language", "country,language"} diff --git a/counter_api/extensions/coverage.py b/counter_api/extensions/coverage.py index 48afa8b..d84d143 100644 --- a/counter_api/extensions/coverage.py +++ b/counter_api/extensions/coverage.py @@ -1,4 +1,5 @@ from calendar import monthrange +from datetime import date from django.conf import settings from opensearchpy import NotFoundError @@ -14,9 +15,13 @@ def resolve_coverage(client, platform, begin, end, policy, segmented, deadline): if policy not in {"strict", "allow_partial"}: raise insufficient_information() - if policy == "strict": + if policy == "strict" and ( + segmented or (begin.day == 1 and end.day == monthrange(end.year, end.month)[1]) + ): return None + today = date.today() + months = { month.strftime("%Y-%m"): monthrange(month.year, month.month)[1] for month in iter_months(begin, end) @@ -44,11 +49,20 @@ def resolve_coverage(client, platform, begin, end, policy, segmented, deadline): details = [] for month, day_count in months.items(): - days = range(1, day_count + 1) + month_date = date.fromisoformat(f"{month}-01") + requested = [ + day + for day in range(1, day_count + 1) + if begin <= month_date.replace(day=day) <= end + ] + future = [day for day in requested if month_date.replace(day=day) > today] + elapsed = [day for day in requested if day not in future] + marker = markers.get(month) confirmed = [] - missing = None - status = "unknown" + missing = None if elapsed else [] + status = "unknown" if elapsed else "future" + complete_month = False if marker is not None: mask = marker.get("day_mask") @@ -62,30 +76,50 @@ def resolve_coverage(client, platform, begin, end, policy, segmented, deadline): ): raise service_unavailable(f"Invalid coverage marker for {month}") - confirmed = [day for day in days if mask & (1 << (day - 1))] - missing = [day for day in days if day not in confirmed] - status = ( - "complete" if not missing else "partial" if confirmed else "unavailable" - ) + confirmed = [day for day in elapsed if mask & (1 << (day - 1))] + missing = [day for day in elapsed if day not in confirmed] + complete_month = mask == full and len(confirmed) == day_count + + if not elapsed: + status = "future" + elif missing: + status = "partial" if confirmed else "unavailable" + else: + status = "ongoing" if future else "complete" details.append( { "month": month, "status": status, + "requested_days": requested, "confirmed_days": confirmed, "missing_days": missing, + "future_days": future, + "ongoing": bool(future), + "complete_month": complete_month, } ) included = [item["month"] for item in details if item["confirmed_days"]] - complete = all(item["status"] == "complete" for item in details) + statuses = {item["status"] for item in details} + ongoing = any(item["future_days"] for item in details) + + if statuses == {"complete"}: + status = "complete" + elif statuses == {"future"}: + status = "future" + elif statuses <= {"complete", "ongoing", "future"}: + status = "ongoing" + else: + status = "partial" if not segmented and not included: raise service_unavailable("No confirmed usage days in requested period") return { "policy": policy, - "status": "complete" if complete else "partial", + "status": status, + "ongoing": ongoing, "basis": "available_annual_totals" if segmented else "confirmed_days", "included_months": None if segmented else included, "months": details, @@ -105,18 +139,22 @@ def coverage_fields(coverage): ) elif coverage["status"] != "complete": warnings.append( - "Partial coverage: counts include only confirmed days; missing or unknown days are not zero usage." + "Partial coverage: counts include only confirmed days; missing or unknown days are not zero usage. " + "Future days are not missing usage." ) return {"coverage": coverage, "warnings": warnings} def metric_sum(metric, coverage): + if coverage and coverage["basis"] == "available_annual_totals": + return {"field": metric} + partial = ( { item["month"]: [f"{day:02d}" for day in item["confirmed_days"]] for item in coverage["months"] - if item["status"] == "partial" + if item["confirmed_days"] and not item["complete_month"] } if coverage else {} @@ -132,7 +170,7 @@ def metric_sum(metric, coverage): "complete": [ item["month"] for item in coverage["months"] - if item["status"] == "complete" + if item["complete_month"] ], "days": partial, }, diff --git a/counter_api/extensions/dates.py b/counter_api/extensions/dates.py new file mode 100644 index 0000000..aadcb49 --- /dev/null +++ b/counter_api/extensions/dates.py @@ -0,0 +1,43 @@ +from calendar import monthrange +from datetime import date + +from counter_api.dates import parse_period_inputs +from counter_api.exceptions import invalid_dates +from counter_api.extensions.contracts import SEGMENT_DIMENSIONS +from counter_api.extensions.query import validate_segmented_period + + +def parse_analytics_period(params): + begin, end = parse_period_inputs(params) + today = date.today() + + dimensions = { + part.strip() + for name in ("group_by", "dimension") + for part in params.get(name, "").split(",") + } + dimensions.update(key for key in SEGMENT_DIMENSIONS if params.get(key)) + + segmented = validate_segmented_period( + begin, end, dimensions, params.get("metric_type", "Total_Item_Requests") + ) + + if segmented: + if begin.day != 1 or end.day != monthrange(end.year, end.month)[1]: + raise invalid_dates("Segmented periods must cover full calendar years") + if end.year > today.year: + raise invalid_dates("end_date must not be later than the current year") + else: + if ( + len(params["end_date"]) == 7 + and end.year == today.year + and end.month == today.month + ): + end = today + + if end > today: + raise invalid_dates("end_date must not be later than today") + if begin > end: + raise invalid_dates("end_date must not be earlier than begin_date") + + return begin, end diff --git a/counter_api/extensions/distribution.py b/counter_api/extensions/distribution.py index 4e2b4a7..28a24d1 100644 --- a/counter_api/extensions/distribution.py +++ b/counter_api/extensions/distribution.py @@ -3,11 +3,12 @@ from django.conf import settings from counter_api.constants import METRICS -from counter_api.exceptions import insufficient_information +from counter_api.exceptions import insufficient_information, invalid_filter from counter_api.extensions.contracts import ( DISTRIBUTION_CROSSES, DISTRIBUTION_DIMENSIONS, DISTRIBUTION_ENTITIES, + MAX_RESULTS, ) from counter_api.extensions.coverage import coverage_fields, resolve_coverage from counter_api.extensions.query import ( @@ -21,7 +22,6 @@ from counter_api.search import composite_pages PAGE_SIZE = 1000 -MAX_BUCKETS = 1000 class DistributionQuery: @@ -48,7 +48,7 @@ def run(self, platform, begin, end, params): segmented, deadline, ) - if coverage is None: + if coverage is None or params.get("coverage_policy") == "strict": require_complete_period(platform, begin, end) key_field, entity_key = self._entity_filter( @@ -210,8 +210,10 @@ def _buckets(self, index_name, body, dimensions, deadline): for combination in self._combinations(bucket, dimensions, metadata): totals[combination] = totals.get(combination, 0) + count - if len(totals) > MAX_BUCKETS: - raise insufficient_information() + if len(totals) > MAX_RESULTS: + raise invalid_filter( + f"Distribution exceeds {MAX_RESULTS} groups; narrow the period or entity." + ) buckets = [ { diff --git a/counter_api/extensions/exports.py b/counter_api/extensions/exports.py index f0579c7..697bd5e 100644 --- a/counter_api/extensions/exports.py +++ b/counter_api/extensions/exports.py @@ -7,7 +7,7 @@ from openpyxl.cell import WriteOnlyCell from openpyxl.styles import Font -from counter_api.constants import EXCEL_MAX_ROWS, SPOOL_SIZE +from counter_api.constants import EXCEL_MAX_CELL_LENGTH, EXCEL_MAX_ROWS, SPOOL_SIZE from counter_api.exceptions import insufficient_information @@ -39,16 +39,23 @@ def build_export(payload, kind, output_format): for bucket in payload["buckets"] ) - metadata = None + metadata_headings = [] + metadata = [] if "coverage" in payload: - metadata = [ - json.dumps(payload["coverage"], separators=(",", ":")), - json.dumps(payload["warnings"]), - payload.get("total", ""), - ] - if output_format == "csv": - headings.extend(["Coverage", "Warnings", "Total"]) - rows = (row + metadata for row in rows) + metadata_headings.extend(["Coverage", "Warnings", "Total"]) + metadata.extend( + [ + json.dumps(payload["coverage"], separators=(",", ":")), + json.dumps(payload["warnings"]), + payload.get("total", ""), + ] + ) + if "pagination" in payload: + metadata_headings.append("Pagination") + metadata.append(json.dumps(payload["pagination"], separators=(",", ":"))) + if metadata and output_format == "csv": + headings.extend(metadata_headings) + rows = (row + metadata for row in rows) output = SpooledTemporaryFile(max_size=SPOOL_SIZE) if output_format == "csv": @@ -67,12 +74,26 @@ def build_export(payload, kind, output_format): line.seek(0) line.truncate(0) - if metadata is not None and not has_rows: + if metadata and not has_rows: writer.writerow([""] * (len(headings) - len(metadata)) + metadata) output.write(line.getvalue().encode("utf-8")) else: + coverage = payload.get("coverage", {}) + months = coverage.get("months") + included = coverage.get("included_months") + if months is not None: + summary = {key: value for key, value in coverage.items() if key != "months"} + if included is not None: + del summary["included_months"] + metadata[0] = json.dumps(summary, separators=(",", ":")) + + if any(len(str(value)) > EXCEL_MAX_CELL_LENGTH for value in metadata): + output.close() + raise insufficient_information("XLSX metadata exceeds the Excel cell limit") + workbook = Workbook(write_only=True) sheet = workbook.create_sheet(title=kind.capitalize()) + heading = [] for label in headings: cell = WriteOnlyCell(sheet, value=label) @@ -87,10 +108,28 @@ def build_export(payload, kind, output_format): raise insufficient_information() sheet.append(row) - if metadata is not None: - coverage_sheet = workbook.create_sheet(title="Coverage") - coverage_sheet.append(["Coverage", "Warnings", "Total"]) - coverage_sheet.append(metadata) + if metadata: + metadata_sheet = workbook.create_sheet( + title="Coverage" if "coverage" in payload else "Pagination" + ) + metadata_sheet.append(metadata_headings) + metadata_sheet.append(metadata) + + if months is not None: + month_sheet = workbook.create_sheet(title="Coverage months") + month_sheet.append(["Month", "Coverage", "Included"]) + included_set = set(included or []) + + for month in months: + month_sheet.append( + [ + month["month"], + json.dumps(month, separators=(",", ":")), + month["month"] in included_set + if included is not None + else None, + ] + ) workbook.save(output) diff --git a/counter_api/extensions/openapi.py b/counter_api/extensions/openapi.py index f3740cd..eef6c73 100644 --- a/counter_api/extensions/openapi.py +++ b/counter_api/extensions/openapi.py @@ -15,9 +15,15 @@ class RankingGroupSerializer(serializers.Serializer): class CoverageMonthSerializer(serializers.Serializer): month = serializers.CharField() status = serializers.ChoiceField( - choices=["complete", "partial", "unavailable", "unknown"] + choices=["complete", "partial", "unavailable", "unknown", "ongoing", "future"] ) + requested_days = serializers.ListField(child=serializers.IntegerField()) confirmed_days = serializers.ListField(child=serializers.IntegerField()) + future_days = serializers.ListField(child=serializers.IntegerField()) + ongoing = serializers.BooleanField() + complete_month = serializers.BooleanField( + help_text="The entire month is selected and confirmed; daily cuts are not full months." + ) missing_days = serializers.ListField( child=serializers.IntegerField(), allow_null=True ) @@ -25,7 +31,10 @@ class CoverageMonthSerializer(serializers.Serializer): class CoverageSerializer(serializers.Serializer): policy = serializers.CharField() - status = serializers.ChoiceField(choices=["complete", "partial"]) + status = serializers.ChoiceField( + choices=["complete", "partial", "ongoing", "future"] + ) + ongoing = serializers.BooleanField() basis = serializers.ChoiceField( choices=["confirmed_days", "available_annual_totals"] ) @@ -42,6 +51,14 @@ class CoverageResponseSerializer(serializers.Serializer): warnings = serializers.ListField(child=serializers.CharField(), required=False) +class RankingPaginationSerializer(serializers.Serializer): + group_limit = serializers.IntegerField() + next_cursor = serializers.CharField( + allow_null=True, + help_text="Continue with this cursor even if groups is empty; null ends pagination.", + ) + + class RankingSerializer(CoverageResponseSerializer): platform = serializers.CharField() begin_date = serializers.DateField() @@ -51,6 +68,7 @@ class RankingSerializer(CoverageResponseSerializer): entity_type = serializers.CharField() metric_type = serializers.CharField() groups = RankingGroupSerializer(many=True) + pagination = RankingPaginationSerializer(required=False) class DistributionBucketSerializer(serializers.Serializer): @@ -81,5 +99,11 @@ class CapabilitiesSerializer(serializers.Serializer): distribution_dimensions = serializers.ListField(child=serializers.CharField()) country_language_granularity = serializers.CharField() other_granularity = serializers.CharField() - first_month_available = serializers.CharField(allow_null=True) - last_month_available = serializers.CharField(allow_null=True) + first_month_available = serializers.CharField( + allow_null=True, + help_text="First month with confirmed days, possibly incomplete.", + ) + last_month_available = serializers.CharField( + allow_null=True, + help_text="Last month with confirmed days, including the current month.", + ) diff --git a/counter_api/extensions/pagination.py b/counter_api/extensions/pagination.py new file mode 100644 index 0000000..86244f3 --- /dev/null +++ b/counter_api/extensions/pagination.py @@ -0,0 +1,56 @@ +from django.core import signing + +from counter_api.exceptions import CounterAPIError + +CURSOR_SALT = "counter_api.ranking.groups.v1" + + +def pagination_error(message): + return CounterAPIError(3060, message, 400) + + +def cursor_scope(platform, begin, end, params): + return { + "platform": platform.acron3, + "begin_date": begin.isoformat(), + "end_date": end.isoformat(), + **{ + name: params.get(name) or None + for name in ( + "entity_type", + "parent_type", + "parent_id", + "group_by", + "country", + "language", + "yop", + "subject_area", + ) + }, + "metric_type": params.get("metric_type", "Total_Item_Requests"), + "coverage_policy": params.get("coverage_policy", "allow_partial"), + } + + +def read_cursor(cursor, scope): + if cursor is None: + return None + + try: + payload = signing.loads(cursor, salt=CURSOR_SALT) + except signing.BadSignature: + raise pagination_error("Invalid ranking cursor") + + if not isinstance(payload, dict) or not isinstance(payload.get("after"), dict): + raise pagination_error("Invalid ranking cursor") + if payload.get("scope") != scope: + raise pagination_error("Ranking cursor does not match the query") + + return payload["after"] + + +def next_cursor(after, scope): + if after is None: + return None + + return signing.dumps({"scope": scope, "after": after}, salt=CURSOR_SALT) diff --git a/counter_api/extensions/ranking.py b/counter_api/extensions/ranking.py index beeb762..0e1c8fa 100644 --- a/counter_api/extensions/ranking.py +++ b/counter_api/extensions/ranking.py @@ -5,8 +5,18 @@ from counter_api.constants import METADATA_BATCH_SIZE, METRICS from counter_api.exceptions import insufficient_information -from counter_api.extensions.contracts import RANKING_ENTITIES, RANKING_GROUPS +from counter_api.extensions.contracts import ( + MAX_RESULTS, + RANKING_ENTITIES, + RANKING_GROUPS, +) from counter_api.extensions.coverage import coverage_fields, resolve_coverage +from counter_api.extensions.pagination import ( + cursor_scope, + next_cursor, + pagination_error, + read_cursor, +) from counter_api.extensions.query import ( metric_composite_body, require_complete_period, @@ -22,8 +32,8 @@ RANKING_PAGE_SIZE = 1000 DEFAULT_LIMIT = 100 MAX_LIMIT = 1000 -MAX_GROUPS = 20 -MAX_RESULTS = 10000 +DEFAULT_GROUP_LIMIT = 5 +MAX_GROUP_LIMIT = 20 MAX_FILTER_KEYS = 10000 @@ -45,6 +55,10 @@ def run(self, platform, begin, end, params): segmented, ) = self._parameters(platform, begin, end, params) + group_limit = self._group_limit(params, group_by, limit) + scope = cursor_scope(platform, begin, end, params) + after = read_cursor(params.get("cursor"), scope) + coverage = resolve_coverage( self.client, platform, @@ -54,12 +68,16 @@ def run(self, platform, begin, end, params): segmented, deadline, ) - if coverage is None: + if coverage is None or params.get("coverage_policy") == "strict": require_complete_period(platform, begin, end) parent_key = self._parent_key(platform, parent_type, parent_id) - key_batches = self._key_batches(platform, params, deadline) - groups = self._ranked_groups( + source_keys = ( + self._source_keys(platform, params["subject_area"], deadline) + if params.get("subject_area") + else None + ) + groups, after = self._ranked_groups( platform, begin, end, @@ -68,14 +86,16 @@ def run(self, platform, begin, end, params): parent_key, metric_type, group_by, - key_batches, + source_keys, limit, deadline, segmented, coverage, + group_limit, + after, ) - return { + result = { "platform": platform.acron3, "begin_date": begin.isoformat(), "end_date": end.isoformat(), @@ -86,6 +106,31 @@ def run(self, platform, begin, end, params): "groups": self._result_groups(groups, relation, group_by, deadline), **coverage_fields(coverage), } + if group_by: + result["pagination"] = { + "group_limit": group_limit, + "next_cursor": next_cursor(after, scope), + } + + return result + + def _group_limit(self, params, group_by, limit): + if not group_by: + if "cursor" in params or "group_limit" in params: + raise pagination_error("cursor and group_limit require group_by") + return None + + try: + group_limit = int(params.get("group_limit", DEFAULT_GROUP_LIMIT)) + except (TypeError, ValueError): + raise pagination_error("group_limit must be between 1 and 20") + + if not 1 <= group_limit <= MAX_GROUP_LIMIT: + raise pagination_error("group_limit must be between 1 and 20") + if group_limit * limit > MAX_RESULTS: + raise pagination_error("group_limit * limit must not exceed 10000") + + return group_limit def _parameters(self, platform, begin, end, params): entity_type = params.get("entity_type") @@ -150,18 +195,6 @@ def _parent_key(self, platform, parent_type, parent_id): return self.report_query.resolve_item_id(platform, "tr", parent_id) - def _key_batches(self, platform, params, deadline): - subject_area = params.get("subject_area") - if not subject_area: - return [None] - - source_keys = self._source_keys(platform, subject_area, deadline) - - return [ - source_keys[start : start + METADATA_BATCH_SIZE] - for start in range(0, len(source_keys), METADATA_BATCH_SIZE) - ] - def _ranked_groups( self, platform, @@ -172,48 +205,100 @@ def _ranked_groups( parent_key, metric_type, group_by, - key_batches, + source_keys, limit, deadline, segmented, - coverage=None, + coverage, + group_limit, + after, ): groups = {} + if source_keys == []: + return groups, None - for source_keys in key_batches: - index_name, body = self._query_body( - platform, - begin, - end, + index_name, body = self._query_body( + platform, + begin, + end, + relation["key_field"], + relation.get("data_type"), + parent_key, + metric_type, + group_by, + params, + source_keys, + segmented, + coverage, + ) + + next_after = None + if group_by: + selected, next_after = self._select_groups( + index_name, + body, + group_limit, + after, + deadline, + ) + if not selected: + return groups, next_after + + clauses = [] + for key in selected: + filters = [] + for field, value in key.items(): + if value is None: + clause = {"bool": {"must_not": [{"exists": {"field": field}}]}} + else: + clause = {"term": {field: value}} + filters.append(clause) + clauses.append({"bool": {"filter": filters}}) + + body["query"]["bool"]["filter"].append( + {"bool": {"should": clauses, "minimum_should_match": 1}} + ) + + for buckets in composite_pages(self.client, index_name, body, deadline): + yop_keys = self._yop_keys( + buckets, relation["key_field"], - relation.get("data_type"), - parent_key, - metric_type, - group_by, params, - source_keys, + deadline, segmented, - coverage, + ) + self._collect_groups( + groups, + buckets, + relation["key_field"], + limit, + yop_keys, ) - for buckets in composite_pages(self.client, index_name, body, deadline): - yop_keys = self._yop_keys( - buckets, - relation["key_field"], - params, - deadline, - segmented, - ) + return groups, next_after - self._collect_groups( - groups, - buckets, - relation["key_field"], - limit, - yop_keys, - ) + def _select_groups(self, index_name, body, group_limit, after, deadline): + item_composite = body["aggs"]["rows"]["composite"] + composite = { + "size": group_limit, + "sources": item_composite["sources"][:-1], + } + if after is not None: + composite["after"] = after - return groups + group_body = { + **body, + "query": {"bool": {"filter": list(body["query"]["bool"]["filter"])}}, + "aggs": {"rows": {"composite": composite}}, + } + + page = search(self.client, index_name, group_body, deadline)["aggregations"][ + "rows" + ] + buckets = page.get("buckets", []) + next_after = page.get("after_key") if len(buckets) == group_limit else None + + return [bucket["key"] for bucket in buckets], next_after def _collect_groups(self, groups, buckets, entity_key, limit, yop_keys): for bucket in buckets: @@ -230,9 +315,6 @@ def _collect_groups(self, groups, buckets, entity_key, limit, yop_keys): if dimension in bucket["key"] ) winners = groups.setdefault(group, []) - if len(groups) > MAX_GROUPS or len(groups) * limit > MAX_RESULTS: - raise insufficient_information() - inverted_key = bytes(255 - byte for byte in key.encode("utf-8")) candidate = (count, inverted_key, key) if len(winners) < limit: diff --git a/counter_api/extensions/views.py b/counter_api/extensions/views.py index 1db9366..6c0a2c3 100644 --- a/counter_api/extensions/views.py +++ b/counter_api/extensions/views.py @@ -8,7 +8,6 @@ from counter_api.access import CounterAPIView from counter_api.availability import available_months from counter_api.constants import METRICS -from counter_api.dates import parse_period from counter_api.exceptions import ( CounterAPIError, insufficient_information, @@ -19,6 +18,7 @@ DOWNLOAD_FORMATS, RANKING_ENTITIES, ) +from counter_api.extensions.dates import parse_analytics_period from counter_api.extensions.distribution import DistributionQuery from counter_api.extensions.exports import build_export from counter_api.extensions.openapi import ( @@ -26,27 +26,30 @@ DistributionSerializer, RankingSerializer, ) -from counter_api.extensions.ranking import RankingQuery +from counter_api.extensions.ranking import DEFAULT_GROUP_LIMIT, RankingQuery from counter_api.openapi import ( - BEGIN_DATE_PARAMETER, - END_DATE_PARAMETER, EXCEL_CONTENT_TYPE, PLATFORM_PARAMETER, ExceptionSerializer, ) from counter_api.platforms import get_platform - -def _download_response(payload, kind, platform, output_format): - content = build_export(payload, kind, output_format) - content_type = "text/csv" if output_format == "csv" else EXCEL_CONTENT_TYPE - response = FileResponse(content, content_type=content_type) - response[ - "Content-Disposition" - ] = f'attachment; filename="{kind}_{platform.acron3}.{output_format}"' - - return response - +BEGIN_DATE_PARAMETER = OpenApiParameter( + "begin_date", + OpenApiTypes.STR, + required=True, + description="First day (YYYY-MM-DD) or month (YYYY-MM). Country/language must start in January.", +) +END_DATE_PARAMETER = OpenApiParameter( + "end_date", + OpenApiTypes.STR, + required=True, + description=( + "Last day (YYYY-MM-DD) or month (YYYY-MM). The current month ends today; " + "explicit future days are rejected. Country/language must end in December " + "and may select the current year for provisional annual totals." + ), +) COVERAGE_PARAMETER = OpenApiParameter( "coverage_policy", @@ -60,6 +63,17 @@ def _download_response(payload, kind, platform, output_format): ) +def _download_response(payload, kind, platform, output_format): + content = build_export(payload, kind, output_format) + content_type = "text/csv" if output_format == "csv" else EXCEL_CONTENT_TYPE + response = FileResponse(content, content_type=content_type) + response[ + "Content-Disposition" + ] = f'attachment; filename="{kind}_{platform.acron3}.{output_format}"' + + return response + + class RankingView(CounterAPIView): @extend_schema( tags=["SciELO extensions"], @@ -80,7 +94,29 @@ class RankingView(CounterAPIView): OpenApiParameter("language", OpenApiTypes.STR, required=False), OpenApiParameter("yop", OpenApiTypes.STR, required=False), OpenApiParameter("subject_area", OpenApiTypes.STR, required=False), - OpenApiParameter("limit", OpenApiTypes.INT, required=False), + OpenApiParameter( + "limit", + OpenApiTypes.INT, + required=False, + default=100, + description="Items per group (1–1000), ranked after aggregating the full period.", + ), + OpenApiParameter( + "group_limit", + OpenApiTypes.INT, + required=False, + default=DEFAULT_GROUP_LIMIT, + description="Groups per page (1–20), only with group_by. group_limit × limit must not exceed 10000.", + ), + OpenApiParameter( + "cursor", + OpenApiTypes.STR, + required=False, + description=( + "Opaque next_cursor from a grouped ranking, with the same filters and period. " + "Follow it even if a page has no visible groups. Limits and output format may change." + ), + ), OpenApiParameter("format", OpenApiTypes.STR, required=False), ], responses={ @@ -98,7 +134,7 @@ def get(self, request): raise insufficient_information() platform = get_platform(request.query_params.get("platform")) - begin, end = parse_period(request.query_params) + begin, end = parse_analytics_period(request.query_params) payload = RankingQuery().run(platform, begin, end, request.query_params) if output_format != "json": @@ -130,7 +166,7 @@ class CapabilitiesView(CounterAPIView): def get(self, request): try: platform = get_platform(request.query_params.get("platform")) - months = available_months(platform) + months = available_months(platform, complete_only=False) except CounterAPIError as error: return Response(error.as_dict(), status=error.status_code) @@ -148,7 +184,7 @@ def get(self, request): "download_formats": list(DOWNLOAD_FORMATS), "distribution_dimensions": list(DISTRIBUTION_CAPABILITIES), "country_language_granularity": "year", - "other_granularity": "month", + "other_granularity": "day", "first_month_available": ( months[0].strftime("%Y-%m") if months else None ), @@ -191,7 +227,7 @@ def get(self, request): raise insufficient_information() platform = get_platform(request.query_params.get("platform")) - begin, end = parse_period(request.query_params) + begin, end = parse_analytics_period(request.query_params) payload = DistributionQuery().run( platform, begin, end, request.query_params ) diff --git a/counter_api/painless.py b/counter_api/painless.py index d715625..5fdadda 100644 --- a/counter_api/painless.py +++ b/counter_api/painless.py @@ -1,6 +1,6 @@ # Sparse daily maps omit days with no usage, but the map itself must exist. CONFIRMED_DAYS_SUM_SCRIPT = """ -String month = params._source.month; +String month = doc['month'].value.toString().substring(0,7); if (params.complete.contains(month)) return doc[params.metric].value; def daily = params._source.daily_metrics; diff --git a/counter_api/tests/test_availability.py b/counter_api/tests/test_availability.py index 90ea0f9..cfca990 100644 --- a/counter_api/tests/test_availability.py +++ b/counter_api/tests/test_availability.py @@ -28,6 +28,24 @@ def test_reads_only_complete_months_from_status_index(self, client_class): self.assertIn({"term": {"collection": "scl"}}, filters) self.assertIn({"term": {"complete": True}}, filters) + @patch("counter_api.availability.OpenSearchUsageClient") + def test_analytics_discovery_includes_confirmed_current_month(self, client_class): + client = client_class.return_value.client + current = date.today().replace(day=1) + client.search.return_value = { + "aggregations": { + "months": {"buckets": [{"key": {"month": current.strftime("%Y-%m")}}]} + } + } + + months = available_months(Mock(acron3="scl"), complete_only=False) + + self.assertEqual(months, [current]) + filters = client.search.call_args.kwargs["body"]["query"]["bool"]["filter"] + self.assertIn({"range": {"day_mask": {"gt": 0}}}, filters) + self.assertIn({"range": {"month": {"lte": current.strftime("%Y-%m")}}}, filters) + self.assertNotIn({"term": {"complete": True}}, filters) + @patch("counter_api.availability.available_months") def test_splits_missing_months_without_treating_gaps_as_no_usage(self, months): months.return_value = [date(2024, 1, 1), date(2024, 3, 1)] diff --git a/counter_api/tests/test_dates.py b/counter_api/tests/test_dates.py index 46510a7..92ad7d8 100644 --- a/counter_api/tests/test_dates.py +++ b/counter_api/tests/test_dates.py @@ -1,5 +1,7 @@ from datetime import date +from unittest.mock import patch +import pytest from django.test import SimpleTestCase from counter_api.dates import parse_period @@ -20,3 +22,18 @@ def test_rejects_current_month(self): parse_period({"begin_date": "2024-01", "end_date": current_month}) self.assertEqual(raised.exception.code, 3020) + + +@pytest.mark.parametrize( + "begin,end", + [ + ("2024-02-02", "2024-02-29"), + ("2024-02-01", "2024-02-28"), + ], +) +def test_official_report_still_requires_whole_past_months(begin, end): + with patch("counter_api.dates.date", wraps=date) as clock: + clock.today.return_value = date(2024, 3, 15) + with pytest.raises(CounterAPIError) as raised: + parse_period({"begin_date": begin, "end_date": end}) + assert raised.value.code == 3020 diff --git a/counter_api/tests/test_extension_coverage.py b/counter_api/tests/test_extension_coverage.py index d8f3cb1..1c3fb3d 100644 --- a/counter_api/tests/test_extension_coverage.py +++ b/counter_api/tests/test_extension_coverage.py @@ -1,16 +1,17 @@ import csv import json +from calendar import monthrange from datetime import date from io import StringIO from time import monotonic from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest from openpyxl import load_workbook from counter_api.exceptions import CounterAPIError -from counter_api.extensions.coverage import resolve_coverage +from counter_api.extensions.coverage import metric_sum, resolve_coverage from counter_api.extensions.distribution import DistributionQuery from counter_api.extensions.exports import build_export from counter_api.extensions.ranking import RankingQuery @@ -135,6 +136,9 @@ def test_default_queries_share_confirmed_sum_across_pages(kind): "days": {"2023-01": ["01"]}, } assert "Missing daily_metrics" in script["source"] + assert script["source"].strip().splitlines()[0] == ( + "String month = doc['month'].value.toString().substring(0,7);" + ) @pytest.mark.parametrize("response", [markers(), markers(**{"01": 1})]) @@ -195,3 +199,113 @@ def test_empty_export_preserves_coverage_metadata(output_format): assert json.loads(metadata["Coverage"]) == payload["coverage"] assert json.loads(metadata["Warnings"]) == payload["warnings"] assert int(metadata["Total"]) == 0 + + +def daily_coverage(begin, end, month_masks, segmented=False): + client = Mock() + client.search.return_value = { + "hits": { + "hits": [ + { + "_source": { + "collection": "scl", + "month": month, + "day_mask": mask, + "complete": mask + == (1 << monthrange(*map(int, month.split("-")))[1]) - 1, + } + } + for month, mask in month_masks.items() + ] + } + } + + with patch("counter_api.extensions.coverage.date", wraps=date) as clock: + clock.today.return_value = date(2024, 3, 15) + coverage = resolve_coverage( + client, PLATFORM, begin, end, "allow_partial", segmented, monotonic() + 30 + ) + + client.search.assert_called_once() + return coverage + + +def test_confirmed_cut_is_complete_without_using_full_month_totals(): + coverage = daily_coverage( + date(2024, 2, 28), date(2024, 2, 29), {"2024-02": (1 << 27) | (1 << 28)} + ) + + month = coverage["months"][0] + params = metric_sum("total_requests", coverage)["script"]["params"] + + assert coverage["status"] == "complete" + assert month["requested_days"] == month["confirmed_days"] == [28, 29] + assert month["missing_days"] == [] + assert not month["complete_month"] + assert params == { + "metric": "total_requests", + "complete": [], + "days": {"2024-02": ["28", "29"]}, + } + + +def test_month_edges_with_complete_middle_month(): + coverage = daily_coverage( + date(2024, 1, 31), + date(2024, 3, 2), + {"2024-01": (1 << 31) - 1, "2024-02": (1 << 29) - 1, "2024-03": 3}, + ) + + params = metric_sum("total_requests", coverage)["script"]["params"] + + assert params["complete"] == ["2024-02"] + assert params["days"] == {"2024-01": ["31"], "2024-03": ["01", "02"]} + + +def test_full_selected_month_uses_field(): + coverage = daily_coverage( + date(2024, 2, 1), date(2024, 2, 29), {"2024-02": (1 << 29) - 1} + ) + + assert metric_sum("total_requests", coverage) == {"field": "total_requests"} + + +def test_middle_gap_is_not_summed(): + coverage = daily_coverage(date(2024, 2, 1), date(2024, 2, 3), {"2024-02": 5}) + + assert coverage["months"][0]["missing_days"] == [2] + assert metric_sum("total_requests", coverage)["script"]["params"]["days"] == { + "2024-02": ["01", "03"] + } + + +def test_current_annual_coverage_separates_future_from_missing(): + coverage = daily_coverage( + date(2024, 1, 1), + date(2024, 12, 31), + {"2024-01": (1 << 31) - 1, "2024-02": (1 << 29) - 1, "2024-03": (1 << 15) - 1}, + segmented=True, + ) + + assert coverage["basis"] == "available_annual_totals" + assert coverage["status"] == "ongoing" + assert coverage["ongoing"] + march, april = coverage["months"][2:4] + assert march["missing_days"] == [] + assert march["future_days"] == list(range(16, 32)) + assert april["status"] == "future" + assert april["missing_days"] == april["confirmed_days"] == [] + assert april["future_days"] == april["requested_days"] == list(range(1, 31)) + assert metric_sum("total_requests", coverage) == {"field": "total_requests"} + + +def test_cut_without_confirmed_requested_days_fails(): + with pytest.raises(CounterAPIError) as raised: + daily_coverage(date(2024, 2, 2), date(2024, 2, 3), {"2024-02": 1}) + + assert raised.value.status_code == 503 + + +def test_cut_still_validates_full_month_marker(): + with pytest.raises(CounterAPIError): + daily_coverage(date(2024, 2, 1), date(2024, 2, 2), {"2024-02": 1 << 29}) diff --git a/counter_api/tests/test_extension_dates.py b/counter_api/tests/test_extension_dates.py new file mode 100644 index 0000000..028ab96 --- /dev/null +++ b/counter_api/tests/test_extension_dates.py @@ -0,0 +1,89 @@ +from datetime import date +from unittest.mock import patch + +import pytest + +from counter_api.exceptions import CounterAPIError +from counter_api.extensions.dates import parse_analytics_period + + +@pytest.fixture(autouse=True) +def today(): + with patch("counter_api.extensions.dates.date", wraps=date) as clock: + clock.today.return_value = date(2024, 3, 15) + yield + + +@pytest.mark.parametrize( + "begin,end,expected", + [ + ("2024-02-29", "2024-03-02", (date(2024, 2, 29), date(2024, 3, 2))), + ("2024-03-12", "2024-03", (date(2024, 3, 12), date(2024, 3, 15))), + ], +) +def test_monthly_analytics_dates(begin, end, expected): + assert parse_analytics_period({"begin_date": begin, "end_date": end}) == expected + + +@pytest.mark.parametrize( + "begin,end", + [ + ("2024-03-01", "2024-03-16"), + ("2024-04", "2024-04"), + ("2024-03-16", "2024-03"), + ], +) +def test_invalid_monthly_dates(begin, end): + with pytest.raises(CounterAPIError): + parse_analytics_period({"begin_date": begin, "end_date": end}) + + +@pytest.mark.parametrize( + "segmentation", + [ + {"group_by": "country,language"}, + {"dimension": "language,yop"}, + {"country": "BR"}, + ], +) +def test_current_year_annual_dates(segmentation): + params = {"begin_date": "2024-01", "end_date": "2024-12", **segmentation} + + assert parse_analytics_period(params) == (date(2024, 1, 1), date(2024, 12, 31)) + + +@pytest.mark.parametrize( + "begin,end", + [ + ("2024-01-02", "2024-12-31"), + ("2024-01-01", "2024-12-30"), + ("2024-02", "2024-12"), + ("2024-01", "2025-12"), + ], +) +def test_annual_requires_whole_calendar_years(begin, end): + with pytest.raises(CounterAPIError): + parse_analytics_period({"begin_date": begin, "end_date": end, "country": "BR"}) + + +def test_whitespace_does_not_bypass_annual_boundaries(): + with pytest.raises(CounterAPIError): + parse_analytics_period( + { + "begin_date": "2023-01-02", + "end_date": "2023-12-30", + "dimension": " country , language ", + } + ) + + +def test_annual_preserves_unique_metric_restriction(): + with pytest.raises(CounterAPIError): + parse_analytics_period( + { + "begin_date": "2023-01", + "end_date": "2024-12", + "dimension": "country", + "metric_type": "Unique_Item_Requests", + } + ) diff --git a/counter_api/tests/test_extension_views.py b/counter_api/tests/test_extension_views.py index e2bdbae..271ae20 100644 --- a/counter_api/tests/test_extension_views.py +++ b/counter_api/tests/test_extension_views.py @@ -40,6 +40,8 @@ def test_extension_capabilities_show_journal_relations(self, available_months): ], ) self.assertNotIn("coverage", response.json()) + self.assertEqual(response.json()["other_granularity"], "day") + available_months.assert_called_once_with(self.platform, complete_only=False) @patch("counter_api.extensions.distribution.ReportQuery") def test_default_http_request_includes_partial_coverage(self, report_query): @@ -193,16 +195,54 @@ def test_public_extensions_do_not_turn_failures_into_empty_success( @patch("counter_api.extensions.views.DistributionQuery") @patch("counter_api.extensions.views.RankingQuery") - def test_public_extensions_still_reject_current_and_future_months( + def test_public_extensions_reject_future_months( self, ranking_class, distribution_class ): - for month in (date.today().strftime("%Y-%m"), "9999-12"): - for kind in ("ranking", "distribution"): - with self.subTest(kind=kind, month=month): - response = self.client.get( - f"/r51/extensions/{kind}", - {"platform": "scl", "begin_date": month, "end_date": month}, - ) - self.assertEqual(response.status_code, 400) + for kind in ("ranking", "distribution"): + with self.subTest(kind=kind): + response = self.client.get( + f"/r51/extensions/{kind}", + {"platform": "scl", "begin_date": "9999-12", "end_date": "9999-12"}, + ) + + self.assertEqual(response.status_code, 400) + ranking_class.return_value.run.assert_not_called() distribution_class.return_value.run.assert_not_called() + + @patch("counter_api.extensions.dates.date", wraps=date) + @patch("counter_api.extensions.views.DistributionQuery") + @patch("counter_api.extensions.views.RankingQuery") + def test_recent_days_and_current_annual_totals_reach_queries( + self, ranking_class, distribution_class, clock + ): + clock.today.return_value = date(2026, 11, 8) + + for kind, query_class in ( + ("ranking", ranking_class), + ("distribution", distribution_class), + ): + query_class.return_value.run.return_value = {} + for params, expected in ( + ( + {"begin_date": "2026-11", "end_date": "2026-11"}, + (date(2026, 11, 1), date(2026, 11, 8)), + ), + ( + { + "begin_date": "2026-01", + "end_date": "2026-12", + "group_by" if kind == "ranking" else "dimension": "country", + }, + (date(2026, 1, 1), date(2026, 12, 31)), + ), + ): + with self.subTest(kind=kind, params=params): + response = self.client.get( + f"/r51/extensions/{kind}", {"platform": "scl", **params} + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual( + query_class.return_value.run.call_args.args[1:3], expected + ) diff --git a/counter_api/tests/test_extensions_distribution.py b/counter_api/tests/test_extensions_distribution.py index 9471c2c..0329e70 100644 --- a/counter_api/tests/test_extensions_distribution.py +++ b/counter_api/tests/test_extensions_distribution.py @@ -3,7 +3,7 @@ from unittest import TestCase from unittest.mock import Mock, patch -from counter_api.exceptions import CounterAPIError +from counter_api.exceptions import CounterAPIError, service_unavailable from counter_api.extensions.distribution import DistributionQuery @@ -15,6 +15,69 @@ def setUp(self): self.period = period.start() self.addCleanup(period.stop) + def _distribution_with_groups(self, count): + client = Mock() + client.search.return_value = { + "aggregations": { + "rows": { + "buckets": [ + {"key": {"country": f"C{number}"}, "count": {"value": 1}} + for number in range(count) + ] + } + } + } + return DistributionQuery(Mock(client=client)).run( + SimpleNamespace(acron3="scl", collection_type="journals"), + date(2024, 1, 1), + date(2024, 12, 31), + { + "entity_type": "collection", + "entity_id": "scl", + "dimension": "country", + "coverage_policy": "strict", + }, + ) + + def test_distribution_allows_more_than_1000_groups(self): + payload = self._distribution_with_groups(1001) + + self.assertEqual(len(payload["buckets"]), 1001) + self.assertEqual(payload["total"], 1001) + + @patch("counter_api.extensions.distribution.MAX_RESULTS", 2) + def test_distribution_limit_is_explicit_and_not_truncated(self): + with self.assertRaises(CounterAPIError) as error: + self._distribution_with_groups(3) + + self.assertEqual(error.exception.status_code, 400) + self.assertIn("exceeds 2 groups", error.exception.data) + + def test_strict_daily_cut_requires_complete_calendar_months(self): + with ( + patch( + "counter_api.extensions.distribution.resolve_coverage", return_value={} + ), + patch( + "counter_api.extensions.distribution.require_complete_period", + side_effect=service_unavailable(), + ) as require_complete, + ): + with self.assertRaises(CounterAPIError): + DistributionQuery(Mock(client=Mock())).run( + SimpleNamespace(acron3="scl", collection_type="journals"), + date(2024, 1, 2), + date(2024, 1, 6), + { + "entity_type": "collection", + "entity_id": "scl", + "dimension": "yop", + "coverage_policy": "strict", + }, + ) + + require_complete.assert_called_once() + def test_incomplete_month_does_not_query_metrics(self): client = Mock() query = DistributionQuery(Mock(client=client)) diff --git a/counter_api/tests/test_extensions_exports.py b/counter_api/tests/test_extensions_exports.py index 980cc52..4ada916 100644 --- a/counter_api/tests/test_extensions_exports.py +++ b/counter_api/tests/test_extensions_exports.py @@ -1,28 +1,110 @@ -from io import BytesIO +import csv +import json +from calendar import monthrange +from io import BytesIO, StringIO from unittest import TestCase from openpyxl import load_workbook +from counter_api.exceptions import CounterAPIError from counter_api.extensions.exports import build_export class ExtensionExportTests(TestCase): - def test_ranking_csv_and_xlsx_keep_group_and_rank(self): + def test_ranking_csv_and_xlsx_preserve_cursor_with_items(self): payload = { "groups": [ { "dimensions": {"country": "BR", "language": "pt"}, "items": [{"id": "article-1", "title": "Artigo", "count": 12}], } - ] + ], + "pagination": {"group_limit": 5, "next_cursor": "signed-next-page"}, } - csv_file = build_export(payload, "ranking", "csv") - lines = csv_file.read().decode("utf-8").splitlines() - xlsx_file = build_export(payload, "ranking", "xlsx") - sheet = load_workbook(BytesIO(xlsx_file.read()), read_only=True).active + with build_export(payload, "ranking", "csv") as output: + rows = list(csv.DictReader(StringIO(output.read().decode()))) + with build_export(payload, "ranking", "xlsx") as output: + workbook = load_workbook(BytesIO(output.read()), read_only=True) - self.assertEqual(lines[0], "Country,Language,Rank,ID,Title,Count") - self.assertEqual(lines[1], "BR,pt,1,article-1,Artigo,12") - self.assertEqual(sheet["D2"].value, "article-1") - self.assertEqual(sheet["F2"].value, 12) + self.assertEqual(json.loads(rows[0]["Pagination"]), payload["pagination"]) + self.assertEqual( + json.loads(workbook["Pagination"]["A2"].value), payload["pagination"] + ) + workbook.close() + + def test_empty_page_preserves_coverage_and_cursor_in_downloads(self): + payload = { + "groups": [], + "coverage": {"status": "partial"}, + "warnings": ["Missing days"], + "pagination": {"group_limit": 5, "next_cursor": "signed-next-page"}, + } + + for output_format in ("csv", "xlsx"): + with self.subTest(format=output_format): + with build_export(payload, "ranking", output_format) as output: + if output_format == "csv": + metadata = next( + csv.DictReader(StringIO(output.read().decode())) + ) + self.assertEqual(metadata["ID"], "") + else: + workbook = load_workbook(output, read_only=True) + headings, values = list(workbook["Coverage"].values) + metadata = dict(zip(headings, values)) + workbook.close() + + self.assertEqual(json.loads(metadata["Coverage"]), payload["coverage"]) + self.assertEqual(json.loads(metadata["Warnings"]), payload["warnings"]) + self.assertEqual( + json.loads(metadata["Pagination"]), payload["pagination"] + ) + + def test_xlsx_long_coverage_round_trips_without_cell_truncation(self): + months = [ + { + "month": f"{year}-{month:02}", + "status": "complete", + "requested_days": list(range(1, monthrange(year, month)[1] + 1)), + "confirmed_days": list(range(1, monthrange(year, month)[1] + 1)), + "missing_days": [], + "future_days": [], + "ongoing": False, + "complete_month": True, + } + for year in range(2014, 2024) + for month in range(1, 13) + ] + coverage = { + "policy": "allow_partial", + "status": "complete", + "ongoing": False, + "basis": "confirmed_days", + "months": months, + "included_months": [month["month"] for month in months], + } + payload = {"groups": [], "coverage": coverage, "warnings": []} + self.assertGreater(len(json.dumps(coverage, separators=(",", ":"))), 32767) + + with build_export(payload, "ranking", "xlsx") as output: + workbook = load_workbook(output, read_only=True) + _, summary = list(workbook["Coverage"].values) + monthly_rows = list(workbook["Coverage months"].values)[1:] + restored = json.loads(summary[0]) + restored["months"] = [json.loads(row[1]) for row in monthly_rows] + restored["included_months"] = [row[0] for row in monthly_rows if row[2]] + workbook.close() + + self.assertEqual(restored, coverage) + + def test_xlsx_rejects_oversized_cursor_instead_of_truncating(self): + payload = { + "groups": [], + "pagination": {"group_limit": 5, "next_cursor": "x" * 40000}, + } + + with self.assertRaises(CounterAPIError) as error: + build_export(payload, "ranking", "xlsx") + + self.assertIn("Excel cell limit", error.exception.data) diff --git a/counter_api/tests/test_extensions_ranking.py b/counter_api/tests/test_extensions_ranking.py index 9aa27d5..afa7488 100644 --- a/counter_api/tests/test_extensions_ranking.py +++ b/counter_api/tests/test_extensions_ranking.py @@ -39,8 +39,19 @@ def test_incomplete_annual_period_does_not_query_metrics(self): client.search.assert_not_called() def test_ranking_reads_all_pages_and_breaks_ties_by_item_key(self): + after = {"country_code": "CA", "content_language": "en"} client = Mock() client.search.side_effect = [ + { + "aggregations": { + "rows": { + "buckets": [ + {"key": {"country_code": "BR", "content_language": "pt"}} + ], + "after_key": after, + } + } + }, { "aggregations": { "rows": { @@ -84,18 +95,21 @@ def test_ranking_reads_all_pages_and_breaks_ties_by_item_key(self): platform = SimpleNamespace(acron3="scl", collection_type="journals") query.report_query.resolve_item_id.return_value = "journal-key" + params = { + "entity_type": "articles", + "parent_type": "journal", + "parent_id": "Print_ISSN:1234-5678", + "group_by": "country,language", + "coverage_policy": "strict", + "group_limit": 1, + "limit": 1, + } + result = query.run( platform, date(2025, 1, 1), date(2025, 12, 31), - { - "entity_type": "articles", - "parent_type": "journal", - "parent_id": "Print_ISSN:1234-5678", - "group_by": "country,language", - "coverage_policy": "strict", - "limit": "1", - }, + params, ) self.assertEqual( @@ -105,13 +119,45 @@ def test_ranking_reads_all_pages_and_breaks_ties_by_item_key(self): result["groups"][0]["items"], [{"id": "item-a", "title": "Article A", "count": 10}], ) - self.assertEqual(client.search.call_count, 2) + self.assertEqual(client.search.call_count, 3) body = client.search.call_args_list[0].kwargs["body"] + self.assertEqual(body["aggs"]["rows"]["composite"]["size"], 1) + self.assertEqual( + [ + next(iter(source)) + for source in body["aggs"]["rows"]["composite"]["sources"] + ], + ["country_code", "content_language"], + ) self.assertIn({"terms": {"year": ["2025"]}}, body["query"]["bool"]["filter"]) self.assertIn( {"term": {"source_key": "journal-key"}}, body["query"]["bool"]["filter"] ) + client.search.side_effect = [{"aggregations": {"rows": {"buckets": []}}}] + + final = query.run( + platform, + date(2025, 1, 1), + date(2025, 12, 31), + { + **params, + "cursor": result["pagination"]["next_cursor"], + "group_limit": 2, + "limit": 7, + "format": "csv", + }, + ) + + self.assertEqual( + client.search.call_args.kwargs["body"]["aggs"]["rows"]["composite"][ + "after" + ], + after, + ) + self.assertEqual(final["pagination"], {"group_limit": 2, "next_cursor": None}) + self.assertEqual(final["groups"], []) + def test_country_ranking_requires_complete_year(self): query = RankingQuery(Mock(client=Mock())) platform = SimpleNamespace(acron3="scl", collection_type="journals") @@ -196,7 +242,7 @@ def test_subject_area_keys_and_yop_are_filtered_before_aggregation(self): def test_country_ranking_filters_yop_with_document_metadata(self): client = Mock() - client.search.return_value = { + item_page = { "aggregations": { "rows": { "buckets": [ @@ -226,6 +272,10 @@ def test_country_ranking_filters_yop_with_document_metadata(self): }, ] } + client.search.side_effect = [ + {"aggregations": {"rows": {"buckets": [{"key": {"country_code": "BR"}}]}}}, + item_page, + ] query = RankingQuery(Mock(client=client)) platform = SimpleNamespace(acron3="scl", collection_type="journals") diff --git a/counter_api/tests/test_ranking_pagination.py b/counter_api/tests/test_ranking_pagination.py new file mode 100644 index 0000000..b0533a8 --- /dev/null +++ b/counter_api/tests/test_ranking_pagination.py @@ -0,0 +1,245 @@ +from datetime import date +from types import SimpleNamespace +from unittest import TestCase +from unittest.mock import Mock, patch + +from counter_api.exceptions import CounterAPIError, service_unavailable +from counter_api.extensions.ranking import RankingQuery + + +def page(keys, after=None): + rows = {"buckets": [{"key": key} for key in keys]} + if after is not None: + rows["after_key"] = after + return {"aggregations": {"rows": rows}} + + +def items(rows, after=None): + result = page([key for key, count in rows], after) + for bucket, (_, count) in zip(result["aggregations"]["rows"]["buckets"], rows): + bucket["count"] = {"value": count} + return result + + +class RankingPaginationTests(TestCase): + def setUp(self): + for target in ("resolve_coverage", "require_complete_period"): + patcher = patch( + "counter_api.extensions.ranking." + target, return_value=None + ) + patcher.start() + self.addCleanup(patcher.stop) + self.client = Mock() + self.client.mget.return_value = {"docs": []} + self.query = RankingQuery(Mock(client=self.client)) + self.query.report_query.resolve_item_id.return_value = "parent-key" + self.platform = SimpleNamespace(acron3="scl", collection_type="journals") + self.params = { + "entity_type": "articles", + "parent_type": "journal", + "parent_id": "journal", + "group_by": "country", + "coverage_policy": "strict", + } + + def run_query(self, **params): + return self.query.run( + self.platform, + date(2024, 1, 1), + date(2024, 12, 31), + {**self.params, **params}, + ) + + def body(self, call): + return self.client.search.call_args_list[call].kwargs["body"] + + def test_exact_tuple_filter_includes_missing_values(self): + keys = [ + {"country_code": "BR", "content_language": "pt"}, + {"country_code": None, "content_language": "en"}, + ] + self.client.search.side_effect = [ + page(keys), + items([({**keys[1], "document_key": "a"}, 3)]), + ] + + result = self.run_query(group_by="country,language") + + selection = self.body(1)["query"]["bool"]["filter"][-1]["bool"] + clauses = [clause["bool"]["filter"] for clause in selection["should"]] + self.assertEqual(selection["minimum_should_match"], 1) + self.assertEqual( + clauses, + [ + [ + {"term": {"country_code": "BR"}}, + {"term": {"content_language": "pt"}}, + ], + [ + {"bool": {"must_not": [{"exists": {"field": "country_code"}}]}}, + {"term": {"content_language": "en"}}, + ], + ], + ) + self.assertEqual( + result["groups"][0]["dimensions"], {"country": "Unknown", "language": "en"} + ) + self.assertIsNone(result["pagination"]["next_cursor"]) + + def test_zero_count_page_still_advances_and_short_page_finishes(self): + self.client.search.side_effect = [ + page([{"country_code": "BR"}], {"country_code": "BR"}), + items([({"country_code": "BR", "document_key": "a"}, 0)]), + ] + + result = self.run_query(group_limit=1) + + self.assertEqual(result["groups"], []) + self.assertIsNotNone(result["pagination"]["next_cursor"]) + + self.client.search.side_effect = [ + page([{"country_code": "US"}], {"country_code": "US"}), + items([]), + ] + + result = self.run_query( + cursor=result["pagination"]["next_cursor"], group_limit=2 + ) + + self.assertIsNone(result["pagination"]["next_cursor"]) + + def test_subject_keys_are_one_filter_and_all_filters_apply_to_both_queries(self): + keys = ["source-" + str(i) for i in range(2500)] + with patch.object(self.query, "_source_keys", return_value=keys): + self.client.search.side_effect = [ + page([{"country_code": "BR"}]), + items([]), + ] + + self.run_query( + subject_area="Health", country="BR", language="pt", yop="2020" + ) + + self.assertEqual(self.client.search.call_count, 2) + for call in (0, 1): + filters = self.body(call)["query"]["bool"]["filter"] + for clause in ( + {"terms": {"source_key": keys}}, + {"term": {"source_key": "parent-key"}}, + {"term": {"country_code": "BR"}}, + {"term": {"content_language": "pt"}}, + {"terms": {"year": ["2024"]}}, + ): + self.assertIn(clause, filters) + self.assertNotIn("publication_year", str(filters)) + + def test_yop_filtered_page_keeps_cursor_and_metadata_filter(self): + self.client.search.side_effect = [ + page([{"country_code": "BR"}], {"country_code": "BR"}), + items([({"country_code": "BR", "document_key": "old"}, 8)]), + ] + self.client.mget.return_value = { + "docs": [ + {"_id": "old", "found": True, "_source": {"publication_year": 2019}}, + ] + } + + result = self.run_query(yop="2020", group_limit=1) + + self.assertEqual(result["groups"], []) + self.assertIsNotNone(result["pagination"]["next_cursor"]) + + def test_invalid_cursor_and_scope_changes_fail_before_search(self): + self.client.search.side_effect = [ + page([{"country_code": "BR"}], {"country_code": "BR"}), + items([]), + ] + + cursor = self.run_query(group_limit=1)["pagination"]["next_cursor"] + self.client.reset_mock() + + changes = [ + {"country": "US"}, + {"language": "en"}, + {"yop": "2020"}, + {"subject_area": "Health"}, + {"parent_id": "other"}, + {"group_by": "language"}, + {"metric_type": "Total_Item_Investigations"}, + {"coverage_policy": "allow_partial"}, + ] + for change in changes: + with self.subTest(change=change), self.assertRaisesRegex( + CounterAPIError, "does not match" + ): + self.run_query(cursor=cursor, **change) + + for invalid in ("", "garbage", cursor + "tampered"): + with self.subTest(cursor=invalid), self.assertRaisesRegex( + CounterAPIError, "Invalid ranking cursor" + ): + self.run_query(cursor=invalid) + + with self.assertRaisesRegex(CounterAPIError, "does not match"): + self.query.run( + self.platform, + date(2023, 1, 1), + date(2023, 12, 31), + {**self.params, "cursor": cursor}, + ) + + self.platform.acron3 = "other" + with self.assertRaisesRegex(CounterAPIError, "does not match"): + self.run_query(cursor=cursor) + self.client.search.assert_not_called() + + def test_limits_and_ungrouped_contract(self): + for params, message in ( + ({"group_limit": 0}, "between 1 and 20"), + ({"group_limit": 21}, "between 1 and 20"), + ({"group_limit": "x"}, "between 1 and 20"), + ({"group_limit": 20, "limit": 501}, "must not exceed 10000"), + ({"group_by": "", "group_limit": 1}, "require group_by"), + ({"group_by": "", "cursor": "x"}, "require group_by"), + ): + with self.subTest(params=params), self.assertRaisesRegex( + CounterAPIError, message + ): + self.run_query(**params) + + self.client.search.assert_not_called() + self.client.search.return_value = items([]) + + result = self.run_query(group_by="") + + self.assertNotIn("pagination", result) + + self.client.search.return_value = page([]) + + result = self.run_query(group_limit=20, limit=500) + + self.assertEqual(result["pagination"], {"group_limit": 20, "next_cursor": None}) + + def test_strict_daily_coverage_still_requires_complete_period(self): + with ( + patch( + "counter_api.extensions.ranking.resolve_coverage", + return_value={}, + ), + patch( + "counter_api.extensions.ranking.require_complete_period", + side_effect=service_unavailable(), + ) as complete, + ): + with self.assertRaises(CounterAPIError): + self.query.run( + self.platform, + date(2024, 1, 2), + date(2024, 1, 6), + {**self.params, "group_by": ""}, + ) + + complete.assert_called_once_with( + self.platform, date(2024, 1, 2), date(2024, 1, 6) + ) + self.client.search.assert_not_called()