From f7483e7aa6f9982d644b7cfa6813dfd71dd8c685 Mon Sep 17 00:00:00 2001 From: Rafael JP Damaceno Date: Sun, 8 Nov 2026 19:42:01 -0300 Subject: [PATCH 1/9] Isola a soma Painless dos dias confirmados MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Centraliza o script sem alterar sua lógica para separar a agregação da resolução de cobertura. --- counter_api/painless.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 counter_api/painless.py diff --git a/counter_api/painless.py b/counter_api/painless.py new file mode 100644 index 0000000..d715625 --- /dev/null +++ b/counter_api/painless.py @@ -0,0 +1,16 @@ +# Sparse daily maps omit days with no usage, but the map itself must exist. +CONFIRMED_DAYS_SUM_SCRIPT = """ +String month = params._source.month; +if (params.complete.contains(month)) return doc[params.metric].value; + +def daily = params._source.daily_metrics; +if (daily == null) throw new IllegalStateException('Missing daily_metrics'); + +long total = 0; +for (def day : params.days[month]) { + def values = daily[day]; + if (values != null) total += values[params.metric]; +} + +return total; +""" From 37740ffbe51d80d7b0cdecf4a5cd3a2a16cba022 Mon Sep 17 00:00:00 2001 From: Rafael JP Damaceno Date: Sun, 8 Nov 2026 19:42:01 -0300 Subject: [PATCH 2/9] Resolve a cobertura por dias confirmados MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distingue lacunas de uso zero e testa marcadores inválidos antes de integrar as consultas. --- counter_api/extensions/coverage.py | 111 +++++++++++++++++++ counter_api/tests/test_extension_coverage.py | 85 ++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 counter_api/extensions/coverage.py create mode 100644 counter_api/tests/test_extension_coverage.py diff --git a/counter_api/extensions/coverage.py b/counter_api/extensions/coverage.py new file mode 100644 index 0000000..f087625 --- /dev/null +++ b/counter_api/extensions/coverage.py @@ -0,0 +1,111 @@ +from calendar import monthrange + +from django.conf import settings +from opensearchpy import NotFoundError + +from counter_api.dates import iter_months +from counter_api.exceptions import insufficient_information, service_unavailable +from counter_api.search import search +from metrics.opensearch.names import generate_month_status_index_name + + +def resolve_coverage(client, platform, begin, end, policy, segmented, deadline): + if policy not in {"strict", "allow_partial"}: + raise insufficient_information() + + if policy == "strict": + return None + + months = { + month.strftime("%Y-%m"): monthrange(month.year, month.month)[1] + for month in iter_months(begin, end) + } + body = { + "size": len(months), + "query": { + "ids": {"values": [f"{platform.acron3}:{month}" for month in months]} + }, + } + + try: + response = search( + client, + generate_month_status_index_name(settings.OPENSEARCH_INDEX_NAME), + body, + deadline, + ) + markers = { + hit["_source"]["month"]: hit["_source"] for hit in response["hits"]["hits"] + } + except NotFoundError: + markers = {} + + details = [] + + for month, day_count in months.items(): + days = range(1, day_count + 1) + marker = markers.get(month) + confirmed = [] + missing = None + status = "unknown" + + if marker is not None: + mask = marker.get("day_mask") + full = (1 << day_count) - 1 + if ( + marker.get("collection") != platform.acron3 + or type(mask) is not int + or not 0 <= mask <= full + or type(marker.get("complete")) is not bool + or marker["complete"] != (mask == full) + ): + raise service_unavailable(f"Invalid coverage marker for {month}") + + confirmed = [day for day in days if mask & (1 << (day - 1))] + missing = [day for day in days if day not in confirmed] + status = ( + "complete" if not missing else "partial" if confirmed else "unavailable" + ) + + details.append( + { + "month": month, + "status": status, + "confirmed_days": confirmed, + "missing_days": missing, + } + ) + + included = [item["month"] for item in details if item["confirmed_days"]] + complete = all(item["status"] == "complete" for item in details) + + if segmented and not complete: + raise service_unavailable( + "Incomplete country/language coverage cannot be calculated from annual totals" + ) + + if not included: + raise service_unavailable("No confirmed usage days in requested period") + + return { + "policy": policy, + "status": "complete" if complete else "partial", + "included_months": included, + "months": details, + } + + +def coverage_fields(coverage): + if coverage is None: + return {} + + return { + "coverage": coverage, + "warnings": ( + [ + "Partial coverage: counts include only confirmed days; missing or unknown days are not zero usage." + ] + if coverage["status"] != "complete" + else [] + ), + } diff --git a/counter_api/tests/test_extension_coverage.py b/counter_api/tests/test_extension_coverage.py new file mode 100644 index 0000000..32ead37 --- /dev/null +++ b/counter_api/tests/test_extension_coverage.py @@ -0,0 +1,85 @@ +from datetime import date +from time import monotonic +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from counter_api.exceptions import CounterAPIError +from counter_api.extensions.coverage import resolve_coverage + +PLATFORM = SimpleNamespace(acron3="scl", collection_type="journals") +BEGIN, END = date(2023, 1, 1), date(2023, 12, 31) + + +def markers(**masks): + return { + "hits": { + "hits": [ + { + "_source": { + "collection": "scl", + "month": f"2023-{month}", + "day_mask": mask, + "complete": mask == (1 << 31) - 1, + } + } + for month, mask in masks.items() + ] + } + } + + +def resolve(client, segmented=False): + return resolve_coverage( + client, PLATFORM, BEGIN, END, "allow_partial", segmented, monotonic() + 30 + ) + + +def test_mixed_coverage(): + client = Mock() + client.search.return_value = markers(**{"01": 1, "03": (1 << 31) - 1, "05": 0}) + + coverage = resolve(client) + + assert coverage["included_months"] == ["2023-01", "2023-03"] + assert [m["status"] for m in coverage["months"][:5]] == [ + "partial", + "unknown", + "complete", + "unknown", + "unavailable", + ] + assert coverage["months"][0]["confirmed_days"] == [1] + assert coverage["months"][0]["missing_days"] == list(range(2, 32)) + assert coverage["months"][1]["missing_days"] is None + client.search.assert_called_once() + + +@pytest.mark.parametrize( + "response", [markers(), markers(**{"01": 0}), markers(**{"01": -1})] +) +def test_unusable_or_inconsistent_coverage_fails(response): + client = Mock() + client.search.return_value = response + + with pytest.raises(CounterAPIError): + resolve(client) + + +def test_annual_partial_rejected_and_strict_skips_markers(): + client = Mock() + client.search.return_value = markers(**{"01": 1}) + + with pytest.raises(CounterAPIError) as error: + resolve(client, segmented=True) + + assert "country/language" in error.value.data + + client.reset_mock() + + assert resolve_coverage(client, PLATFORM, BEGIN, END, "strict", False, 0) is None + + with pytest.raises(CounterAPIError): + resolve_coverage(client, PLATFORM, BEGIN, END, "other", False, 0) + client.search.assert_not_called() From 0cc876278be358fae541c82df05dbfcbc25de9b3 Mon Sep 17 00:00:00 2001 From: Rafael JP Damaceno Date: Sun, 8 Nov 2026 19:42:01 -0300 Subject: [PATCH 3/9] Monta filtros e somas de cobertura parcial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reutiliza os meses e dias confirmados na montagem das agregações para manter a paginação consistente. --- counter_api/extensions/coverage.py | 30 ++++++++++++++++++++++++++++++ counter_api/extensions/query.py | 13 +++++++++---- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/counter_api/extensions/coverage.py b/counter_api/extensions/coverage.py index f087625..5521de9 100644 --- a/counter_api/extensions/coverage.py +++ b/counter_api/extensions/coverage.py @@ -5,6 +5,7 @@ from counter_api.dates import iter_months from counter_api.exceptions import insufficient_information, service_unavailable +from counter_api.painless import CONFIRMED_DAYS_SUM_SCRIPT from counter_api.search import search from metrics.opensearch.names import generate_month_status_index_name @@ -109,3 +110,32 @@ def coverage_fields(coverage): else [] ), } + + +def metric_sum(metric, coverage): + partial = ( + { + item["month"]: [f"{day:02d}" for day in item["confirmed_days"]] + for item in coverage["months"] + if item["status"] == "partial" + } + if coverage + else {} + ) + if not partial: + return {"field": metric} + + return { + "script": { + "source": CONFIRMED_DAYS_SUM_SCRIPT, + "params": { + "metric": metric, + "complete": [ + item["month"] + for item in coverage["months"] + if item["status"] == "complete" + ], + "days": partial, + }, + } + } diff --git a/counter_api/extensions/query.py b/counter_api/extensions/query.py index 5c62146..ad1fb53 100644 --- a/counter_api/extensions/query.py +++ b/counter_api/extensions/query.py @@ -9,13 +9,14 @@ service_unavailable, ) from counter_api.extensions.contracts import SEGMENT_DIMENSIONS +from counter_api.extensions.coverage import metric_sum from metrics.opensearch.names import ( generate_analytics_index_name, generate_month_index_name, ) -def metric_composite_body(filters, sources, metric_type, page_size): +def metric_composite_body(filters, sources, metric_type, page_size, coverage=None): return { "size": 0, "track_total_hits": False, @@ -23,7 +24,7 @@ def metric_composite_body(filters, sources, metric_type, page_size): "aggs": { "rows": { "composite": {"size": page_size, "sources": sources}, - "aggs": {"count": {"sum": {"field": METRICS[metric_type]}}}, + "aggs": {"count": {"sum": metric_sum(METRICS[metric_type], coverage)}}, } }, } @@ -49,7 +50,7 @@ def validate_segmented_period(begin, end, dimensions, metric_type): return True -def usage_scope(platform, begin, end, segmented, data_type=None): +def usage_scope(platform, begin, end, segmented, data_type=None, included_months=None): if segmented: index_name = generate_analytics_index_name( settings.OPENSEARCH_INDEX_NAME, @@ -62,7 +63,11 @@ def usage_scope(platform, begin, end, segmented, data_type=None): settings.OPENSEARCH_INDEX_NAME, platform.acron3, ) - periods = [month.strftime("%Y-%m") for month in iter_months(begin, end)] + periods = ( + included_months + if included_months is not None + else [month.strftime("%Y-%m") for month in iter_months(begin, end)] + ) filters = [{"terms": {"month": periods}}] filters.extend( From e39c1717046cf6818b5440d3d8df4f82dca4da33 Mon Sep 17 00:00:00 2001 From: Rafael JP Damaceno Date: Sun, 8 Nov 2026 19:42:01 -0300 Subject: [PATCH 4/9] Integra cobertura parcial ao ranking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aplica a política de cobertura ao ranking e verifica a soma confirmada em páginas sucessivas. --- counter_api/extensions/ranking.py | 21 +++++++- counter_api/tests/test_extension_coverage.py | 54 ++++++++++++++++++++ counter_api/tests/test_extensions_ranking.py | 4 ++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/counter_api/extensions/ranking.py b/counter_api/extensions/ranking.py index 9baa043..beeb762 100644 --- a/counter_api/extensions/ranking.py +++ b/counter_api/extensions/ranking.py @@ -6,6 +6,7 @@ from counter_api.constants import METADATA_BATCH_SIZE, METRICS from counter_api.exceptions import insufficient_information from counter_api.extensions.contracts import RANKING_ENTITIES, RANKING_GROUPS +from counter_api.extensions.coverage import coverage_fields, resolve_coverage from counter_api.extensions.query import ( metric_composite_body, require_complete_period, @@ -43,7 +44,18 @@ def run(self, platform, begin, end, params): metric_type, segmented, ) = self._parameters(platform, begin, end, params) - require_complete_period(platform, begin, end) + + coverage = resolve_coverage( + self.client, + platform, + begin, + end, + params.get("coverage_policy", "allow_partial"), + segmented, + deadline, + ) + if coverage is None: + require_complete_period(platform, begin, end) parent_key = self._parent_key(platform, parent_type, parent_id) key_batches = self._key_batches(platform, params, deadline) @@ -60,6 +72,7 @@ def run(self, platform, begin, end, params): limit, deadline, segmented, + coverage, ) return { @@ -71,6 +84,7 @@ def run(self, platform, begin, end, params): "entity_type": params["entity_type"], "metric_type": metric_type, "groups": self._result_groups(groups, relation, group_by, deadline), + **coverage_fields(coverage), } def _parameters(self, platform, begin, end, params): @@ -162,6 +176,7 @@ def _ranked_groups( limit, deadline, segmented, + coverage=None, ): groups = {} @@ -178,6 +193,7 @@ def _ranked_groups( params, source_keys, segmented, + coverage, ) for buckets in composite_pages(self.client, index_name, body, deadline): @@ -288,6 +304,7 @@ def _query_body( params, source_keys, segmented, + coverage=None, ): index_name, filters = usage_scope( platform, @@ -295,6 +312,7 @@ def _query_body( end, segmented, data_type, + included_months=coverage["included_months"] if coverage else None, ) if parent_key: @@ -328,6 +346,7 @@ def _query_body( sources, metric_type, RANKING_PAGE_SIZE, + coverage=None if segmented else coverage, ) return index_name, body diff --git a/counter_api/tests/test_extension_coverage.py b/counter_api/tests/test_extension_coverage.py index 32ead37..bb213dc 100644 --- a/counter_api/tests/test_extension_coverage.py +++ b/counter_api/tests/test_extension_coverage.py @@ -7,6 +7,7 @@ from counter_api.exceptions import CounterAPIError from counter_api.extensions.coverage import resolve_coverage +from counter_api.extensions.ranking import RankingQuery PLATFORM = SimpleNamespace(acron3="scl", collection_type="journals") BEGIN, END = date(2023, 1, 1), date(2023, 12, 31) @@ -83,3 +84,56 @@ def test_annual_partial_rejected_and_strict_skips_markers(): with pytest.raises(CounterAPIError): resolve_coverage(client, PLATFORM, BEGIN, END, "other", False, 0) client.search.assert_not_called() + + +@pytest.mark.parametrize("kind", ["ranking"]) +def test_default_queries_share_confirmed_sum_across_pages(kind): + client = Mock() + key = "source_key" if kind == "ranking" else "yop" + client.search.side_effect = [ + markers(**{"01": 1, "03": (1 << 31) - 1}), + { + "aggregations": { + "rows": { + "buckets": [{"key": {key: None}, "count": {"value": 5}}], + "after_key": {key: None}, + } + } + }, + {"aggregations": {"rows": {"buckets": []}}}, + ] + query, params = ( + ( + RankingQuery, + { + "parent_type": "collection", + "parent_id": "scl", + "entity_type": "journals", + }, + ) + if kind == "ranking" + else ( + None, + {"entity_type": "collection", "entity_id": "scl", "dimension": "yop"}, + ) + ) + + payload = query(Mock(client=client)).run(PLATFORM, BEGIN, END, params) + + assert payload["coverage"]["status"] == "partial" + if kind == "distribution": + assert payload["total"] == sum(b["count"] for b in payload["buckets"]) == 5 + bodies = [call.kwargs["body"] for call in client.search.call_args_list[1:]] + assert len(bodies) == 2 + + for body in bodies: + assert {"terms": {"month": ["2023-01", "2023-03"]}} in body["query"]["bool"][ + "filter" + ] + script = body["aggs"]["rows"]["aggs"]["count"]["sum"]["script"] + assert script["params"] == { + "metric": "total_requests", + "complete": ["2023-03"], + "days": {"2023-01": ["01"]}, + } + assert "Missing daily_metrics" in script["source"] diff --git a/counter_api/tests/test_extensions_ranking.py b/counter_api/tests/test_extensions_ranking.py index 4a4144c..9aa27d5 100644 --- a/counter_api/tests/test_extensions_ranking.py +++ b/counter_api/tests/test_extensions_ranking.py @@ -27,6 +27,7 @@ def test_incomplete_annual_period_does_not_query_metrics(self): date(2025, 1, 1), date(2025, 12, 31), { + "coverage_policy": "strict", "entity_type": "journals", "parent_type": "collection", "parent_id": "scl", @@ -92,6 +93,7 @@ def test_ranking_reads_all_pages_and_breaks_ties_by_item_key(self): "parent_type": "journal", "parent_id": "Print_ISSN:1234-5678", "group_by": "country,language", + "coverage_policy": "strict", "limit": "1", }, ) @@ -165,6 +167,7 @@ def test_subject_area_keys_and_yop_are_filtered_before_aggregation(self): "parent_type": "collection", "parent_id": "scl", "subject_area": "Health", + "coverage_policy": "strict", "yop": "2020|2022-2023", }, ) @@ -236,6 +239,7 @@ def test_country_ranking_filters_yop_with_document_metadata(self): "parent_id": "Print_ISSN:1234-5678", "group_by": "country", "yop": "2020-2023", + "coverage_policy": "strict", }, ) From 9ba6c086deaa89d0a59af52030d094804d83f5a0 Mon Sep 17 00:00:00 2001 From: Rafael JP Damaceno Date: Sun, 8 Nov 2026 19:42:01 -0300 Subject: [PATCH 5/9] =?UTF-8?q?Integra=20cobertura=20parcial=20=C3=A0=20di?= =?UTF-8?q?stribui=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mantém totais e filtros da distribuição alinhados aos dias confirmados e amplia o teste compartilhado. --- counter_api/extensions/distribution.py | 26 +++++++++++++++++-- counter_api/tests/test_extension_coverage.py | 5 ++-- .../tests/test_extensions_distribution.py | 11 +++++++- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/counter_api/extensions/distribution.py b/counter_api/extensions/distribution.py index 5fa9e09..4e2b4a7 100644 --- a/counter_api/extensions/distribution.py +++ b/counter_api/extensions/distribution.py @@ -9,6 +9,7 @@ DISTRIBUTION_DIMENSIONS, DISTRIBUTION_ENTITIES, ) +from counter_api.extensions.coverage import coverage_fields, resolve_coverage from counter_api.extensions.query import ( metric_composite_body, require_complete_period, @@ -37,7 +38,18 @@ def run(self, platform, begin, end, params): end, params, ) - require_complete_period(platform, begin, end) + + coverage = resolve_coverage( + self.client, + platform, + begin, + end, + params.get("coverage_policy", "allow_partial"), + segmented, + deadline, + ) + if coverage is None: + require_complete_period(platform, begin, end) key_field, entity_key = self._entity_filter( platform, @@ -54,6 +66,7 @@ def run(self, platform, begin, end, params): key_field, entity_key, segmented, + coverage, ) total, buckets = self._buckets( index_name, @@ -73,6 +86,7 @@ def run(self, platform, begin, end, params): "metric_type": metric_type, "total": total, "buckets": buckets, + **coverage_fields(coverage), } def _parameters(self, platform, begin, end, params): @@ -137,6 +151,7 @@ def _query_body( key_field, entity_key, segmented, + coverage=None, ): data_type = DISTRIBUTION_ENTITIES[entity_type].get("data_type") index_name, filters = usage_scope( @@ -145,6 +160,7 @@ def _query_body( end, segmented, data_type, + included_months=coverage["included_months"] if coverage else None, ) if key_field: @@ -168,7 +184,13 @@ def _query_body( } } ) - body = metric_composite_body(filters, sources, metric_type, PAGE_SIZE) + body = metric_composite_body( + filters, + sources, + metric_type, + PAGE_SIZE, + coverage=None if segmented else coverage, + ) return index_name, body diff --git a/counter_api/tests/test_extension_coverage.py b/counter_api/tests/test_extension_coverage.py index bb213dc..8600604 100644 --- a/counter_api/tests/test_extension_coverage.py +++ b/counter_api/tests/test_extension_coverage.py @@ -7,6 +7,7 @@ from counter_api.exceptions import CounterAPIError from counter_api.extensions.coverage import resolve_coverage +from counter_api.extensions.distribution import DistributionQuery from counter_api.extensions.ranking import RankingQuery PLATFORM = SimpleNamespace(acron3="scl", collection_type="journals") @@ -86,7 +87,7 @@ def test_annual_partial_rejected_and_strict_skips_markers(): client.search.assert_not_called() -@pytest.mark.parametrize("kind", ["ranking"]) +@pytest.mark.parametrize("kind", ["ranking", "distribution"]) def test_default_queries_share_confirmed_sum_across_pages(kind): client = Mock() key = "source_key" if kind == "ranking" else "yop" @@ -113,7 +114,7 @@ def test_default_queries_share_confirmed_sum_across_pages(kind): ) if kind == "ranking" else ( - None, + DistributionQuery, {"entity_type": "collection", "entity_id": "scl", "dimension": "yop"}, ) ) diff --git a/counter_api/tests/test_extensions_distribution.py b/counter_api/tests/test_extensions_distribution.py index 131997f..9471c2c 100644 --- a/counter_api/tests/test_extensions_distribution.py +++ b/counter_api/tests/test_extensions_distribution.py @@ -26,7 +26,12 @@ def test_incomplete_month_does_not_query_metrics(self): platform, date(2025, 2, 1), date(2025, 2, 28), - {"entity_type": "collection", "entity_id": "scl", "dimension": "yop"}, + { + "entity_type": "collection", + "entity_id": "scl", + "dimension": "yop", + "coverage_policy": "strict", + }, ) self.assertEqual(error.exception.status_code, 503) @@ -70,6 +75,7 @@ def test_subject_areas_overlap_without_inflating_total(self): "entity_type": "collection", "entity_id": "scl", "dimension": "subject_area", + "coverage_policy": "strict", }, ) @@ -139,6 +145,7 @@ def test_yop_distribution_combines_missing_and_unknown_year(self): "entity_type": "collection", "entity_id": "scl", "dimension": "yop", + "coverage_policy": "strict", }, ) @@ -189,6 +196,7 @@ def test_country_language_cross_uses_annual_index(self): "entity_type": "collection", "entity_id": "scl", "dimension": "country,language", + "coverage_policy": "strict", }, ) @@ -258,6 +266,7 @@ def test_country_yop_reads_year_from_document_metadata(self): "entity_type": "collection", "entity_id": "scl", "dimension": "country,yop", + "coverage_policy": "strict", }, ) From 9cad128bffedea43b6ee057b9cc246ec58378649 Mon Sep 17 00:00:00 2001 From: Rafael JP Damaceno Date: Sun, 8 Nov 2026 19:42:01 -0300 Subject: [PATCH 6/9] Preserva cobertura nos arquivos exportados MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Conserva os metadados em CSV e XLSX para que resultados vazios não ocultem lacunas de cobertura. --- counter_api/extensions/exports.py | 23 ++++++++++++++ counter_api/tests/test_extension_coverage.py | 33 ++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/counter_api/extensions/exports.py b/counter_api/extensions/exports.py index 7b3a640..f0579c7 100644 --- a/counter_api/extensions/exports.py +++ b/counter_api/extensions/exports.py @@ -1,4 +1,5 @@ import csv +import json from io import StringIO from tempfile import SpooledTemporaryFile @@ -38,6 +39,17 @@ def build_export(payload, kind, output_format): for bucket in payload["buckets"] ) + metadata = None + if "coverage" in payload: + metadata = [ + json.dumps(payload["coverage"], separators=(",", ":")), + json.dumps(payload["warnings"]), + payload.get("total", ""), + ] + if output_format == "csv": + headings.extend(["Coverage", "Warnings", "Total"]) + rows = (row + metadata for row in rows) + output = SpooledTemporaryFile(max_size=SPOOL_SIZE) if output_format == "csv": line = StringIO(newline="") @@ -47,11 +59,17 @@ def build_export(payload, kind, output_format): line.seek(0) line.truncate(0) + has_rows = False for row in rows: + has_rows = True writer.writerow(row) output.write(line.getvalue().encode("utf-8")) line.seek(0) line.truncate(0) + + if metadata is not None and not has_rows: + writer.writerow([""] * (len(headings) - len(metadata)) + metadata) + output.write(line.getvalue().encode("utf-8")) else: workbook = Workbook(write_only=True) sheet = workbook.create_sheet(title=kind.capitalize()) @@ -69,6 +87,11 @@ def build_export(payload, kind, output_format): raise insufficient_information() sheet.append(row) + if metadata is not None: + coverage_sheet = workbook.create_sheet(title="Coverage") + coverage_sheet.append(["Coverage", "Warnings", "Total"]) + coverage_sheet.append(metadata) + workbook.save(output) output.seek(0) diff --git a/counter_api/tests/test_extension_coverage.py b/counter_api/tests/test_extension_coverage.py index 8600604..faa1d23 100644 --- a/counter_api/tests/test_extension_coverage.py +++ b/counter_api/tests/test_extension_coverage.py @@ -1,13 +1,18 @@ +import csv +import json from datetime import date +from io import StringIO from time import monotonic from types import SimpleNamespace from unittest.mock import Mock import pytest +from openpyxl import load_workbook from counter_api.exceptions import CounterAPIError from counter_api.extensions.coverage import resolve_coverage from counter_api.extensions.distribution import DistributionQuery +from counter_api.extensions.exports import build_export from counter_api.extensions.ranking import RankingQuery PLATFORM = SimpleNamespace(acron3="scl", collection_type="journals") @@ -138,3 +143,31 @@ def test_default_queries_share_confirmed_sum_across_pages(kind): "days": {"2023-01": ["01"]}, } assert "Missing daily_metrics" in script["source"] + + +@pytest.mark.parametrize("output_format", ["csv", "xlsx"]) +def test_empty_export_preserves_coverage_metadata(output_format): + payload = { + "dimension": "yop", + "buckets": [], + "total": 0, + "coverage": {"status": "partial", "included_months": ["2023-03"]}, + "warnings": ["Only confirmed days are included."], + } + + with build_export(payload, "distribution", output_format) as output: + if output_format == "csv": + rows = list(csv.DictReader(StringIO(output.read().decode()))) + assert len(rows) == 1 + metadata = rows[0] + assert metadata["Yop"] == metadata["Count"] == "" + else: + workbook = load_workbook(output, read_only=True) + assert list(workbook["Distribution"].values) == [("Yop", "Count")] + headings, values = list(workbook["Coverage"].values) + metadata = dict(zip(headings, values)) + workbook.close() + + assert json.loads(metadata["Coverage"]) == payload["coverage"] + assert json.loads(metadata["Warnings"]) == payload["warnings"] + assert int(metadata["Total"]) == 0 From c0b78f2a11a00d8bba76795fffc6ebd75a026868 Mon Sep 17 00:00:00 2001 From: Rafael JP Damaceno Date: Sun, 8 Nov 2026 19:42:01 -0300 Subject: [PATCH 7/9] =?UTF-8?q?Exp=C3=B5e=20a=20cobertura=20no=20contrato?= =?UTF-8?q?=20HTTP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documenta a política e os metadados no esquema e verifica sua entrega numa requisição HTTP real. --- counter_api/extensions/openapi.py | 27 +++++++++++++- counter_api/extensions/views.py | 11 ++++++ counter_api/tests/test_extension_views.py | 45 +++++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/counter_api/extensions/openapi.py b/counter_api/extensions/openapi.py index bb4603a..5e55bc5 100644 --- a/counter_api/extensions/openapi.py +++ b/counter_api/extensions/openapi.py @@ -12,7 +12,30 @@ class RankingGroupSerializer(serializers.Serializer): items = RankingItemSerializer(many=True) -class RankingSerializer(serializers.Serializer): +class CoverageMonthSerializer(serializers.Serializer): + month = serializers.CharField() + status = serializers.ChoiceField( + choices=["complete", "partial", "unavailable", "unknown"] + ) + confirmed_days = serializers.ListField(child=serializers.IntegerField()) + missing_days = serializers.ListField( + child=serializers.IntegerField(), allow_null=True + ) + + +class CoverageSerializer(serializers.Serializer): + policy = serializers.CharField() + status = serializers.ChoiceField(choices=["complete", "partial"]) + included_months = serializers.ListField(child=serializers.CharField()) + months = CoverageMonthSerializer(many=True) + + +class CoverageResponseSerializer(serializers.Serializer): + coverage = CoverageSerializer(required=False) + warnings = serializers.ListField(child=serializers.CharField(), required=False) + + +class RankingSerializer(CoverageResponseSerializer): platform = serializers.CharField() begin_date = serializers.DateField() end_date = serializers.DateField() @@ -29,7 +52,7 @@ class DistributionBucketSerializer(serializers.Serializer): count = serializers.IntegerField() -class DistributionSerializer(serializers.Serializer): +class DistributionSerializer(CoverageResponseSerializer): platform = serializers.CharField() begin_date = serializers.DateField() end_date = serializers.DateField() diff --git a/counter_api/extensions/views.py b/counter_api/extensions/views.py index 08426c8..58144c4 100644 --- a/counter_api/extensions/views.py +++ b/counter_api/extensions/views.py @@ -48,6 +48,15 @@ def _download_response(payload, kind, platform, output_format): return response +COVERAGE_PARAMETER = OpenApiParameter( + "coverage_policy", + OpenApiTypes.STR, + enum=["strict", "allow_partial"], + default="allow_partial", + description="Allow confirmed days of partial months. Incomplete country/language periods remain unsupported.", +) + + class RankingView(CounterAPIView): @extend_schema( tags=["SciELO extensions"], @@ -62,6 +71,7 @@ class RankingView(CounterAPIView): OpenApiParameter("parent_id", OpenApiTypes.STR, required=True), OpenApiParameter("entity_type", OpenApiTypes.STR, required=True), OpenApiParameter("metric_type", OpenApiTypes.STR, required=False), + COVERAGE_PARAMETER, OpenApiParameter("group_by", OpenApiTypes.STR, required=False), OpenApiParameter("country", OpenApiTypes.STR, required=False), OpenApiParameter("language", OpenApiTypes.STR, required=False), @@ -159,6 +169,7 @@ class DistributionView(CounterAPIView): OpenApiParameter("entity_type", OpenApiTypes.STR, required=True), OpenApiParameter("entity_id", OpenApiTypes.STR, required=True), OpenApiParameter("dimension", OpenApiTypes.STR, required=True), + COVERAGE_PARAMETER, OpenApiParameter("metric_type", OpenApiTypes.STR, required=False), OpenApiParameter("format", OpenApiTypes.STR, required=False), ], diff --git a/counter_api/tests/test_extension_views.py b/counter_api/tests/test_extension_views.py index 73d0828..e2bdbae 100644 --- a/counter_api/tests/test_extension_views.py +++ b/counter_api/tests/test_extension_views.py @@ -41,6 +41,51 @@ def test_extension_capabilities_show_journal_relations(self, available_months): ) self.assertNotIn("coverage", response.json()) + @patch("counter_api.extensions.distribution.ReportQuery") + def test_default_http_request_includes_partial_coverage(self, report_query): + search = report_query.return_value.client.search + search.side_effect = [ + { + "hits": { + "hits": [ + { + "_source": { + "collection": "scl", + "month": "2023-03", + "day_mask": (1 << 31) - 1 - (1 << 15), + "complete": False, + } + } + ] + } + }, + {"aggregations": {"rows": {"buckets": []}}}, + ] + + response = self.client.get( + "/r51/extensions/distribution", + { + "platform": "scl", + "begin_date": "2023-03", + "end_date": "2023-03", + "entity_type": "collection", + "entity_id": "scl", + "dimension": "yop", + }, + ) + + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertEqual(payload["coverage"]["status"], "partial") + self.assertEqual(payload["coverage"]["months"][0]["missing_days"], [16]) + self.assertTrue(payload["warnings"]) + self.assertEqual(search.call_count, 2) + script = search.call_args.kwargs["body"]["aggs"]["rows"]["aggs"]["count"][ + "sum" + ]["script"] + self.assertNotIn("16", script["params"]["days"]["2023-03"]) + self.assertEqual(len(script["params"]["days"]["2023-03"]), 30) + @patch("counter_api.extensions.views.DistributionQuery") @patch("counter_api.extensions.views.RankingQuery") def test_public_extensions_preserve_values_in_all_formats( From 83655ecb26119534bad6392f6ac75cd816d6b829 Mon Sep 17 00:00:00 2001 From: Rafael JP Damaceno Date: Sun, 8 Nov 2026 19:42:01 -0300 Subject: [PATCH 8/9] Explica os modos de cobertura Orienta consumidores sobre dados parciais sem confundir dias ausentes com uso zero. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 1f78177..f22b50a 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,16 @@ make format_check # confere Black e isort Os alvos aceitam `compose=`; o padrão é `local.yml`. Não publique arquivos de configuração de ambiente que contenham credenciais. +## Cobertura parcial nas extensões + +Ranking e distribuição usam `coverage_policy=allow_partial` por padrão: +contam somente dias confirmados, sem estimar dias ausentes ou desconhecidos. +JSON, CSV e XLSX preservam a cobertura e os avisos, inclusive sem resultados. +Use `coverage_policy=strict` para exigir cobertura completa, como antes. +País/idioma exigem anos completos e cobertura completa, pois usam totais anuais. +Períodos sem dias confirmados e marcadores inconsistentes retornam erro, não zero. +Relatórios COUNTER padrão não mudam. Consulte o OpenAPI para o contrato completo. + ## API COUNTER e política de acesso As rotas GET sob `/r51/` oferecem descoberta, relatórios COUNTER R5.1 e From 04d8f6811da62f5521062a9f2d0e70efeafceff4 Mon Sep 17 00:00:00 2001 From: Rafael JP Damaceno Date: Sun, 8 Nov 2026 19:58:28 -0300 Subject: [PATCH 9/9] Disponibiliza os totais anuais com cobertura parcial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mostra os valores existentes de país e idioma sem aguardar todas as cargas, distinguindo cobertura informativa de filtro por dias. --- README.md | 7 ++-- counter_api/extensions/coverage.py | 33 ++++++++------- counter_api/extensions/openapi.py | 9 ++++- counter_api/extensions/views.py | 5 ++- counter_api/tests/test_extension_coverage.py | 42 +++++++++++++++----- 5 files changed, 65 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index f22b50a..c36b805 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,12 @@ arquivos de configuração de ambiente que contenham credenciais. ## Cobertura parcial nas extensões Ranking e distribuição usam `coverage_policy=allow_partial` por padrão: -contam somente dias confirmados, sem estimar dias ausentes ou desconhecidos. +contam dias confirmados no mensal e os totais disponíveis no anual de país/idioma. JSON, CSV e XLSX preservam a cobertura e os avisos, inclusive sem resultados. Use `coverage_policy=strict` para exigir cobertura completa, como antes. -País/idioma exigem anos completos e cobertura completa, pois usam totais anuais. -Períodos sem dias confirmados e marcadores inconsistentes retornam erro, não zero. +País/idioma exigem intervalos de anos inteiros; seus valores podem ser atualizados +com novas cargas. A cobertura anual é informativa, não um filtro dos totais. +Consultas mensais sem dias confirmados e marcadores inconsistentes retornam erro. Relatórios COUNTER padrão não mudam. Consulte o OpenAPI para o contrato completo. ## API COUNTER e política de acesso diff --git a/counter_api/extensions/coverage.py b/counter_api/extensions/coverage.py index 5521de9..48afa8b 100644 --- a/counter_api/extensions/coverage.py +++ b/counter_api/extensions/coverage.py @@ -80,18 +80,14 @@ def resolve_coverage(client, platform, begin, end, policy, segmented, deadline): included = [item["month"] for item in details if item["confirmed_days"]] complete = all(item["status"] == "complete" for item in details) - if segmented and not complete: - raise service_unavailable( - "Incomplete country/language coverage cannot be calculated from annual totals" - ) - - if not included: + if not segmented and not included: raise service_unavailable("No confirmed usage days in requested period") return { "policy": policy, "status": "complete" if complete else "partial", - "included_months": included, + "basis": "available_annual_totals" if segmented else "confirmed_days", + "included_months": None if segmented else included, "months": details, } @@ -100,16 +96,19 @@ def coverage_fields(coverage): if coverage is None: return {} - return { - "coverage": coverage, - "warnings": ( - [ - "Partial coverage: counts include only confirmed days; missing or unknown days are not zero usage." - ] - if coverage["status"] != "complete" - else [] - ), - } + warnings = [] + + if coverage["basis"] == "available_annual_totals": + warnings.append( + "Available annual totals may change as data arrives or processing completes. " + "Coverage is informative and does not restrict the days included in these totals." + ) + elif coverage["status"] != "complete": + warnings.append( + "Partial coverage: counts include only confirmed days; missing or unknown days are not zero usage." + ) + + return {"coverage": coverage, "warnings": warnings} def metric_sum(metric, coverage): diff --git a/counter_api/extensions/openapi.py b/counter_api/extensions/openapi.py index 5e55bc5..f3740cd 100644 --- a/counter_api/extensions/openapi.py +++ b/counter_api/extensions/openapi.py @@ -26,7 +26,14 @@ class CoverageMonthSerializer(serializers.Serializer): class CoverageSerializer(serializers.Serializer): policy = serializers.CharField() status = serializers.ChoiceField(choices=["complete", "partial"]) - included_months = serializers.ListField(child=serializers.CharField()) + basis = serializers.ChoiceField( + choices=["confirmed_days", "available_annual_totals"] + ) + included_months = serializers.ListField( + child=serializers.CharField(), + allow_null=True, + help_text="Null for annual totals: coverage is informative, not a day filter.", + ) months = CoverageMonthSerializer(many=True) diff --git a/counter_api/extensions/views.py b/counter_api/extensions/views.py index 58144c4..1db9366 100644 --- a/counter_api/extensions/views.py +++ b/counter_api/extensions/views.py @@ -53,7 +53,10 @@ def _download_response(payload, kind, platform, output_format): OpenApiTypes.STR, enum=["strict", "allow_partial"], default="allow_partial", - description="Allow confirmed days of partial months. Incomplete country/language periods remain unsupported.", + description=( + "Use confirmed days for monthly counts and available annual totals for " + "country/language. Use strict to require complete coverage." + ), ) diff --git a/counter_api/tests/test_extension_coverage.py b/counter_api/tests/test_extension_coverage.py index faa1d23..d8f3cb1 100644 --- a/counter_api/tests/test_extension_coverage.py +++ b/counter_api/tests/test_extension_coverage.py @@ -74,16 +74,8 @@ def test_unusable_or_inconsistent_coverage_fails(response): resolve(client) -def test_annual_partial_rejected_and_strict_skips_markers(): +def test_strict_skips_markers_and_invalid_policy_fails(): client = Mock() - client.search.return_value = markers(**{"01": 1}) - - with pytest.raises(CounterAPIError) as error: - resolve(client, segmented=True) - - assert "country/language" in error.value.data - - client.reset_mock() assert resolve_coverage(client, PLATFORM, BEGIN, END, "strict", False, 0) is None @@ -145,6 +137,38 @@ def test_default_queries_share_confirmed_sum_across_pages(kind): assert "Missing daily_metrics" in script["source"] +@pytest.mark.parametrize("response", [markers(), markers(**{"01": 1})]) +def test_annual_totals_do_not_depend_on_confirmed_days(response): + client = Mock() + client.search.side_effect = [ + response, + { + "aggregations": { + "rows": { + "buckets": [{"key": {"country": "BR"}, "count": {"value": 15}}] + } + } + }, + ] + + payload = DistributionQuery(Mock(client=client)).run( + PLATFORM, + BEGIN, + END, + {"entity_type": "collection", "entity_id": "scl", "dimension": "country"}, + ) + + assert payload["total"] == 15 + assert payload["coverage"]["status"] == "partial" + assert payload["coverage"]["basis"] == "available_annual_totals" + assert payload["coverage"]["included_months"] is None + assert "informative" in payload["warnings"][0] + + body = client.search.call_args.kwargs["body"] + assert {"terms": {"year": ["2023"]}} in body["query"]["bool"]["filter"] + assert body["aggs"]["rows"]["aggs"]["count"]["sum"] == {"field": "total_requests"} + + @pytest.mark.parametrize("output_format", ["csv", "xlsx"]) def test_empty_export_preserves_coverage_metadata(output_format): payload = {