Disponibiliza meses incompletos nas extensões da API

README.md

@@ -107,6 +107,17 @@ make format_check   # confere Black e isort
 Os alvos aceitam `compose=<arquivo>`; o padrão é `local.yml`. Não publique
 arquivos de configuração de ambiente que contenham credenciais.
 
+## Cobertura parcial nas extensões
+
+Ranking e distribuição usam `coverage_policy=allow_partial` por padrão:
+contam dias confirmados no mensal e os totais disponíveis no anual de país/idioma.
+JSON, CSV e XLSX preservam a cobertura e os avisos, inclusive sem resultados.
+Use `coverage_policy=strict` para exigir cobertura completa, como antes.
+País/idioma exigem intervalos de anos inteiros; seus valores podem ser atualizados
+com novas cargas. A cobertura anual é informativa, não um filtro dos totais.
+Consultas mensais sem dias confirmados e marcadores inconsistentes retornam erro.
+Relatórios COUNTER padrão não mudam. Consulte o OpenAPI para o contrato completo.
+
 ## API COUNTER e política de acesso
 
 As rotas GET sob `/r51/` oferecem descoberta, relatórios COUNTER R5.1 e

counter_api/extensions/coverage.py

@@ -0,0 +1,140 @@
+from calendar import monthrange
+
+from django.conf import settings
+from opensearchpy import NotFoundError
+
+from counter_api.dates import iter_months
+from counter_api.exceptions import insufficient_information, service_unavailable
+from counter_api.painless import CONFIRMED_DAYS_SUM_SCRIPT
+from counter_api.search import search
+from metrics.opensearch.names import generate_month_status_index_name
+
+
+def resolve_coverage(client, platform, begin, end, policy, segmented, deadline):
+    if policy not in {"strict", "allow_partial"}:
+        raise insufficient_information()
+
+    if policy == "strict":
+        return None
+
+    months = {
+        month.strftime("%Y-%m"): monthrange(month.year, month.month)[1]
+        for month in iter_months(begin, end)
+    }
+    body = {
+        "size": len(months),
+        "query": {
+            "ids": {"values": [f"{platform.acron3}:{month}" for month in months]}
+        },
+    }
+
+    try:
+        response = search(
+            client,
+            generate_month_status_index_name(settings.OPENSEARCH_INDEX_NAME),
+            body,
+            deadline,
+        )
+        markers = {
+            hit["_source"]["month"]: hit["_source"] for hit in response["hits"]["hits"]
+        }
+    except NotFoundError:
+        markers = {}
+
+    details = []
+
+    for month, day_count in months.items():
+        days = range(1, day_count + 1)
+        marker = markers.get(month)
+        confirmed = []
+        missing = None
+        status = "unknown"
+
+        if marker is not None:
+            mask = marker.get("day_mask")
+            full = (1 << day_count) - 1
+            if (
+                marker.get("collection") != platform.acron3
+                or type(mask) is not int
+                or not 0 <= mask <= full
+                or type(marker.get("complete")) is not bool
+                or marker["complete"] != (mask == full)
+            ):
+                raise service_unavailable(f"Invalid coverage marker for {month}")
+
+            confirmed = [day for day in days if mask & (1 << (day - 1))]
+            missing = [day for day in days if day not in confirmed]
+            status = (
+                "complete" if not missing else "partial" if confirmed else "unavailable"
+            )
+
+        details.append(
+            {
+                "month": month,
+                "status": status,
+                "confirmed_days": confirmed,
+                "missing_days": missing,
+            }
+        )
+
+    included = [item["month"] for item in details if item["confirmed_days"]]
+    complete = all(item["status"] == "complete" for item in details)
+
+    if not segmented and not included:
+        raise service_unavailable("No confirmed usage days in requested period")
+
+    return {
+        "policy": policy,
+        "status": "complete" if complete else "partial",
+        "basis": "available_annual_totals" if segmented else "confirmed_days",
+        "included_months": None if segmented else included,
+        "months": details,
+    }
+
+
+def coverage_fields(coverage):
+    if coverage is None:
+        return {}
+
+    warnings = []
+
+    if coverage["basis"] == "available_annual_totals":
+        warnings.append(
+            "Available annual totals may change as data arrives or processing completes. "
+            "Coverage is informative and does not restrict the days included in these totals."
+        )
+    elif coverage["status"] != "complete":
+        warnings.append(
+            "Partial coverage: counts include only confirmed days; missing or unknown days are not zero usage."
+        )
+
+    return {"coverage": coverage, "warnings": warnings}
+
+
+def metric_sum(metric, coverage):
+    partial = (
+        {
+            item["month"]: [f"{day:02d}" for day in item["confirmed_days"]]
+            for item in coverage["months"]
+            if item["status"] == "partial"
+        }
+        if coverage
+        else {}
+    )
+    if not partial:
+        return {"field": metric}
+
+    return {
+        "script": {
+            "source": CONFIRMED_DAYS_SUM_SCRIPT,
+            "params": {
+                "metric": metric,
+                "complete": [
+                    item["month"]
+                    for item in coverage["months"]
+                    if item["status"] == "complete"
+                ],
+                "days": partial,
+            },
+        }
+    }

counter_api/extensions/distribution.py

@@ -9,6 +9,7 @@
     DISTRIBUTION_DIMENSIONS,
     DISTRIBUTION_ENTITIES,
 )
+from counter_api.extensions.coverage import coverage_fields, resolve_coverage
 from counter_api.extensions.query import (
     metric_composite_body,
     require_complete_period,
@@ -37,7 +38,18 @@ def run(self, platform, begin, end, params):
             end,
             params,
         )
-        require_complete_period(platform, begin, end)
+
+        coverage = resolve_coverage(
+            self.client,
+            platform,
+            begin,
+            end,
+            params.get("coverage_policy", "allow_partial"),
+            segmented,
+            deadline,
+        )
+        if coverage is None:
+            require_complete_period(platform, begin, end)
 
         key_field, entity_key = self._entity_filter(
             platform,
@@ -54,6 +66,7 @@ def run(self, platform, begin, end, params):
             key_field,
             entity_key,
             segmented,
+            coverage,
         )
         total, buckets = self._buckets(
             index_name,
@@ -73,6 +86,7 @@ def run(self, platform, begin, end, params):
             "metric_type": metric_type,
             "total": total,
             "buckets": buckets,
+            **coverage_fields(coverage),
         }
 
     def _parameters(self, platform, begin, end, params):
@@ -137,6 +151,7 @@ def _query_body(
         key_field,
         entity_key,
         segmented,
+        coverage=None,
     ):
         data_type = DISTRIBUTION_ENTITIES[entity_type].get("data_type")
         index_name, filters = usage_scope(
@@ -145,6 +160,7 @@ def _query_body(
             end,
             segmented,
             data_type,
+            included_months=coverage["included_months"] if coverage else None,
         )
 
         if key_field:
@@ -168,7 +184,13 @@ def _query_body(
                     }
                 }
             )
-        body = metric_composite_body(filters, sources, metric_type, PAGE_SIZE)
+        body = metric_composite_body(
+            filters,
+            sources,
+            metric_type,
+            PAGE_SIZE,
+            coverage=None if segmented else coverage,
+        )
 
         return index_name, body
 

counter_api/extensions/exports.py

@@ -1,4 +1,5 @@
 import csv
+import json
 from io import StringIO
 from tempfile import SpooledTemporaryFile
 
@@ -38,6 +39,17 @@ def build_export(payload, kind, output_format):
             for bucket in payload["buckets"]
         )
 
+    metadata = None
+    if "coverage" in payload:
+        metadata = [
+            json.dumps(payload["coverage"], separators=(",", ":")),
+            json.dumps(payload["warnings"]),
+            payload.get("total", ""),
+        ]
+        if output_format == "csv":
+            headings.extend(["Coverage", "Warnings", "Total"])
+            rows = (row + metadata for row in rows)
+
     output = SpooledTemporaryFile(max_size=SPOOL_SIZE)
     if output_format == "csv":
         line = StringIO(newline="")
@@ -47,11 +59,17 @@ def build_export(payload, kind, output_format):
         line.seek(0)
         line.truncate(0)
 
+        has_rows = False
         for row in rows:
+            has_rows = True
             writer.writerow(row)
             output.write(line.getvalue().encode("utf-8"))
             line.seek(0)
             line.truncate(0)
+
+        if metadata is not None and not has_rows:
+            writer.writerow([""] * (len(headings) - len(metadata)) + metadata)
+            output.write(line.getvalue().encode("utf-8"))
     else:
         workbook = Workbook(write_only=True)
         sheet = workbook.create_sheet(title=kind.capitalize())
@@ -69,6 +87,11 @@ def build_export(payload, kind, output_format):
                 raise insufficient_information()
             sheet.append(row)
 
+        if metadata is not None:
+            coverage_sheet = workbook.create_sheet(title="Coverage")
+            coverage_sheet.append(["Coverage", "Warnings", "Total"])
+            coverage_sheet.append(metadata)
+
         workbook.save(output)
 
     output.seek(0)

counter_api/extensions/openapi.py

@@ -12,7 +12,37 @@ class RankingGroupSerializer(serializers.Serializer):
     items = RankingItemSerializer(many=True)
 
 
-class RankingSerializer(serializers.Serializer):
+class CoverageMonthSerializer(serializers.Serializer):
+    month = serializers.CharField()
+    status = serializers.ChoiceField(
+        choices=["complete", "partial", "unavailable", "unknown"]
+    )
+    confirmed_days = serializers.ListField(child=serializers.IntegerField())
+    missing_days = serializers.ListField(
+        child=serializers.IntegerField(), allow_null=True
+    )
+
+
+class CoverageSerializer(serializers.Serializer):
+    policy = serializers.CharField()
+    status = serializers.ChoiceField(choices=["complete", "partial"])
+    basis = serializers.ChoiceField(
+        choices=["confirmed_days", "available_annual_totals"]
+    )
+    included_months = serializers.ListField(
+        child=serializers.CharField(),
+        allow_null=True,
+        help_text="Null for annual totals: coverage is informative, not a day filter.",
+    )
+    months = CoverageMonthSerializer(many=True)
+
+
+class CoverageResponseSerializer(serializers.Serializer):
+    coverage = CoverageSerializer(required=False)
+    warnings = serializers.ListField(child=serializers.CharField(), required=False)
+
+
+class RankingSerializer(CoverageResponseSerializer):
     platform = serializers.CharField()
     begin_date = serializers.DateField()
     end_date = serializers.DateField()
@@ -29,7 +59,7 @@ class DistributionBucketSerializer(serializers.Serializer):
     count = serializers.IntegerField()
 
 
-class DistributionSerializer(serializers.Serializer):
+class DistributionSerializer(CoverageResponseSerializer):
     platform = serializers.CharField()
     begin_date = serializers.DateField()
     end_date = serializers.DateField()

counter_api/extensions/query.py

@@ -9,21 +9,22 @@
     service_unavailable,
 )
 from counter_api.extensions.contracts import SEGMENT_DIMENSIONS
+from counter_api.extensions.coverage import metric_sum
 from metrics.opensearch.names import (
     generate_analytics_index_name,
     generate_month_index_name,
 )
 
 
-def metric_composite_body(filters, sources, metric_type, page_size):
+def metric_composite_body(filters, sources, metric_type, page_size, coverage=None):
     return {
         "size": 0,
         "track_total_hits": False,
         "query": {"bool": {"filter": filters}},
         "aggs": {
             "rows": {
                 "composite": {"size": page_size, "sources": sources},
-                "aggs": {"count": {"sum": {"field": METRICS[metric_type]}}},
+                "aggs": {"count": {"sum": metric_sum(METRICS[metric_type], coverage)}},
             }
         },
     }
@@ -49,7 +50,7 @@ def validate_segmented_period(begin, end, dimensions, metric_type):
     return True
 
 
-def usage_scope(platform, begin, end, segmented, data_type=None):
+def usage_scope(platform, begin, end, segmented, data_type=None, included_months=None):
     if segmented:
         index_name = generate_analytics_index_name(
             settings.OPENSEARCH_INDEX_NAME,
@@ -62,7 +63,11 @@ def usage_scope(platform, begin, end, segmented, data_type=None):
             settings.OPENSEARCH_INDEX_NAME,
             platform.acron3,
         )
-        periods = [month.strftime("%Y-%m") for month in iter_months(begin, end)]
+        periods = (
+            included_months
+            if included_months is not None
+            else [month.strftime("%Y-%m") for month in iter_months(begin, end)]
+        )
         filters = [{"terms": {"month": periods}}]
 
     filters.extend(