Amplia os períodos analíticos e pagina os rankings na versão 2.7.1

README.md

@@ -109,12 +109,29 @@ arquivos de configuração de ambiente que contenham credenciais.
 
 ## Cobertura parcial nas extensões
 
-Ranking e distribuição usam `coverage_policy=allow_partial` por padrão:
-contam dias confirmados no mensal e os totais disponíveis no anual de país/idioma.
-JSON, CSV e XLSX preservam a cobertura e os avisos, inclusive sem resultados.
-Use `coverage_policy=strict` para exigir cobertura completa, como antes.
-País/idioma exigem intervalos de anos inteiros; seus valores podem ser atualizados
-com novas cargas. A cobertura anual é informativa, não um filtro dos totais.
+Ranking e distribuição aceitam cortes diários e o mês corrente. Use datas
+`YYYY-MM-DD` ou meses `YYYY-MM`; o mês corrente como fim termina em hoje.
+Datas diárias futuras são rejeitadas. `coverage_policy=allow_partial` conta
+somente dias confirmados dentro do intervalo; meses completos usam os totais
+mensais. A cobertura distingue dias solicitados, confirmados, ausentes e futuros.
+Use `coverage_policy=strict` para exigir os meses inteiros processados, mesmo
+quando a consulta corta apenas alguns dias.
+
+País/idioma continuam anuais: selecione janeiro–dezembro, inclusive do ano
+corrente, para obter os totais disponíveis. A cobertura é informativa, não um
+filtro desses totais; métricas `Unique` não são suportadas nesses recortes.
+A descoberta de disponibilidade das extensões inclui meses com dias confirmados.
+
+Rankings com `group_by` paginam os grupos: `group_limit=5` por padrão (máximo 20),
+com `limit=100` itens por grupo (máximo 1000). O produto dos limites não pode
+exceder 10000. Continue com `pagination.next_cursor` e os mesmos filtros/período,
+mesmo se a página estiver vazia; `null` encerra a navegação. Formato e limites
+podem mudar entre páginas. Distribuições retornam até 10000 grupos; consultas
+maiores falham explicitamente, sem truncamento.
+
+JSON, CSV e XLSX preservam cobertura, avisos e cursor, inclusive sem resultados.
+No XLSX, `Coverage` contém o resumo e `Coverage months` detalha um mês por linha,
+com a indicação `Included`, evitando o limite de texto por célula do Excel.
 Consultas mensais sem dias confirmados e marcadores inconsistentes retornam erro.
 Relatórios COUNTER padrão não mudam. Consulte o OpenAPI para o contrato completo.
 

VERSION

@@ -1 +1 @@
-2.7.0
+2.7.1

counter_api/availability.py

@@ -12,27 +12,34 @@
 PAGE_SIZE = 1000
 
 
-def available_months(platform):
+def available_months(platform, complete_only=True):
     index_name = generate_month_status_index_name(settings.OPENSEARCH_INDEX_NAME)
     client = OpenSearchUsageClient().client
     deadline = monotonic() + settings.COUNTER_QUERY_BUDGET_SECONDS
     current_month = date.today().strftime("%Y-%m")
+    filters = [{"term": {"collection": platform.acron3}}]
+    if complete_only:
+        filters.extend(
+            [
+                {"term": {"complete": True}},
+                {"range": {"month": {"lt": current_month}}},
+            ]
+        )
+    else:
+        filters.extend(
+            [
+                {"range": {"day_mask": {"gt": 0}}},
+                {"range": {"month": {"lte": current_month}}},
+            ]
+        )
     months = []
     after = None
 
     while True:
         body = {
             "size": 0,
             "track_total_hits": False,
-            "query": {
-                "bool": {
-                    "filter": [
-                        {"term": {"collection": platform.acron3}},
-                        {"term": {"complete": True}},
-                        {"range": {"month": {"lt": current_month}}},
-                    ]
-                }
-            },
+            "query": {"bool": {"filter": filters}},
             "aggs": {
                 "months": {
                     "composite": {

counter_api/constants.py

@@ -3,6 +3,7 @@
 COMPOSITE_PAGE_SIZE = 1000
 METADATA_BATCH_SIZE = 1000
 EXCEL_MAX_ROWS = 1048576
+EXCEL_MAX_CELL_LENGTH = 32767
 SPOOL_SIZE = 8 * 1024 * 1024
 METRICS = {
     "Total_Item_Investigations": "total_investigations",

counter_api/dates.py

@@ -4,7 +4,7 @@
 from counter_api.exceptions import insufficient_information, invalid_dates
 
 
-def parse_period(params):
+def parse_period_inputs(params):
     begin_value = params.get("begin_date")
     end_value = params.get("end_date")
     if not begin_value or not end_value:
@@ -14,6 +14,11 @@ def parse_period(params):
     end = _parse_date(end_value, first=False)
     if begin > end:
         raise invalid_dates("end_date must not be earlier than begin_date")
+    return begin, end
+
+
+def parse_period(params):
+    begin, end = parse_period_inputs(params)
     if begin.day != 1:
         raise invalid_dates("begin_date must be the first day of a month")
     if end.day != calendar.monthrange(end.year, end.month)[1]:

counter_api/extensions/contracts.py

@@ -1,3 +1,4 @@
+MAX_RESULTS = 10000
 DOWNLOAD_FORMATS = ("json", "csv", "xlsx")
 SEGMENT_DIMENSIONS = {"country", "language"}
 RANKING_GROUPS = {"", "country", "language", "country,language"}

counter_api/extensions/coverage.py

@@ -1,4 +1,5 @@
 from calendar import monthrange
+from datetime import date
 
 from django.conf import settings
 from opensearchpy import NotFoundError
@@ -14,9 +15,13 @@ def resolve_coverage(client, platform, begin, end, policy, segmented, deadline):
     if policy not in {"strict", "allow_partial"}:
         raise insufficient_information()
 
-    if policy == "strict":
+    if policy == "strict" and (
+        segmented or (begin.day == 1 and end.day == monthrange(end.year, end.month)[1])
+    ):
         return None
 
+    today = date.today()
+
     months = {
         month.strftime("%Y-%m"): monthrange(month.year, month.month)[1]
         for month in iter_months(begin, end)
@@ -44,11 +49,20 @@ def resolve_coverage(client, platform, begin, end, policy, segmented, deadline):
     details = []
 
     for month, day_count in months.items():
-        days = range(1, day_count + 1)
+        month_date = date.fromisoformat(f"{month}-01")
+        requested = [
+            day
+            for day in range(1, day_count + 1)
+            if begin <= month_date.replace(day=day) <= end
+        ]
+        future = [day for day in requested if month_date.replace(day=day) > today]
+        elapsed = [day for day in requested if day not in future]
+
         marker = markers.get(month)
         confirmed = []
-        missing = None
-        status = "unknown"
+        missing = None if elapsed else []
+        status = "unknown" if elapsed else "future"
+        complete_month = False
 
         if marker is not None:
             mask = marker.get("day_mask")
@@ -62,30 +76,50 @@ def resolve_coverage(client, platform, begin, end, policy, segmented, deadline):
             ):
                 raise service_unavailable(f"Invalid coverage marker for {month}")
 
-            confirmed = [day for day in days if mask & (1 << (day - 1))]
-            missing = [day for day in days if day not in confirmed]
-            status = (
-                "complete" if not missing else "partial" if confirmed else "unavailable"
-            )
+            confirmed = [day for day in elapsed if mask & (1 << (day - 1))]
+            missing = [day for day in elapsed if day not in confirmed]
+            complete_month = mask == full and len(confirmed) == day_count
+
+            if not elapsed:
+                status = "future"
+            elif missing:
+                status = "partial" if confirmed else "unavailable"
+            else:
+                status = "ongoing" if future else "complete"
 
         details.append(
             {
                 "month": month,
                 "status": status,
+                "requested_days": requested,
                 "confirmed_days": confirmed,
                 "missing_days": missing,
+                "future_days": future,
+                "ongoing": bool(future),
+                "complete_month": complete_month,
             }
         )
 
     included = [item["month"] for item in details if item["confirmed_days"]]
-    complete = all(item["status"] == "complete" for item in details)
+    statuses = {item["status"] for item in details}
+    ongoing = any(item["future_days"] for item in details)
+
+    if statuses == {"complete"}:
+        status = "complete"
+    elif statuses == {"future"}:
+        status = "future"
+    elif statuses <= {"complete", "ongoing", "future"}:
+        status = "ongoing"
+    else:
+        status = "partial"
 
     if not segmented and not included:
         raise service_unavailable("No confirmed usage days in requested period")
 
     return {
         "policy": policy,
-        "status": "complete" if complete else "partial",
+        "status": status,
+        "ongoing": ongoing,
         "basis": "available_annual_totals" if segmented else "confirmed_days",
         "included_months": None if segmented else included,
         "months": details,
@@ -105,18 +139,22 @@ def coverage_fields(coverage):
         )
     elif coverage["status"] != "complete":
         warnings.append(
-            "Partial coverage: counts include only confirmed days; missing or unknown days are not zero usage."
+            "Partial coverage: counts include only confirmed days; missing or unknown days are not zero usage. "
+            "Future days are not missing usage."
         )
 
     return {"coverage": coverage, "warnings": warnings}
 
 
 def metric_sum(metric, coverage):
+    if coverage and coverage["basis"] == "available_annual_totals":
+        return {"field": metric}
+
     partial = (
         {
             item["month"]: [f"{day:02d}" for day in item["confirmed_days"]]
             for item in coverage["months"]
-            if item["status"] == "partial"
+            if item["confirmed_days"] and not item["complete_month"]
         }
         if coverage
         else {}
@@ -132,7 +170,7 @@ def metric_sum(metric, coverage):
                 "complete": [
                     item["month"]
                     for item in coverage["months"]
-                    if item["status"] == "complete"
+                    if item["complete_month"]
                 ],
                 "days": partial,
             },

counter_api/extensions/dates.py

@@ -0,0 +1,43 @@
+from calendar import monthrange
+from datetime import date
+
+from counter_api.dates import parse_period_inputs
+from counter_api.exceptions import invalid_dates
+from counter_api.extensions.contracts import SEGMENT_DIMENSIONS
+from counter_api.extensions.query import validate_segmented_period
+
+
+def parse_analytics_period(params):
+    begin, end = parse_period_inputs(params)
+    today = date.today()
+
+    dimensions = {
+        part.strip()
+        for name in ("group_by", "dimension")
+        for part in params.get(name, "").split(",")
+    }
+    dimensions.update(key for key in SEGMENT_DIMENSIONS if params.get(key))
+
+    segmented = validate_segmented_period(
+        begin, end, dimensions, params.get("metric_type", "Total_Item_Requests")
+    )
+
+    if segmented:
+        if begin.day != 1 or end.day != monthrange(end.year, end.month)[1]:
+            raise invalid_dates("Segmented periods must cover full calendar years")
+        if end.year > today.year:
+            raise invalid_dates("end_date must not be later than the current year")
+    else:
+        if (
+            len(params["end_date"]) == 7
+            and end.year == today.year
+            and end.month == today.month
+        ):
+            end = today
+
+        if end > today:
+            raise invalid_dates("end_date must not be later than today")
+        if begin > end:
+            raise invalid_dates("end_date must not be earlier than begin_date")
+
+    return begin, end

counter_api/extensions/distribution.py

@@ -3,11 +3,12 @@
 from django.conf import settings
 
 from counter_api.constants import METRICS
-from counter_api.exceptions import insufficient_information
+from counter_api.exceptions import insufficient_information, invalid_filter
 from counter_api.extensions.contracts import (
     DISTRIBUTION_CROSSES,
     DISTRIBUTION_DIMENSIONS,
     DISTRIBUTION_ENTITIES,
+    MAX_RESULTS,
 )
 from counter_api.extensions.coverage import coverage_fields, resolve_coverage
 from counter_api.extensions.query import (
@@ -21,7 +22,6 @@
 from counter_api.search import composite_pages
 
 PAGE_SIZE = 1000
-MAX_BUCKETS = 1000
 
 
 class DistributionQuery:
@@ -48,7 +48,7 @@ def run(self, platform, begin, end, params):
             segmented,
             deadline,
         )
-        if coverage is None:
+        if coverage is None or params.get("coverage_policy") == "strict":
             require_complete_period(platform, begin, end)
 
         key_field, entity_key = self._entity_filter(
@@ -210,8 +210,10 @@ def _buckets(self, index_name, body, dimensions, deadline):
                 for combination in self._combinations(bucket, dimensions, metadata):
                     totals[combination] = totals.get(combination, 0) + count
 
-                if len(totals) > MAX_BUCKETS:
-                    raise insufficient_information()
+                if len(totals) > MAX_RESULTS:
+                    raise invalid_filter(
+                        f"Distribution exceeds {MAX_RESULTS} groups; narrow the period or entity."
+                    )
 
         buckets = [
             {