diff --git a/.github/workflows/data-processing.yml b/.github/workflows/data-processing.yml index 6500c7666d6..8b45c49706f 100644 --- a/.github/workflows/data-processing.yml +++ b/.github/workflows/data-processing.yml @@ -60,6 +60,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 # Checkout the repository code to the runner environment + with: + # Full history: the generators read per-file commit dates to bootstrap + # `lastmod`, which a shallow clone would report as HEAD for every file. + fetch-depth: 0 #====================== # Workflow Configuration @@ -224,12 +228,44 @@ jobs: continue-on-error: true run: python3 scripts/build_get_involved.py + #======================================== + # Fetch the previous generated curated resources from the build-resources + # branch, so unchanged entries keep their lastmod date. + # If the branch or the extraction is unavailable, the directory stays + # absent and resource.py falls back to the seed copy in the checkout -- + # which silently re-dates every drifted resource on every run, so the + # step logs the file count to make that fallback visible in the log. + #======================================== + - name: Fetch previous curated resources state + id: previous-curated-resources + continue-on-error: true + run: | + mkdir -p /tmp/previous-generated + # Explicit refspec: actions/checkout may configure a narrow + # remote.origin.fetch, in which case a bare `git fetch origin + # build-resources` updates FETCH_HEAD without creating the + # origin/build-resources tracking ref that git archive needs. + git fetch origin +refs/heads/build-resources:refs/remotes/origin/build-resources || true + git archive origin/build-resources content/curated_resources \ + | tar -x -C /tmp/previous-generated || true + + PREV_DIR=/tmp/previous-generated/content/curated_resources + if [ -d "$PREV_DIR" ]; then + echo "✅ Previous curated resources: $(find "$PREV_DIR" -name '*.md' | wc -l) files" + else + echo "⚠️ build-resources state unavailable - falling back to the" + echo " seed copy in this checkout. Resources that have drifted" + echo " from the seed will be stamped with today's date." + fi + #======================================== # Process and organize curated resources data #======================================== - name: Run Curated Resources script id: curated-resources continue-on-error: true # Continue even if this step fails + env: + CURATED_PREVIOUS_DIR: /tmp/previous-generated/content/curated_resources run: python3 content/resources/resource.py # Execute the curated resources script that processes and organizes resource data diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 2b5a3cec30f..efe3d603560 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -41,6 +41,12 @@ jobs: with: fetch-depth: 0 + # Hugo derives page dates from `git log --name-only`, and Git quotes + # non-ASCII paths (glossary terms, accented resource titles) unless this + # is off. Quoted paths never match, leaving those pages without a date. + - name: Let Git report non-ASCII paths unquoted + run: git config core.quotepath false + # ======================= # Data Artifact Retrieval # ======================= @@ -208,6 +214,17 @@ jobs: env: HUGO_ENV: production + # ======================= + # Sitemap Quality Check + # ======================= + # Reports pages missing a , a timestamp stamped on most of the + # site, and a sample of pages whose date disagrees with their source + # (front-matter lastmod or Git). Warns only: a stale sitemap date must + # not block a deployment. + - name: Check sitemap lastmod + continue-on-error: true + run: python3 scripts/check_sitemap_lastmod.py public/sitemap.xml --compare-source --sample 200 + # ======================= # Deployment Artifact #======================================== diff --git a/.github/workflows/staging-aggregate.yaml b/.github/workflows/staging-aggregate.yaml index f4a817d08b5..f225775f6ee 100644 --- a/.github/workflows/staging-aggregate.yaml +++ b/.github/workflows/staging-aggregate.yaml @@ -263,6 +263,12 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ needs.aggregate-prs.outputs.aggregated-branch || 'main' }} + fetch-depth: 0 # full history so Hugo's Git-derived dates match production + + # See deploy.yaml: Git quotes non-ASCII paths unless this is off, and + # Hugo then finds no date for them. + - name: Let Git report non-ASCII paths unquoted + run: git config core.quotepath false - name: Configure Git run: | diff --git a/config/_default/config.toml b/config/_default/config.toml index 0a4c8fb3a18..b74ce656e62 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -113,3 +113,19 @@ ignoreFiles = ["\\.ipynb$", ".ipynb_checkpoints$", "\\.Rmd$", "\\.Rmarkdown$", " [build] no404check = true timeout = 60 + +# Date resolution order for front matter. +# +# NOTE: keep this table at the end of the file. A TOML table header captures +# every bare key that follows it, so placing this higher up would silently +# demote the top-level settings below it (enableRobotsTXT, removePathAccents, +# summaryLength, ...) into `frontmatter.*`, where Hugo ignores them. +# +# An explicit `lastmod` in front matter wins over the Git commit date (Hugo's +# default puts `:git` first). Generated collections (glossary, curated +# resources) carry their own `lastmod`, set by their generators only when an +# entry's content changes; hand-maintained pages without one fall back to Git. +# Requires `core.quotepath=false` in the build checkout so Git reports +# non-ASCII filenames unquoted (see deploy.yaml). +[frontmatter] + lastmod = ["lastmod", ":git", "date", "publishDate"] diff --git a/content/glossary/_create_glossaries.py b/content/glossary/_create_glossaries.py index 479f682685f..c0e17714262 100755 --- a/content/glossary/_create_glossaries.py +++ b/content/glossary/_create_glossaries.py @@ -3,10 +3,15 @@ import json import pandas as pd import os +import sys from io import StringIO +from pathlib import Path from pypinyin import lazy_pinyin, Style script_dir = os.path.dirname(os.path.abspath(__file__)) + +sys.path.insert(0, str(Path(script_dir).resolve().parents[1] / 'scripts')) +from generated_lastmod import load_previous, resolve_lastmod, git_commit_dates language_map = { 'EN': 'english', 'AR': 'arabic', @@ -289,6 +294,14 @@ def clean_filename(title, max_length=200): # Create markdown files for language_name, entries in formatted_data.items(): language_dir = os.path.join(script_dir, language_name) + + # Read the previous entries before deleting them, so each term can keep its + # `lastmod` for as long as its content is unchanged. Commit dates are + # collected in one pass up front; they only bootstrap entries written + # before `lastmod` existed. + previous_entries = load_previous(language_dir) + commit_dates = git_commit_dates(language_dir) + # Remove existing glossary entry files to ensure deleted entries don't persist # Preserve _index.md files as they are not regenerated if os.path.exists(language_dir): @@ -304,7 +317,13 @@ def clean_filename(title, max_length=200): for entry in entries: file_name = clean_filename(entry['title']) file_path = os.path.join(language_dir, file_name + ".md") - + + entry["lastmod"] = resolve_lastmod( + entry, + previous_entries.get(file_name + ".md"), + lambda: commit_dates.get(file_name + ".md"), + ) + with open(file_path, 'w', encoding='utf-8') as f: json.dump(entry, f, ensure_ascii=False, indent=4) diff --git a/content/resources/resource.py b/content/resources/resource.py index 6066c7b2efd..fd61791b1ca 100644 --- a/content/resources/resource.py +++ b/content/resources/resource.py @@ -17,10 +17,44 @@ """ import json -import pandas as pd +import os import re +import sys from pathlib import Path +import pandas as pd +from dateutil import parser as date_parser + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'scripts')) +from generated_lastmod import load_previous, resolve_lastmod + + +def previous_directory(fpath: Path) -> Path: + """Where the previous run's files are. + + The daily workflow points CURATED_PREVIOUS_DIR at a checkout of the + `build-resources` branch, which holds the last generated state. Without it + (local runs, or a missing branch) the checkout's own seed copy is used. + """ + env_dir = os.environ.get('CURATED_PREVIOUS_DIR') + if env_dir and Path(env_dir).is_dir(): + return Path(env_dir) + return fpath + + +def submission_date(timestamp) -> str: + """Sheet submission date as YYYY-MM-DD, or None if it cannot be parsed. + + Used as the `lastmod` for entries carried over from before this key + existed. The sheet mixes ISO and US month-first formats. + """ + if not timestamp or not isinstance(timestamp, str): + return None + try: + return date_parser.parse(timestamp).strftime('%Y-%m-%d') + except (ValueError, OverflowError): + return None + ## Function definition @@ -87,6 +121,10 @@ def convert_row_to_file(df, fpath): If there are duplicates, an index is appended to the filename. """ + # Read the previous run's files before deleting them, so each resource can + # keep its `lastmod` for as long as its content is unchanged. + previous = load_previous(previous_directory(fpath)) + # Replace the generated collection rather than leaving files for rows that # have been removed from the source sheet. for path in fpath.iterdir(): @@ -108,8 +146,14 @@ def convert_row_to_file(df, fpath): else: filename_counts[filename] = 1 filename_md = fpath / f"{filename}.md" - - filename_md.write_text(json.dumps(row.to_dict(), indent=4)) + + entry = row.to_dict() + entry["lastmod"] = resolve_lastmod( + entry, + previous.get(filename_md.name), + lambda: submission_date(entry.get("timestamp")), + ) + filename_md.write_text(json.dumps(entry, indent=4)) # Import data and prettify it: diff --git a/scripts/check_sitemap_lastmod.py b/scripts/check_sitemap_lastmod.py new file mode 100644 index 00000000000..7bbaa6d776e --- /dev/null +++ b/scripts/check_sitemap_lastmod.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Check that sitemap.xml carries a plausible for every URL. + +Two levels of checking: + +* Structural (always): counts URLs without (grouped by the first path + segment) and lists the most widely shared timestamps. A timestamp on more + than --max-bulk-share of all URLs fails the check: that is the signature of + every page being stamped with the build or checkout time. Smaller blocks are + reported but tolerated, since one commit can legitimately touch a whole + generated collection. + + Do not lower --max-bulk-share below 0.5 hoping to catch a stale block: the + two generated collections are 47% (curated_resources) and 41% (glossary) of + the site, and regenerating either in one commit produces exactly the same + shape as the bug. This check only catches a *site-wide* stamp; a single + collection frozen on a stale date is caught by the generators writing their + own `lastmod` and by --compare-source verifying it reaches the sitemap. +* Source comparison (--compare-source): rebuilds the expected lastmod for each + page from the repository -- explicit front-matter `lastmod`, else the Git + commit date of the source file -- and reports disagreements. Needs `hugo` + and the full Git history. + +Usage: + python3 scripts/check_sitemap_lastmod.py public/sitemap.xml + python3 scripts/check_sitemap_lastmod.py https://forrt.org/sitemap.xml + python3 scripts/check_sitemap_lastmod.py public/sitemap.xml --compare-source + +Exits non-zero when a check fails, so it can gate a build. +""" + +import argparse +import csv +import io +import json +import random +import re +import subprocess +import sys +import urllib.request +import xml.etree.ElementTree as ET +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlparse + +REPO = Path(__file__).resolve().parents[1] +SM_NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}" +YAML_FIELD = r"^{key}:\s*[\"']?([^\"'\n#]+)" + + +def load_sitemap(source): + """Return [(loc, lastmod_or_None)] from a local path or an http(s) URL.""" + if source.startswith(("http://", "https://")): + with urllib.request.urlopen(source, timeout=60) as response: + data = response.read() + else: + data = Path(source).read_bytes() + root = ET.fromstring(data) + entries = [] + for url in root.iter(SM_NS + "url"): + loc = url.findtext(SM_NS + "loc") or "" + lastmod = url.findtext(SM_NS + "lastmod") + entries.append((loc.strip(), (lastmod or "").strip() or None)) + return entries + + +def section_of(loc): + path = urlparse(loc).path.strip("/") + return path.split("/")[0] if path else "(home)" + + +def to_instant(value): + """Parse an ISO date or datetime into an aware datetime, or None.""" + if not value: + return None + text = value.strip().replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed + + +def check_structure(entries, max_missing, max_bulk_share): + total = len(entries) + print(f"URLs in sitemap: {total}") + if not total: + return False + + missing = [loc for loc, lastmod in entries if lastmod is None] + print(f"\nURLs without : {len(missing)} (limit {max_missing})") + by_section = defaultdict(list) + for loc in missing: + by_section[section_of(loc)].append(loc) + for name, locs in sorted(by_section.items(), key=lambda kv: -len(kv[1])): + print(f" {name}: {len(locs)} e.g. {', '.join(locs[:3])}") + + counts = Counter(lastmod for _, lastmod in entries if lastmod) + limit = max_bulk_share * total + bulk = [(stamp, n) for stamp, n in counts.most_common() if n > limit] + print(f"\nMost widely shared timestamps (fail above {max_bulk_share:.0%} of URLs):") + # Every offender is printed, not just the leaders: a block ranked below the + # top few can still fail the check, and a silent failure is unactionable. + for stamp, n in dict.fromkeys(counts.most_common(3) + bulk): + sections = Counter( + section_of(loc) for loc, lastmod in entries if lastmod == stamp + ) + where = ", ".join(f"{name} ({c})" for name, c in sections.most_common(3)) + flag = " <-- FAILS" if n > limit else "" + print(f" {stamp}: {n} URLs -- {where}{flag}") + + return len(missing) <= max_missing and not bulk + + +def hugo_pages(): + """Map URL path -> source path for every regular page, via `hugo list all`.""" + result = subprocess.run( + ["hugo", "list", "all"], + cwd=REPO, + capture_output=True, + text=True, + check=True, + ) + pages = {} + for row in csv.DictReader(io.StringIO(result.stdout)): + if row.get("kind") != "page": + continue + pages[urlparse(row["permalink"]).path] = row["path"] + return pages + + +def front_matter_field(path, key): + """Value of a front-matter `key` (JSON or YAML front matter), or None.""" + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None + stripped = text.lstrip() + if stripped.startswith("{"): + try: + data, _ = json.JSONDecoder().raw_decode(stripped) + except ValueError: + return None + value = data.get(key) if isinstance(data, dict) else None + return str(value) if value else None + if stripped.startswith("---"): + end = stripped.find("\n---", 3) + head = stripped[:end] if end > 0 else stripped + match = re.search(YAML_FIELD.format(key=key), head, re.MULTILINE) + return match.group(1).strip() if match else None + return None + + +def git_dates(rel_path): + """(author date, committer date) of the last commit touching the file.""" + result = subprocess.run( + ["git", "-c", "core.quotepath=false", "log", "-1", "--format=%aI\t%cI", + "--", rel_path], + cwd=REPO, + capture_output=True, + text=True, + ) + parts = result.stdout.strip().split("\t") + return tuple(to_instant(p) for p in parts) if len(parts) == 2 else (None, None) + + +def check_against_source(entries, sample_size): + pages = hugo_pages() + candidates = sorted( + (loc, lastmod) for loc, lastmod in entries + if urlparse(loc).path in pages + ) + print(f"\nSitemap URLs matching a regular page: {len(candidates)}") + if not candidates: + print("No sitemap URL maps to a page: `hugo list all` output not usable") + return False + if sample_size and sample_size < len(candidates): + # Seeded on the date so a run is reproducible while the sample rotates + # between deploys: a fixed seed would check the same few hundred pages + # forever and never look at the rest of the site. + seed = datetime.now(timezone.utc).strftime("%Y-%m-%d") + candidates = random.Random(seed).sample(candidates, sample_size) + print(f"Checking a random sample of {len(candidates)} (seed {seed})") + + mismatches = [] + for loc, lastmod in candidates: + # Mirrors the Hugo `[frontmatter] lastmod` order: lastmod, then Git, + # then date (files overlaid from a build artifact have no Git history). + rel = pages[urlparse(loc).path] + explicit = front_matter_field(REPO / rel, "lastmod") + if explicit: + expected_raw = explicit + alternatives = [to_instant(explicit)] + else: + author, committer = git_dates(rel) + if author: + expected_raw = author.isoformat() + alternatives = [author, committer] + else: + expected_raw = front_matter_field(REPO / rel, "date") + alternatives = [to_instant(expected_raw)] + alternatives = [d for d in alternatives if d] + actual = to_instant(lastmod) + if actual and any(abs((actual - d).total_seconds()) <= 1 for d in alternatives): + continue + mismatches.append((loc, lastmod, expected_raw, rel)) + + print(f"Mismatches: {len(mismatches)} of {len(candidates)} checked") + for loc, lastmod, expected_raw, rel in mismatches: + print(f" {loc}\n sitemap={lastmod} expected={expected_raw} source={rel}") + return not mismatches + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("sitemap", nargs="?", default="public/sitemap.xml") + parser.add_argument("--max-missing", type=int, default=5, + help="tolerated URLs without lastmod (default: 5)") + parser.add_argument("--max-bulk-share", type=float, default=0.5, + help="max share of URLs sharing one timestamp (default: 0.5)") + parser.add_argument("--compare-source", action="store_true", + help="also rebuild expected dates from the repository") + parser.add_argument("--sample", type=int, default=300, + help="pages to compare, 0 for all (default: 300)") + args = parser.parse_args() + + entries = load_sitemap(args.sitemap) + ok = check_structure(entries, args.max_missing, args.max_bulk_share) + if args.compare_source: + ok = check_against_source(entries, args.sample) and ok + print("\nOK" if ok else "\nFAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generated_lastmod.py b/scripts/generated_lastmod.py new file mode 100644 index 00000000000..e226c78717c --- /dev/null +++ b/scripts/generated_lastmod.py @@ -0,0 +1,100 @@ +"""Stable `lastmod` dates for generated JSON-front-matter content. + +Hugo prefers an explicit `lastmod` front-matter key over the Git commit date +(see `[frontmatter]` in config/_default/config.toml). Generators that rewrite +their whole content directory on every run therefore have to carry the date +forward themselves: an entry keeps its previous `lastmod` while its content is +unchanged, and gets today's date when the content differs. +""" + +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path + + +def today_utc() -> str: + return datetime.now(timezone.utc).strftime('%Y-%m-%d') + + +def load_previous(directory) -> dict: + """Read every generated `*.md` JSON file in `directory`, keyed by filename. + + `_index.md` is hand-written, not generated, and files that do not parse are + skipped so a stray file cannot abort a run. + """ + previous = {} + directory = Path(directory) + if not directory.is_dir(): + return previous + for path in sorted(directory.glob('*.md')): + if path.name == '_index.md': + continue + try: + previous[path.name] = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError): + continue + return previous + + +def _content_text(entry: dict) -> str: + """Serialized form of an entry, ignoring its `lastmod`. + + Compared as text rather than as dicts because curated resource rows carry + pandas NaN values, and `nan != nan` would mark every row as changed. + """ + without_lastmod = {k: v for k, v in entry.items() if k != 'lastmod'} + return json.dumps(without_lastmod, sort_keys=True, ensure_ascii=False) + + +def resolve_lastmod(new_entry: dict, previous_entry, bootstrap) -> str: + """Return the `YYYY-MM-DD` (UTC) date to record for `new_entry`. + + Unchanged content keeps the previous date; if the previous file predates + `lastmod`, `bootstrap()` supplies a date (today when it returns None). + """ + if previous_entry is None: + return today_utc() + if _content_text(previous_entry) != _content_text(new_entry): + return today_utc() + return previous_entry.get('lastmod') or bootstrap() or today_utc() + + +# Prefix that cannot collide with a path line, so date and path lines in +# `git log --name-only` output can be told apart without guessing. +_DATE_PREFIX = 'commit-date:' + + +def git_commit_dates(directory) -> dict: + """Map filename -> `YYYY-MM-DD` of the last commit touching it. + + A single `git log` walk over the directory rather than one subprocess per + file: bootstrapping a fresh collection means ~1,300 lookups, measured at + ~10s of pure process spawning locally and worse on a CI runner, against + ~0.1s for the batched walk. `core.quotepath=false` is required here -- + without it Git escapes the non-ASCII glossary filenames and none of them + match, which is the same defect this whole mechanism exists to fix. + + Returns an empty map (callers fall back to today) if Git is unavailable. + """ + directory = Path(directory) + cwd = directory if directory.is_dir() else directory.parent + try: + result = subprocess.run( + ['git', '-c', 'core.quotepath=false', 'log', + f'--format={_DATE_PREFIX}%cs', '--name-only', '--', '.'], + cwd=str(cwd), capture_output=True, text=True, check=True, + ) + except (OSError, subprocess.CalledProcessError): + return {} + + dates = {} + current = None + for line in result.stdout.splitlines(): + if line.startswith(_DATE_PREFIX): + current = line[len(_DATE_PREFIX):].strip() or None + elif line.strip() and current: + # git log walks newest-first, so the first sighting of a path is + # its most recent commit; later sightings are older history. + dates.setdefault(Path(line.strip()).name, current) + return dates