From 867b0d6bc2b609093fc8d04afb081422525da8a4 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 14:54:52 +0200 Subject: [PATCH 01/11] refactor!: flat configs repo and signed release artifacts --- .agents/skills/html2rss-config/SKILL.md | 8 +- .../skills/html2rss-config/reference/batch.md | 6 +- .../html2rss-config/reference/catalog.md | 11 +- .../skills/html2rss-config/reference/new.md | 2 +- .../html2rss-config/reference/pitfalls.md | 2 +- .../html2rss-config/scripts/check_config | 12 +- .../scripts/register_botasaurus | 12 +- .github/workflows/release.yml | 37 +++ .github/workflows/ruby.yml | 43 +-- .gitignore | 5 +- .rspec | 2 +- .rubocop.yml | 14 +- .vscode/settings.json | 2 +- Gemfile.lock | 194 ------------ Makefile | 50 ++-- bin/console | 15 - bin/json_to_xml | 10 - bin/rspec_botasaurus_configs | 6 +- bin/rspec_changed_configs | 7 +- .../configs => configs}/aalto.fi/news.yml | 0 .../academictransfer.com/jobs.yml | 0 .../adfc.de/pressemitteilungen.yml | 0 .../ahram.org.eg/egypt.yml | 0 .../configs => configs}/ai.meta.com/blog.yml | 0 .../configs => configs}/akorda.kz/events.yml | 0 .../configs => configs}/angop.ao/noticias.yml | 0 .../anthropic.com/news.yml | 0 .../configs => configs}/anu.edu.au/news.yml | 0 .../configs => configs}/apnews.com/hub.yml | 0 .../apple.com/newsroom.yml | 0 .../aps.dz/national-news.yml | 0 .../asml.com/press-releases.yml | 0 .../aspi.org.au/strategist.yml | 0 .../bbc.co.uk/available_episodes.yml | 0 .../configs => configs}/bbc.com/mundo.yml | 0 .../blog.mondediplo.net/feed.yml | 0 .../brookings.edu/articles.yml | 0 .../bundesregierung.de/pressemitteilungen.yml | 0 .../canarianweekly.com/front.yml | 0 .../configs => configs}/caricom.org/news.yml | 0 .../carnegieendowment.org/research.yml | 0 .../cepal.org/press-releases.yml | 0 .../configs => configs}/chalmers.se/news.yml | 0 .../research-publications.yml | 0 .../cinemascore.com/index.yml | 0 .../cleanenergywire.org/news.yml | 0 .../cnet.com/section_sub.yml | 0 .../configs => configs}/cohere.com/blog.yml | 0 .../computerbase.de/meistgelesen.yml | 0 .../configs => configs}/cplp.org/noticias.yml | 0 .../configs => configs}/csiro.au/news.yml | 0 .../curia.europa.eu/press-releases.yml | 0 .../configs => configs}/cutle.fish/index.yml | 0 .../deepmind.google/blog.yml | 0 .../deraktionaer.de/meistgelesen.yml | 0 ...ials_data_documentation_technotes_json.yml | 0 .../dfs.de/pressemitteilungen.yml | 0 .../configs => configs}/diis.dk/research.yml | 0 .../dirco.gov.za/media-statements.yml | 0 .../dsw-info.de/presse.yml | 0 .../elasticsearch-release-notes.yml | 0 .../elysee.fr/actualites.yml | 0 .../configs => configs}/equinor.com/news.yml | 0 .../configs => configs}/espn.com/f1.yml | 0 .../configs => configs}/ethz.ch/eth-news.yml | 0 .../eures.europa.eu/news.yml | 0 .../configs => configs}/faa.gov/newsroom.yml | 0 .../configs => configs}/fao.org/newsroom.yml | 0 .../configs => configs}/fia.com/documents.yml | 0 .../configs => configs}/figma.com/blog.yml | 0 .../formula1.com/latest.yml | 0 .../forumsec.org/publications.yml | 0 .../ftc.gov/press-releases.yml | 0 .../github.com/releases.yml | 0 .../go.dev/release-history.yml | 0 .../grafana.com/whatsnew.yml | 0 .../haaretz.com/israel-news.yml | 0 .../hai.stanford.edu/news.yml | 0 .../configs => configs}/iaapa.org/news.yml | 0 .../icj-cij.org/press-releases.yml | 0 .../configs => configs}/icrc.org/news.yml | 0 .../configs => configs}/iea.org/news.yml | 0 .../iiss.org/online-analysis.yml | 0 .../configs => configs}/ikea.com/newsroom.yml | 0 .../configs => configs}/imdb.com/ratings.yml | 0 .../imo.org/press-briefings.yml | 0 .../imperial.ac.uk/news.yml | 0 .../karriere_arbeitsleben_heiko_mell.yml | 0 .../configs => configs}/isro.gov.in/press.yml | 0 .../issafrica.org/iss-today.yml | 0 .../issafrica.org/press-releases.yml | 0 .../configs => configs}/kaist.ac.kr/news.yml | 0 .../kinocheck.de/filmstarts.yml | 0 .../kompas.com/nasional.yml | 0 .../lego.com/press-releases.yml | 0 .../machinelearning.apple.com/research.yml | 0 .../maersk.com/press-releases.yml | 0 .../microsoft.com/azure-products.yml | 0 .../configs => configs}/mistral.ai/news.yml | 0 .../mozilla.org/security-advisories.yml | 0 .../configs => configs}/msf.org/latest.yml | 0 .../nbim.no/press-releases.yml | 0 .../configs => configs}/news.cn/latest.yml | 0 .../news24.com/top-stories.yml | 0 .../newyorker.com/magazine.yml | 0 .../nigerianstat.gov.ng/news.yml | 0 .../nikkei.com/asia-technology.yml | 0 .../nokia.com/newsroom.yml | 0 .../nomanssky.com/news.yml | 0 .../configs => configs}/notion.com/blog.yml | 0 .../novonordisk.com/latest-news.yml | 0 .../configs => configs}/ntu.edu.sg/news.yml | 0 .../configs => configs}/nupi.no/news.yml | 0 .../configs => configs}/occrp.org/news.yml | 0 .../configs => configs}/oecd.org/newsroom.yml | 0 .../opec.org/press-releases.yml | 0 .../panapress.com/latest.yml | 0 .../search.yml | 0 .../philstar.com/nation.yml | 0 .../configs => configs}/phys.org/weekly.yml | 0 .../pib.gov.in/press-releases.yml | 0 .../rbb24.de/meistgeklickt.yml | 0 .../robinwood.de/aktuelles.yml | 0 .../s3.amazonaws.com/popular_movies.yml | 0 .../configs => configs}/sacu.int/news.yml | 0 .../sadc.int/latest-news.yml | 0 .../sebastianvettel.de/news.yml | 0 .../configs => configs}/shopify.com/blog.yml | 0 .../configs => configs}/sipri.org/news.yml | 0 .../softwareleadweekly.com/issues.yml | 0 .../solarthermalworld.org/news.yml | 0 .../spektrum.de/meistgelesen.yml | 0 .../spiegel.de/impressum_autor.yml | 0 .../spotify.com/newsroom.yml | 0 .../hot_network_questions.yml | 0 .../steuerzahler.de/news.yml | 0 .../configs => configs}/stripes.com/index.yml | 0 .../support.apple.com/en_gb_ht201222.yml | 0 .../support.apple.com/exchange_repair.yml | 0 .../tap.info.tn/politics.yml | 0 .../telenor.com/newsroom.yml | 0 .../teneriffa-news.com/news.yml | 0 .../configs => configs}/tennet.eu/news.yml | 0 .../configs => configs}/test.de/archiv.yml | 0 .../theeastafrican.co.ke/news.yml | 0 .../international_mostpopular.yml | 0 .../thenationalnews.com/news.yml | 0 .../thoughtworks.com/insights.yml | 0 .../configs => configs}/tno.nl/news.yml | 0 .../aktuelle_nachrichten.yml | 0 .../configs => configs}/tralac.org/news.yml | 0 .../configs => configs}/tudelft.nl/news.yml | 0 .../configs => configs}/tue.nl/news.yml | 0 .../vattenfall.com/newsroom.yml | 0 .../visitfinland.com/press-releases.yml | 0 .../webentwickler-jobs.de/in.yml | 0 .../configs => configs}/who.int/news.yml | 0 .../worldbank.org/news.yml | 0 .../press-releases.yml | 0 .../www.ecdc.europa.eu/news-events.yml | 0 .../www.enisa.europa.eu/news.yml | 0 .../www.europarl.europa.eu/press-room.yml | 0 .../www.gob.mx/profeco-prensa.yml | 0 html2rss-configs.gemspec | 46 --- lib/html2rss/configs.rb | 38 --- lib/html2rss/configs/catalog.rb | 162 ---------- lib/html2rss/configs/catalog_entry.rb | 31 -- lib/html2rss/configs/version.rb | 7 - package-lock.json | 28 -- package.json | 6 - spec/html2rss/configs_dynamic_spec.rb | 27 -- spec/html2rss/configs_spec.rb | 45 --- spec/lib/html2rss/configs/catalog_spec.rb | 44 --- .../botasaurus_fetch_configs_spec.rb | 4 +- test/configs_dynamic_spec.rb | 24 ++ {spec => test}/helper_spec.rb | 0 test/registry.spec.rb | 42 +++ .../support/botasaurus_fetch_configs.rb | 0 {spec => test}/support/helper.rb | 0 .../shared_examples/config.yml_spec.rb | 8 +- spec/spec_helper.rb => test/test_helper.rb | 12 +- .../validate.spec.rb | 14 +- Gemfile => tool/Gemfile | 6 +- tool/Gemfile.lock | 281 ++++++++++++++++++ tool/registry-build | 136 +++++++++ tool/validate | 58 ++++ 186 files changed, 679 insertions(+), 778 deletions(-) create mode 100644 .github/workflows/release.yml delete mode 100644 Gemfile.lock delete mode 100755 bin/console delete mode 100755 bin/json_to_xml rename {lib/html2rss/configs => configs}/aalto.fi/news.yml (100%) rename {lib/html2rss/configs => configs}/academictransfer.com/jobs.yml (100%) rename {lib/html2rss/configs => configs}/adfc.de/pressemitteilungen.yml (100%) rename {lib/html2rss/configs => configs}/ahram.org.eg/egypt.yml (100%) rename {lib/html2rss/configs => configs}/ai.meta.com/blog.yml (100%) rename {lib/html2rss/configs => configs}/akorda.kz/events.yml (100%) rename {lib/html2rss/configs => configs}/angop.ao/noticias.yml (100%) rename {lib/html2rss/configs => configs}/anthropic.com/news.yml (100%) rename {lib/html2rss/configs => configs}/anu.edu.au/news.yml (100%) rename {lib/html2rss/configs => configs}/apnews.com/hub.yml (100%) rename {lib/html2rss/configs => configs}/apple.com/newsroom.yml (100%) rename {lib/html2rss/configs => configs}/aps.dz/national-news.yml (100%) rename {lib/html2rss/configs => configs}/asml.com/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/aspi.org.au/strategist.yml (100%) rename {lib/html2rss/configs => configs}/bbc.co.uk/available_episodes.yml (100%) rename {lib/html2rss/configs => configs}/bbc.com/mundo.yml (100%) rename {lib/html2rss/configs => configs}/blog.mondediplo.net/feed.yml (100%) rename {lib/html2rss/configs => configs}/brookings.edu/articles.yml (100%) rename {lib/html2rss/configs => configs}/bundesregierung.de/pressemitteilungen.yml (100%) rename {lib/html2rss/configs => configs}/canarianweekly.com/front.yml (100%) rename {lib/html2rss/configs => configs}/caricom.org/news.yml (100%) rename {lib/html2rss/configs => configs}/carnegieendowment.org/research.yml (100%) rename {lib/html2rss/configs => configs}/cepal.org/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/chalmers.se/news.yml (100%) rename {lib/html2rss/configs => configs}/chathamhouse.org/research-publications.yml (100%) rename {lib/html2rss/configs => configs}/cinemascore.com/index.yml (100%) rename {lib/html2rss/configs => configs}/cleanenergywire.org/news.yml (100%) rename {lib/html2rss/configs => configs}/cnet.com/section_sub.yml (100%) rename {lib/html2rss/configs => configs}/cohere.com/blog.yml (100%) rename {lib/html2rss/configs => configs}/computerbase.de/meistgelesen.yml (100%) rename {lib/html2rss/configs => configs}/cplp.org/noticias.yml (100%) rename {lib/html2rss/configs => configs}/csiro.au/news.yml (100%) rename {lib/html2rss/configs => configs}/curia.europa.eu/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/cutle.fish/index.yml (100%) rename {lib/html2rss/configs => configs}/deepmind.google/blog.yml (100%) rename {lib/html2rss/configs => configs}/deraktionaer.de/meistgelesen.yml (100%) rename {lib/html2rss/configs => configs}/developer.apple.com/tutorials_data_documentation_technotes_json.yml (100%) rename {lib/html2rss/configs => configs}/dfs.de/pressemitteilungen.yml (100%) rename {lib/html2rss/configs => configs}/diis.dk/research.yml (100%) rename {lib/html2rss/configs => configs}/dirco.gov.za/media-statements.yml (100%) rename {lib/html2rss/configs => configs}/dsw-info.de/presse.yml (100%) rename {lib/html2rss/configs => configs}/elastic.co/elasticsearch-release-notes.yml (100%) rename {lib/html2rss/configs => configs}/elysee.fr/actualites.yml (100%) rename {lib/html2rss/configs => configs}/equinor.com/news.yml (100%) rename {lib/html2rss/configs => configs}/espn.com/f1.yml (100%) rename {lib/html2rss/configs => configs}/ethz.ch/eth-news.yml (100%) rename {lib/html2rss/configs => configs}/eures.europa.eu/news.yml (100%) rename {lib/html2rss/configs => configs}/faa.gov/newsroom.yml (100%) rename {lib/html2rss/configs => configs}/fao.org/newsroom.yml (100%) rename {lib/html2rss/configs => configs}/fia.com/documents.yml (100%) rename {lib/html2rss/configs => configs}/figma.com/blog.yml (100%) rename {lib/html2rss/configs => configs}/formula1.com/latest.yml (100%) rename {lib/html2rss/configs => configs}/forumsec.org/publications.yml (100%) rename {lib/html2rss/configs => configs}/ftc.gov/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/github.com/releases.yml (100%) rename {lib/html2rss/configs => configs}/go.dev/release-history.yml (100%) rename {lib/html2rss/configs => configs}/grafana.com/whatsnew.yml (100%) rename {lib/html2rss/configs => configs}/haaretz.com/israel-news.yml (100%) rename {lib/html2rss/configs => configs}/hai.stanford.edu/news.yml (100%) rename {lib/html2rss/configs => configs}/iaapa.org/news.yml (100%) rename {lib/html2rss/configs => configs}/icj-cij.org/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/icrc.org/news.yml (100%) rename {lib/html2rss/configs => configs}/iea.org/news.yml (100%) rename {lib/html2rss/configs => configs}/iiss.org/online-analysis.yml (100%) rename {lib/html2rss/configs => configs}/ikea.com/newsroom.yml (100%) rename {lib/html2rss/configs => configs}/imdb.com/ratings.yml (100%) rename {lib/html2rss/configs => configs}/imo.org/press-briefings.yml (100%) rename {lib/html2rss/configs => configs}/imperial.ac.uk/news.yml (100%) rename {lib/html2rss/configs => configs}/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml (100%) rename {lib/html2rss/configs => configs}/isro.gov.in/press.yml (100%) rename {lib/html2rss/configs => configs}/issafrica.org/iss-today.yml (100%) rename {lib/html2rss/configs => configs}/issafrica.org/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/kaist.ac.kr/news.yml (100%) rename {lib/html2rss/configs => configs}/kinocheck.de/filmstarts.yml (100%) rename {lib/html2rss/configs => configs}/kompas.com/nasional.yml (100%) rename {lib/html2rss/configs => configs}/lego.com/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/machinelearning.apple.com/research.yml (100%) rename {lib/html2rss/configs => configs}/maersk.com/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/microsoft.com/azure-products.yml (100%) rename {lib/html2rss/configs => configs}/mistral.ai/news.yml (100%) rename {lib/html2rss/configs => configs}/mozilla.org/security-advisories.yml (100%) rename {lib/html2rss/configs => configs}/msf.org/latest.yml (100%) rename {lib/html2rss/configs => configs}/nbim.no/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/news.cn/latest.yml (100%) rename {lib/html2rss/configs => configs}/news24.com/top-stories.yml (100%) rename {lib/html2rss/configs => configs}/newyorker.com/magazine.yml (100%) rename {lib/html2rss/configs => configs}/nigerianstat.gov.ng/news.yml (100%) rename {lib/html2rss/configs => configs}/nikkei.com/asia-technology.yml (100%) rename {lib/html2rss/configs => configs}/nokia.com/newsroom.yml (100%) rename {lib/html2rss/configs => configs}/nomanssky.com/news.yml (100%) rename {lib/html2rss/configs => configs}/notion.com/blog.yml (100%) rename {lib/html2rss/configs => configs}/novonordisk.com/latest-news.yml (100%) rename {lib/html2rss/configs => configs}/ntu.edu.sg/news.yml (100%) rename {lib/html2rss/configs => configs}/nupi.no/news.yml (100%) rename {lib/html2rss/configs => configs}/occrp.org/news.yml (100%) rename {lib/html2rss/configs => configs}/oecd.org/newsroom.yml (100%) rename {lib/html2rss/configs => configs}/opec.org/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/panapress.com/latest.yml (100%) rename {lib/html2rss/configs => configs}/pankow.lebensmittel-kontrollergebnisse.de/search.yml (100%) rename {lib/html2rss/configs => configs}/philstar.com/nation.yml (100%) rename {lib/html2rss/configs => configs}/phys.org/weekly.yml (100%) rename {lib/html2rss/configs => configs}/pib.gov.in/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/rbb24.de/meistgeklickt.yml (100%) rename {lib/html2rss/configs => configs}/robinwood.de/aktuelles.yml (100%) rename {lib/html2rss/configs => configs}/s3.amazonaws.com/popular_movies.yml (100%) rename {lib/html2rss/configs => configs}/sacu.int/news.yml (100%) rename {lib/html2rss/configs => configs}/sadc.int/latest-news.yml (100%) rename {lib/html2rss/configs => configs}/sebastianvettel.de/news.yml (100%) rename {lib/html2rss/configs => configs}/shopify.com/blog.yml (100%) rename {lib/html2rss/configs => configs}/sipri.org/news.yml (100%) rename {lib/html2rss/configs => configs}/softwareleadweekly.com/issues.yml (100%) rename {lib/html2rss/configs => configs}/solarthermalworld.org/news.yml (100%) rename {lib/html2rss/configs => configs}/spektrum.de/meistgelesen.yml (100%) rename {lib/html2rss/configs => configs}/spiegel.de/impressum_autor.yml (100%) rename {lib/html2rss/configs => configs}/spotify.com/newsroom.yml (100%) rename {lib/html2rss/configs => configs}/stackoverflow.com/hot_network_questions.yml (100%) rename {lib/html2rss/configs => configs}/steuerzahler.de/news.yml (100%) rename {lib/html2rss/configs => configs}/stripes.com/index.yml (100%) rename {lib/html2rss/configs => configs}/support.apple.com/en_gb_ht201222.yml (100%) rename {lib/html2rss/configs => configs}/support.apple.com/exchange_repair.yml (100%) rename {lib/html2rss/configs => configs}/tap.info.tn/politics.yml (100%) rename {lib/html2rss/configs => configs}/telenor.com/newsroom.yml (100%) rename {lib/html2rss/configs => configs}/teneriffa-news.com/news.yml (100%) rename {lib/html2rss/configs => configs}/tennet.eu/news.yml (100%) rename {lib/html2rss/configs => configs}/test.de/archiv.yml (100%) rename {lib/html2rss/configs => configs}/theeastafrican.co.ke/news.yml (100%) rename {lib/html2rss/configs => configs}/theguardian.com/international_mostpopular.yml (100%) rename {lib/html2rss/configs => configs}/thenationalnews.com/news.yml (100%) rename {lib/html2rss/configs => configs}/thoughtworks.com/insights.yml (100%) rename {lib/html2rss/configs => configs}/tno.nl/news.yml (100%) rename {lib/html2rss/configs => configs}/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml (100%) rename {lib/html2rss/configs => configs}/tralac.org/news.yml (100%) rename {lib/html2rss/configs => configs}/tudelft.nl/news.yml (100%) rename {lib/html2rss/configs => configs}/tue.nl/news.yml (100%) rename {lib/html2rss/configs => configs}/vattenfall.com/newsroom.yml (100%) rename {lib/html2rss/configs => configs}/visitfinland.com/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/webentwickler-jobs.de/in.yml (100%) rename {lib/html2rss/configs => configs}/who.int/news.yml (100%) rename {lib/html2rss/configs => configs}/worldbank.org/news.yml (100%) rename {lib/html2rss/configs => configs}/www.consilium.europa.eu/press-releases.yml (100%) rename {lib/html2rss/configs => configs}/www.ecdc.europa.eu/news-events.yml (100%) rename {lib/html2rss/configs => configs}/www.enisa.europa.eu/news.yml (100%) rename {lib/html2rss/configs => configs}/www.europarl.europa.eu/press-room.yml (100%) rename {lib/html2rss/configs => configs}/www.gob.mx/profeco-prensa.yml (100%) delete mode 100644 html2rss-configs.gemspec delete mode 100644 lib/html2rss/configs.rb delete mode 100644 lib/html2rss/configs/catalog.rb delete mode 100644 lib/html2rss/configs/catalog_entry.rb delete mode 100644 lib/html2rss/configs/version.rb delete mode 100644 package-lock.json delete mode 100644 package.json delete mode 100644 spec/html2rss/configs_dynamic_spec.rb delete mode 100644 spec/html2rss/configs_spec.rb delete mode 100644 spec/lib/html2rss/configs/catalog_spec.rb rename {spec => test}/botasaurus_fetch_configs_spec.rb (85%) create mode 100644 test/configs_dynamic_spec.rb rename {spec => test}/helper_spec.rb (100%) create mode 100644 test/registry.spec.rb rename {spec => test}/support/botasaurus_fetch_configs.rb (100%) rename {spec => test}/support/helper.rb (100%) rename {spec => test}/support/shared_examples/config.yml_spec.rb (93%) rename spec/spec_helper.rb => test/test_helper.rb (58%) rename spec/bin/validate_configs_spec.rb => test/validate.spec.rb (71%) rename Gemfile => tool/Gemfile (67%) create mode 100644 tool/Gemfile.lock create mode 100755 tool/registry-build create mode 100755 tool/validate diff --git a/.agents/skills/html2rss-config/SKILL.md b/.agents/skills/html2rss-config/SKILL.md index 9081648f..748b7eea 100644 --- a/.agents/skills/html2rss-config/SKILL.md +++ b/.agents/skills/html2rss-config/SKILL.md @@ -1,7 +1,7 @@ --- name: html2rss-config description: >- - Create or repair curated html2rss YAML feed configs in this repo (lib/html2rss/configs/), + Create or repair curated html2rss YAML feed configs in this repo (configs/), including directory.topics, directory.title, selectors, Faraday vs Botasaurus triage, RSS probe-before-write, and the AGENTS.md quality gate. Use when adding a new feed config, fixing a broken/zero-item config, tightening selectors, diagnosing fetch failures, or shipping a user-requested batch @@ -17,7 +17,7 @@ Thin workflow skill for **one config quality loop at a time** (a multi-config PR | Mode | When | Reference | | -------- | -------------------------------------------------------- | ------------------------------------------ | -| `new` | Add a YAML under `lib/html2rss/configs//` | [reference/new.md](reference/new.md) | +| `new` | Add a YAML under `configs//` | [reference/new.md](reference/new.md) | | `repair` | Fix existing config (zero items, fetch fail, noisy feed) | [reference/repair.md](reference/repair.md) | Pick mode from the user ask. Grow later with more modes/references; keep this file short. @@ -55,7 +55,7 @@ Run from repo root. Prefer these over ad‑hoc CLI glue: | [`scripts/analyze_html`](scripts/analyze_html) | Selector hints from cached HTML / ledger (`--from-ledger`). No network. | | [`scripts/probe_rss`](scripts/probe_rss) | First-party RSS probe (HTML `rel=alternate` then path guesses). Exit `0` = none; `3` = found (consider drop). Under `set -e`, check `$?` — do not treat `3` as failure. | | [`scripts/check_config`](scripts/check_config) | `validate` + `feed` (fail on 0 items); optional `--fetch` / `--botasaurus`. Resolves CLI via PATH or sibling `../html2rss`. | -| [`scripts/register_botasaurus`](scripts/register_botasaurus) | Idempotent sorted add to `spec/support/botasaurus_fetch_configs.rb`. | +| [`scripts/register_botasaurus`](scripts/register_botasaurus) | Idempotent sorted add to `test/support/botasaurus_fetch_configs.rb`. | Examples: @@ -76,7 +76,7 @@ From AGENTS.md Quality Gate, in order: 2. `make validate` (this repo) when touching shared support files or multiple configs 3. `make test` (non-fetch) 4. Focused fetch via `scripts/check_config … --fetch` or: - - Faraday: `bundle exec rspec --tag fetch --example 'domain/file.yml' spec/html2rss/configs_dynamic_spec.rb` + - Faraday: `bundle exec rspec --tag fetch --example 'domain/file.yml' test/configs_dynamic_spec.rb` - Botasaurus: same with `BOTASAURUS_SCRAPER_URL=http://localhost:4010` 5. If `strategy: botasaurus` (or fetch only works via Botasaurus): `scripts/register_botasaurus domain/file.yml` — **required**. diff --git a/.agents/skills/html2rss-config/reference/batch.md b/.agents/skills/html2rss-config/reference/batch.md index 08c6b9f6..aa0c05b2 100644 --- a/.agents/skills/html2rss-config/reference/batch.md +++ b/.agents/skills/html2rss-config/reference/batch.md @@ -44,7 +44,7 @@ Dry plan (no network): `batch_recon --dry-run --file candidates.tsv` --from-ledger tmp/html2rss-recon/ledger.tsv ``` -Write YAMLs only for `BUILD` rows under `lib/html2rss/configs//`. Use MCP/`check_config` only when cache analysis is insufficient. Botasaurus scrape: thin/empty/blocked HTML only — one retry max (`wait_timeout_seconds` ≤ 30, work budget), then drop. +Write YAMLs only for `BUILD` rows under `configs//`. Use MCP/`check_config` only when cache analysis is insufficient. Botasaurus scrape: thin/empty/blocked HTML only — one retry max (`wait_timeout_seconds` ≤ 30, work budget), then drop. YAML notes: [new.md](new.md). Topics: [topics.md](topics.md). @@ -52,7 +52,7 @@ YAML notes: [new.md](new.md). Topics: [topics.md](topics.md). ```bash # offline validate (example: sibling CLI) -html2rss validate lib/html2rss/configs/domain/*.yml +html2rss validate configs/domain/*.yml # parallel feed checks (Faraday group) .agents/skills/html2rss-config/scripts/check_config domain/a.yml & @@ -63,7 +63,7 @@ wait bundle exec rspec --tag fetch \ --example 'domain/a.yml' \ --example 'domain/b.yml' \ - spec/html2rss/configs_dynamic_spec.rb + test/configs_dynamic_spec.rb ``` ## Phase 3 — campaign gate diff --git a/.agents/skills/html2rss-config/reference/catalog.md b/.agents/skills/html2rss-config/reference/catalog.md index a289b97c..4b129029 100644 --- a/.agents/skills/html2rss-config/reference/catalog.md +++ b/.agents/skills/html2rss-config/reference/catalog.md @@ -1,6 +1,6 @@ # Catalog metadata (Feed Directory) -Every YAML in this repo is a catalog entry. Embedded serialization is owned by `Html2rss::Configs::Catalog` — consumers call `Catalog.entries`; do not re-walk YAML in other repos. +Every YAML in this repo is a catalog entry. Registry bundle serialization is owned by `Html2rss::Registry::CatalogBuilder` in the core `html2rss` gem — build with `make registry-build`; do not re-walk YAML in other repos. ## Required fields @@ -44,17 +44,18 @@ Describe feed intent in `directory.title`, not the template URL. ## Verification ```bash -bundle exec rspec spec/lib/html2rss/configs/catalog_spec.rb +bundle exec rspec test/registry.spec.rb make validate +make registry-build ``` -`Catalog.build_entry` raises `MissingDirectoryTitle` when `directory.title` is absent. +`CatalogBuilder.build_entry` raises `MissingDirectoryTitle` when `directory.title` is absent. ## Downstream consumers | Consumer | How it reads catalog data | | -------------------------- | ------------------------------------------------------------------------------------------------------------ | -| `html2rss-web` | `GET /api/v1/configs` merges `Catalog.entries` with local `feeds.yml` entries that include `directory.title` | +| `html2rss-web` | `GET /api/v1/configs` merges registry bundles with local `feeds.yml` entries that include `directory.title` | | Feed Directory (docs site) | Fetches catalog JSON from a running `html2rss-web` instance | -Do not duplicate catalog expansion logic outside `Html2rss::Configs::Catalog`. +Do not duplicate catalog expansion logic outside `Html2rss::Registry::CatalogBuilder`. diff --git a/.agents/skills/html2rss-config/reference/new.md b/.agents/skills/html2rss-config/reference/new.md index ada7a4aa..f2be6cde 100644 --- a/.agents/skills/html2rss-config/reference/new.md +++ b/.agents/skills/html2rss-config/reference/new.md @@ -7,7 +7,7 @@ Add one curated config. SSOT details: [AGENTS.md](../../../../AGENTS.md). 1. Pick the cleanest list URL (newsroom / archive / category — not marketing homepage). Confirm with `curl -I -L` (canonical host; no HTTPS→HTTP downgrade unless you plan Botasaurus). 2. Confirm no useful first-party RSS **for that exact URL** (else drop/defer). Use `scripts/probe_rss`. Exit `3` = feed found. 3. Capture items via skill tool order (MCP → CLI → Botasaurus → Chrome). If `auto` is empty, still inspect HTML before assuming JS-only — see [pitfalls.md](pitfalls.md). -4. Write YAML under `lib/html2rss/configs//.yml`. +4. Write YAML under `configs//.yml`. 5. Run `scripts/check_config …` (and `--fetch` / `--botasaurus` as needed); verify real `` rows, not only the summary. `scripts/register_botasaurus` if Botasaurus-backed. 6. Handoff per skill. diff --git a/.agents/skills/html2rss-config/reference/pitfalls.md b/.agents/skills/html2rss-config/reference/pitfalls.md index 834342f2..f25d80d9 100644 --- a/.agents/skills/html2rss-config/reference/pitfalls.md +++ b/.agents/skills/html2rss-config/reference/pitfalls.md @@ -33,7 +33,7 @@ Present constraints for config campaigns (N≥1). Quality-gate commands stay in bundle exec rspec --tag fetch \ --example 'domain/a.yml' \ --example 'domain/b.yml' \ - spec/html2rss/configs_dynamic_spec.rb + test/configs_dynamic_spec.rb ``` Botasaurus lane: same command with `BOTASAURUS_SCRAPER_URL=http://localhost:4010`. diff --git a/.agents/skills/html2rss-config/scripts/check_config b/.agents/skills/html2rss-config/scripts/check_config index 0bb00da9..a4d43b4f 100755 --- a/.agents/skills/html2rss-config/scripts/check_config +++ b/.agents/skills/html2rss-config/scripts/check_config @@ -3,7 +3,7 @@ # Validate + feed (+ optional focused fetch) for one curated config. # Usage (from repo root): -# .agents/skills/html2rss-config/scripts/check_config lib/html2rss/configs/domain/file.yml +# .agents/skills/html2rss-config/scripts/check_config configs/domain/file.yml # .agents/skills/html2rss-config/scripts/check_config domain/file.yml --fetch # .agents/skills/html2rss-config/scripts/check_config domain/file.yml --fetch --botasaurus @@ -18,7 +18,7 @@ def usage! end def repo_root - Pathname.new(__dir__).ascend.find { |p| (p / 'lib/html2rss/configs').directory? } || + Pathname.new(__dir__).ascend.find { |p| (p / 'configs').directory? } || abort('Could not find html2rss-configs repo root') end @@ -27,14 +27,14 @@ def resolve_config(arg, root) candidates = [] candidates << path if path.absolute? candidates << (root / path) - candidates << (root / 'lib/html2rss/configs' / path) + candidates << (root / 'configs' / path) found = candidates.find(&:file?) abort("Config not found: #{arg}") unless found found.expand_path end def feed_example_name(config_path, root) - config_path.relative_path_from(root / 'lib/html2rss/configs').to_s + config_path.relative_path_from(root / 'configs').to_s end def invoke_entry(entry, sibling) @@ -86,7 +86,7 @@ config = resolve_config(ARGV.fetch(0), root) example = feed_example_name(config, root) cli = resolve_html2rss_invocation(root) cli_chdir = cli.first == 'bundle' ? (root.parent / 'html2rss').to_s : nil -env = {} +env = { 'BUNDLE_GEMFILE' => (root / 'tool/Gemfile').to_s } env['BOTASAURUS_SCRAPER_URL'] = ENV.fetch('BOTASAURUS_SCRAPER_URL', 'http://localhost:4010') if botasaurus Dir.chdir(root) do @@ -101,7 +101,7 @@ Dir.chdir(root) do if fetch cmd = ['bundle', 'exec', 'rspec', '--tag', 'fetch', '--example', example, - 'spec/html2rss/configs_dynamic_spec.rb'] + 'test/configs_dynamic_spec.rb'] run!(*cmd, env: env) end end diff --git a/.agents/skills/html2rss-config/scripts/register_botasaurus b/.agents/skills/html2rss-config/scripts/register_botasaurus index 7fe2e57c..2a2210d7 100755 --- a/.agents/skills/html2rss-config/scripts/register_botasaurus +++ b/.agents/skills/html2rss-config/scripts/register_botasaurus @@ -1,10 +1,10 @@ #!/usr/bin/env ruby # frozen_string_literal: true -# Idempotently add a config path to spec/support/botasaurus_fetch_configs.rb (sorted). +# Idempotently add a config path to test/support/botasaurus_fetch_configs.rb (sorted). # Usage (from repo root): # .agents/skills/html2rss-config/scripts/register_botasaurus domain/file.yml -# .agents/skills/html2rss-config/scripts/register_botasaurus lib/html2rss/configs/domain/file.yml +# .agents/skills/html2rss-config/scripts/register_botasaurus configs/domain/file.yml require 'pathname' @@ -16,15 +16,15 @@ def usage! end def repo_root - Pathname.new(__dir__).ascend.find { |p| (p / 'spec/support/botasaurus_fetch_configs.rb').file? } || + Pathname.new(__dir__).ascend.find { |p| (p / 'test/support/botasaurus_fetch_configs.rb').file? } || abort('Could not find html2rss-configs repo root') end def normalize_name(arg, root) - rel = arg.to_s.split('lib/html2rss/configs/', 2).last.delete_prefix('/').delete_prefix('./') + rel = arg.to_s.split('configs/', 2).last.delete_prefix('/').delete_prefix('./') abort('Expected something like domain/file.yml') unless rel.match?(%r{\A[^/]+/.+\.yml\z}) - warn "Warning: missing #{root / 'lib/html2rss/configs' / rel}" unless (root / 'lib/html2rss/configs' / rel).file? + warn "Warning: missing #{root / 'configs' / rel}" unless (root / 'configs' / rel).file? rel end @@ -32,7 +32,7 @@ usage! if ARGV.size != 1 || ARGV.include?('-h') || ARGV.include?('--help') root = repo_root name = normalize_name(ARGV.fetch(0), root) -registry = root / 'spec/support/botasaurus_fetch_configs.rb' +registry = root / 'test/support/botasaurus_fetch_configs.rb' text = registry.read unless text =~ /CONFIGS = %w\[(.*?)\].freeze/m diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..6bd13dd4 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: Release + +"on": + push: + tags: + - '*' + +env: + BUNDLE_GEMFILE: tool/Gemfile + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.3 + bundler-cache: true + working-directory: tool + + - name: Build signed registry bundle + env: + REGISTRY_VERSION: ${{ github.ref_name }} + REGISTRY_SIGNING_KEY: ${{ secrets.REGISTRY_SIGNING_KEY }} + run: make registry-build -- --sign + + - name: Upload release artifact + uses: softprops/action-gh-release@v2 + with: + files: dist/registry-bundle.tar.gz + generate_release_notes: true diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 2e3a9702..ccb910c6 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -6,57 +6,35 @@ name: Ruby pull_request: branches: [master] +env: + BUNDLE_GEMFILE: tool/Gemfile + jobs: - lint: + ready: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "20" - name: Install dependencies run: sudo apt-get install -yqq yamllint - - name: Install Node dependencies - run: npm ci --ignore-scripts --no-fund --no-audit - - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: 3.3 bundler-cache: true + working-directory: tool - - name: Run linters - run: make lint - - - name: Validate configs - run: make validate - - test: - strategy: - fail-fast: false - matrix: - ruby: ["3.3"] - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby }} - bundler-cache: true - - - name: Run tests - run: make test + - name: Run quality gate + run: make ready test_changed_configs: runs-on: ubuntu-latest + env: + BUNDLE_GEMFILE: tool/Gemfile + steps: - uses: actions/checkout@v4 @@ -65,6 +43,7 @@ jobs: with: ruby-version: 3.3 bundler-cache: true + working-directory: tool - name: setup remote origin run: >- diff --git a/.gitignore b/.gitignore index e89d97f6..797943fb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,10 +3,11 @@ /_yardoc/ /coverage/ /doc/ +/dist/ +/node_modules/ /pkg/ -/spec/reports/ +/test/reports/ /tmp/ -/node_modules/ /schema/ # rspec failure tracking diff --git a/.rspec b/.rspec index 7b8568a5..03543283 100644 --- a/.rspec +++ b/.rspec @@ -1,3 +1,3 @@ --format p --color ---require spec_helper +--require ./test/test_helper diff --git a/.rubocop.yml b/.rubocop.yml index 172518b2..290e8ea7 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -3,17 +3,27 @@ plugins: - rubocop-performance AllCops: - DisplayCopNames: true + DisplayCopName: true NewCops: enable + TargetRubyVersion: 3.3 Exclude: - vendor/**/* + - tool/Gemfile.lock Layout/ClassStructure: Enabled: true RSpec/DescribeClass: Exclude: - - "spec/html2rss/configs/**/*.yml_spec.rb" + - "test/**/*.yml_spec.rb" + - "test/configs_dynamic_spec.rb" + - "test/registry.spec.rb" + - "test/validate.spec.rb" + +Metrics/BlockLength: + Exclude: + - "test/registry.spec.rb" + - "test/validate.spec.rb" Naming/RescuedExceptionsVariableName: PreferredName: error diff --git a/.vscode/settings.json b/.vscode/settings.json index 616d1a61..b80492fd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { "yaml.schemas": { - "https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json": "lib/html2rss/configs/**/*.yml" + "https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json": "configs/**/*.yml" } } diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index 2c7e94f8..00000000 --- a/Gemfile.lock +++ /dev/null @@ -1,194 +0,0 @@ -PATH - remote: . - specs: - html2rss-configs (0.2.0) - html2rss - -GEM - remote: https://rubygems.org/ - specs: - addressable (2.9.0) - public_suffix (>= 2.0.2, < 8.0) - ast (2.4.3) - bigdecimal (4.1.2) - brotli (0.8.0) - concurrent-ruby (1.3.8) - crass (1.0.7) - diff-lcs (1.6.2) - dry-configurable (1.4.0) - dry-core (~> 1.0) - zeitwerk (~> 2.6) - dry-core (1.2.0) - concurrent-ruby (~> 1.0) - logger - zeitwerk (~> 2.6) - dry-inflector (1.3.1) - dry-initializer (3.2.0) - dry-logic (1.6.0) - bigdecimal - concurrent-ruby (~> 1.0) - dry-core (~> 1.1) - zeitwerk (~> 2.6) - dry-schema (1.16.0) - concurrent-ruby (~> 1.0) - dry-configurable (~> 1.0, >= 1.0.1) - dry-core (~> 1.1) - dry-initializer (~> 3.2) - dry-logic (~> 1.6) - dry-types (~> 1.9, >= 1.9.1) - zeitwerk (~> 2.6) - dry-types (1.9.1) - bigdecimal (>= 3.0) - concurrent-ruby (~> 1.0) - dry-core (~> 1.0) - dry-inflector (~> 1.0) - dry-logic (~> 1.4) - zeitwerk (~> 2.6) - dry-validation (1.11.1) - concurrent-ruby (~> 1.0) - dry-core (~> 1.1) - dry-initializer (~> 3.2) - dry-schema (~> 1.14) - zeitwerk (~> 2.6) - faraday (2.14.3) - faraday-net_http (>= 2.0, < 3.5) - json - logger - faraday-follow_redirects (0.5.0) - faraday (>= 1, < 3) - faraday-gzip (3.1.0) - faraday (>= 2.0, < 3) - zlib (~> 3.0) - faraday-net_http (3.4.4) - net-http (~> 0.5) - hana (1.3.7) - html2rss (0.27.1) - addressable (~> 2.7) - brotli - dry-validation - faraday (> 2.0.1, < 3.0) - faraday-follow_redirects - faraday-gzip (~> 3) - kramdown - mcp (~> 1.2) - mime-types (> 3.0) - nokogiri (>= 1.10, < 2.0) - rack (~> 3.0) - rackup (~> 2.0) - regexp_parser - reverse_markdown (~> 3.0) - rss - sanitize - thor - tzinfo - webrick (~> 1.9) - zeitwerk - json (2.21.2) - json_schemer (2.5.0) - bigdecimal - hana (~> 1.3) - regexp_parser (~> 2.0) - simpleidn (~> 0.2) - kramdown (2.5.2) - rexml (>= 3.4.4) - language_server-protocol (3.17.0.5) - lint_roller (1.1.0) - logger (1.7.0) - mcp (1.2.0) - json_schemer (>= 2.4) - mime-types (3.7.0) - logger - mime-types-data (~> 3.2025, >= 3.2025.0507) - mime-types-data (3.2026.0701) - net-http (0.9.1) - uri (>= 0.11.1) - nokogiri (1.19.4-arm64-darwin) - racc (~> 1.4) - nokogiri (1.19.4-x86_64-darwin) - racc (~> 1.4) - nokogiri (1.19.4-x86_64-linux-gnu) - racc (~> 1.4) - parallel (1.28.0) - parser (3.3.11.1) - ast (~> 2.4.1) - racc - prism (1.9.0) - public_suffix (7.0.5) - racc (1.8.1) - rack (3.2.7) - rackup (2.3.1) - rack (>= 3) - rainbow (3.1.1) - regexp_parser (2.12.0) - reverse_markdown (3.0.2) - nokogiri - rexml (3.4.4) - rspec (3.13.2) - rspec-core (~> 3.13.0) - rspec-expectations (~> 3.13.0) - rspec-mocks (~> 3.13.0) - rspec-core (3.13.6) - rspec-support (~> 3.13.0) - rspec-expectations (3.13.5) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-mocks (3.13.8) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-support (3.13.7) - rss (0.3.3) - rexml - rubocop (1.86.0) - json (~> 2.3) - language_server-protocol (~> 3.17.0.2) - lint_roller (~> 1.1.0) - parallel (~> 1.10) - parser (>= 3.3.0.2) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.49.0, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.49.1) - parser (>= 3.3.7.2) - prism (~> 1.7) - rubocop-performance (1.26.1) - lint_roller (~> 1.1) - rubocop (>= 1.75.0, < 2.0) - rubocop-ast (>= 1.47.1, < 2.0) - rubocop-rspec (3.9.0) - lint_roller (~> 1.1) - rubocop (~> 1.81) - ruby-progressbar (1.13.0) - sanitize (7.0.0) - crass (~> 1.0.2) - nokogiri (>= 1.16.8) - simpleidn (0.2.3) - thor (1.5.0) - tzinfo (2.0.6) - concurrent-ruby (~> 1.0) - unicode-display_width (2.6.0) - uri (1.1.1) - webrick (1.9.2) - zeitwerk (2.8.3) - zlib (3.2.3) - -PLATFORMS - arm64-darwin-23 - arm64-darwin-24 - arm64-darwin-25 - x86_64-darwin-20 - x86_64-linux - -DEPENDENCIES - html2rss (~> 0.27) - html2rss-configs! - nokogiri - public_suffix - rspec (~> 3.0) - rubocop - rubocop-performance - rubocop-rspec - -BUNDLED WITH - 2.6.6 diff --git a/Makefile b/Makefile index 5dd5fffd..b78eeec1 100644 --- a/Makefile +++ b/Makefile @@ -1,60 +1,50 @@ -default: lint validate test +BUNDLE = BUNDLE_GEMFILE=tool/Gemfile bundle exec +BUNDLE_RUBY = BUNDLE_GEMFILE=tool/Gemfile bundle exec ruby + +.PHONY: default check ready lint validate test registry-build test-fetch-changed-configs test-fetch-all-configs test-fetch-botasaurus-configs test-config test-domain lintfix + +default: check + +check: validate + ready: lint validate test lint: - yamllint lib/html2rss/configs/ .github/ - bundle exec rubocop -P --cache false -f quiet - ./node_modules/.bin/prettier --check lib/**/*.yml .github/**/*.yml README.md + yamllint configs/ .github/ + $(BUNDLE) rubocop -P --cache false -f quiet validate: - bundle exec ruby bin/validate_configs - -schema: - mkdir -p schema - bundle exec html2rss schema --write schema/html2rss-config.schema.json + $(BUNDLE_RUBY) tool/validate test: - bundle exec rspec + $(BUNDLE) rspec test + +registry-build: + $(BUNDLE_RUBY) tool/registry-build test-fetch-changed-configs: bin/rspec_changed_configs test-fetch-all-configs: - bundle exec rspec --tag fetch spec/html2rss/configs_dynamic_spec.rb + $(BUNDLE) rspec --tag fetch test/configs_dynamic_spec.rb test-fetch-botasaurus-configs: bin/rspec_botasaurus_configs -test-all: test test-fetch-all-configs - -# Dynamic test commands test-config: @if [ -z "$(CONFIG)" ]; then \ echo "Usage: make test-config CONFIG=github.com/releases.yml"; \ echo " make test-config CONFIG=github.com"; \ exit 1; \ fi - bundle exec rspec --example "$(CONFIG)" spec/html2rss/configs_dynamic_spec.rb + $(BUNDLE) rspec --example "$(CONFIG)" test/configs_dynamic_spec.rb test-domain: @if [ -z "$(DOMAIN)" ]; then \ echo "Usage: make test-domain DOMAIN=github.com"; \ exit 1; \ fi - bundle exec rspec --example "$(DOMAIN)" spec/html2rss/configs_dynamic_spec.rb - -# Migration commands -migrate-tests: - bin/migrate_to_dynamic_tests - -restore-tests: - @if [ -d "spec/html2rss/configs_backup" ]; then \ - cp -r spec/html2rss/configs_backup/* spec/html2rss/configs/; \ - echo "✅ Restored tests from backup"; \ - else \ - echo "❌ No backup found"; \ - fi + $(BUNDLE) rspec --example "$(DOMAIN)" test/configs_dynamic_spec.rb lintfix: - bundle exec rubocop -a - ./node_modules/.bin/prettier --write lib/**/*.yml .github/**/*.yml README.md + $(BUNDLE) rubocop -a diff --git a/bin/console b/bin/console deleted file mode 100755 index df7d6b26..00000000 --- a/bin/console +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'bundler/setup' -require 'html2rss/configs' - -# You can add fixtures and/or initialization code here to make experimenting -# with your gem easier. You can also use a different console, if you like. - -# (If you use this, don't forget to add pry to your Gemfile!) -# require "pry" -# Pry.start - -require 'irb' -IRB.start(__FILE__) diff --git a/bin/json_to_xml b/bin/json_to_xml deleted file mode 100755 index 20e62947..00000000 --- a/bin/json_to_xml +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'bundler/setup' -require 'html2rss/item' -require 'html2rss/utils' - -raise ArgumentError unless ARGV[0].is_a?(String) - -puts Html2rss::Utils.request_body_from_url(ARGV[0], convert_json_to_xml: true) diff --git a/bin/rspec_botasaurus_configs b/bin/rspec_botasaurus_configs index cde8eaea..8ed0546f 100755 --- a/bin/rspec_botasaurus_configs +++ b/bin/rspec_botasaurus_configs @@ -1,7 +1,7 @@ #!/usr/bin/env ruby # frozen_string_literal: true -require_relative '../spec/support/botasaurus_fetch_configs' +require_relative '../test/support/botasaurus_fetch_configs' unless BotasaurusFetchConfigs.botasaurus_env_configured? warn 'BOTASAURUS_SCRAPER_URL is required for botasaurus fetch tests.' @@ -13,6 +13,6 @@ BotasaurusFetchConfigs::CONFIGS.each do |config| args << '--example' args << config end -args << 'spec/html2rss/configs_dynamic_spec.rb' +args << 'test/configs_dynamic_spec.rb' -exec(*args) +exec({ 'BUNDLE_GEMFILE' => 'tool/Gemfile' }, *args) diff --git a/bin/rspec_changed_configs b/bin/rspec_changed_configs index 4bed75ef..a1b50c28 100755 --- a/bin/rspec_changed_configs +++ b/bin/rspec_changed_configs @@ -1,7 +1,7 @@ #!/usr/bin/env ruby # frozen_string_literal: true -CONFIG_PATH = %r{\Alib/html2rss/configs/.+/.+\.yml\z} +CONFIG_PATH = %r{\Aconfigs/.+/.+\.yml\z} SCHEMA_MODELINE = /\A[+-]# yaml-language-server: \$schema=/ def comment_only_change?(file) @@ -26,9 +26,10 @@ end failed = false test_files.each do |file| - config_name = file.sub('lib/html2rss/configs/', '') + config_name = file.delete_prefix('configs/') puts "Testing changed config: #{config_name}" - failed ||= !system("bundle exec rspec --example '#{config_name}' --tag fetch spec/html2rss/configs_dynamic_spec.rb") + failed ||= !system('bundle', 'exec', 'rspec', '--example', config_name, '--tag', 'fetch', + 'test/configs_dynamic_spec.rb', env: { 'BUNDLE_GEMFILE' => 'tool/Gemfile' }) end exit 1 if failed diff --git a/lib/html2rss/configs/aalto.fi/news.yml b/configs/aalto.fi/news.yml similarity index 100% rename from lib/html2rss/configs/aalto.fi/news.yml rename to configs/aalto.fi/news.yml diff --git a/lib/html2rss/configs/academictransfer.com/jobs.yml b/configs/academictransfer.com/jobs.yml similarity index 100% rename from lib/html2rss/configs/academictransfer.com/jobs.yml rename to configs/academictransfer.com/jobs.yml diff --git a/lib/html2rss/configs/adfc.de/pressemitteilungen.yml b/configs/adfc.de/pressemitteilungen.yml similarity index 100% rename from lib/html2rss/configs/adfc.de/pressemitteilungen.yml rename to configs/adfc.de/pressemitteilungen.yml diff --git a/lib/html2rss/configs/ahram.org.eg/egypt.yml b/configs/ahram.org.eg/egypt.yml similarity index 100% rename from lib/html2rss/configs/ahram.org.eg/egypt.yml rename to configs/ahram.org.eg/egypt.yml diff --git a/lib/html2rss/configs/ai.meta.com/blog.yml b/configs/ai.meta.com/blog.yml similarity index 100% rename from lib/html2rss/configs/ai.meta.com/blog.yml rename to configs/ai.meta.com/blog.yml diff --git a/lib/html2rss/configs/akorda.kz/events.yml b/configs/akorda.kz/events.yml similarity index 100% rename from lib/html2rss/configs/akorda.kz/events.yml rename to configs/akorda.kz/events.yml diff --git a/lib/html2rss/configs/angop.ao/noticias.yml b/configs/angop.ao/noticias.yml similarity index 100% rename from lib/html2rss/configs/angop.ao/noticias.yml rename to configs/angop.ao/noticias.yml diff --git a/lib/html2rss/configs/anthropic.com/news.yml b/configs/anthropic.com/news.yml similarity index 100% rename from lib/html2rss/configs/anthropic.com/news.yml rename to configs/anthropic.com/news.yml diff --git a/lib/html2rss/configs/anu.edu.au/news.yml b/configs/anu.edu.au/news.yml similarity index 100% rename from lib/html2rss/configs/anu.edu.au/news.yml rename to configs/anu.edu.au/news.yml diff --git a/lib/html2rss/configs/apnews.com/hub.yml b/configs/apnews.com/hub.yml similarity index 100% rename from lib/html2rss/configs/apnews.com/hub.yml rename to configs/apnews.com/hub.yml diff --git a/lib/html2rss/configs/apple.com/newsroom.yml b/configs/apple.com/newsroom.yml similarity index 100% rename from lib/html2rss/configs/apple.com/newsroom.yml rename to configs/apple.com/newsroom.yml diff --git a/lib/html2rss/configs/aps.dz/national-news.yml b/configs/aps.dz/national-news.yml similarity index 100% rename from lib/html2rss/configs/aps.dz/national-news.yml rename to configs/aps.dz/national-news.yml diff --git a/lib/html2rss/configs/asml.com/press-releases.yml b/configs/asml.com/press-releases.yml similarity index 100% rename from lib/html2rss/configs/asml.com/press-releases.yml rename to configs/asml.com/press-releases.yml diff --git a/lib/html2rss/configs/aspi.org.au/strategist.yml b/configs/aspi.org.au/strategist.yml similarity index 100% rename from lib/html2rss/configs/aspi.org.au/strategist.yml rename to configs/aspi.org.au/strategist.yml diff --git a/lib/html2rss/configs/bbc.co.uk/available_episodes.yml b/configs/bbc.co.uk/available_episodes.yml similarity index 100% rename from lib/html2rss/configs/bbc.co.uk/available_episodes.yml rename to configs/bbc.co.uk/available_episodes.yml diff --git a/lib/html2rss/configs/bbc.com/mundo.yml b/configs/bbc.com/mundo.yml similarity index 100% rename from lib/html2rss/configs/bbc.com/mundo.yml rename to configs/bbc.com/mundo.yml diff --git a/lib/html2rss/configs/blog.mondediplo.net/feed.yml b/configs/blog.mondediplo.net/feed.yml similarity index 100% rename from lib/html2rss/configs/blog.mondediplo.net/feed.yml rename to configs/blog.mondediplo.net/feed.yml diff --git a/lib/html2rss/configs/brookings.edu/articles.yml b/configs/brookings.edu/articles.yml similarity index 100% rename from lib/html2rss/configs/brookings.edu/articles.yml rename to configs/brookings.edu/articles.yml diff --git a/lib/html2rss/configs/bundesregierung.de/pressemitteilungen.yml b/configs/bundesregierung.de/pressemitteilungen.yml similarity index 100% rename from lib/html2rss/configs/bundesregierung.de/pressemitteilungen.yml rename to configs/bundesregierung.de/pressemitteilungen.yml diff --git a/lib/html2rss/configs/canarianweekly.com/front.yml b/configs/canarianweekly.com/front.yml similarity index 100% rename from lib/html2rss/configs/canarianweekly.com/front.yml rename to configs/canarianweekly.com/front.yml diff --git a/lib/html2rss/configs/caricom.org/news.yml b/configs/caricom.org/news.yml similarity index 100% rename from lib/html2rss/configs/caricom.org/news.yml rename to configs/caricom.org/news.yml diff --git a/lib/html2rss/configs/carnegieendowment.org/research.yml b/configs/carnegieendowment.org/research.yml similarity index 100% rename from lib/html2rss/configs/carnegieendowment.org/research.yml rename to configs/carnegieendowment.org/research.yml diff --git a/lib/html2rss/configs/cepal.org/press-releases.yml b/configs/cepal.org/press-releases.yml similarity index 100% rename from lib/html2rss/configs/cepal.org/press-releases.yml rename to configs/cepal.org/press-releases.yml diff --git a/lib/html2rss/configs/chalmers.se/news.yml b/configs/chalmers.se/news.yml similarity index 100% rename from lib/html2rss/configs/chalmers.se/news.yml rename to configs/chalmers.se/news.yml diff --git a/lib/html2rss/configs/chathamhouse.org/research-publications.yml b/configs/chathamhouse.org/research-publications.yml similarity index 100% rename from lib/html2rss/configs/chathamhouse.org/research-publications.yml rename to configs/chathamhouse.org/research-publications.yml diff --git a/lib/html2rss/configs/cinemascore.com/index.yml b/configs/cinemascore.com/index.yml similarity index 100% rename from lib/html2rss/configs/cinemascore.com/index.yml rename to configs/cinemascore.com/index.yml diff --git a/lib/html2rss/configs/cleanenergywire.org/news.yml b/configs/cleanenergywire.org/news.yml similarity index 100% rename from lib/html2rss/configs/cleanenergywire.org/news.yml rename to configs/cleanenergywire.org/news.yml diff --git a/lib/html2rss/configs/cnet.com/section_sub.yml b/configs/cnet.com/section_sub.yml similarity index 100% rename from lib/html2rss/configs/cnet.com/section_sub.yml rename to configs/cnet.com/section_sub.yml diff --git a/lib/html2rss/configs/cohere.com/blog.yml b/configs/cohere.com/blog.yml similarity index 100% rename from lib/html2rss/configs/cohere.com/blog.yml rename to configs/cohere.com/blog.yml diff --git a/lib/html2rss/configs/computerbase.de/meistgelesen.yml b/configs/computerbase.de/meistgelesen.yml similarity index 100% rename from lib/html2rss/configs/computerbase.de/meistgelesen.yml rename to configs/computerbase.de/meistgelesen.yml diff --git a/lib/html2rss/configs/cplp.org/noticias.yml b/configs/cplp.org/noticias.yml similarity index 100% rename from lib/html2rss/configs/cplp.org/noticias.yml rename to configs/cplp.org/noticias.yml diff --git a/lib/html2rss/configs/csiro.au/news.yml b/configs/csiro.au/news.yml similarity index 100% rename from lib/html2rss/configs/csiro.au/news.yml rename to configs/csiro.au/news.yml diff --git a/lib/html2rss/configs/curia.europa.eu/press-releases.yml b/configs/curia.europa.eu/press-releases.yml similarity index 100% rename from lib/html2rss/configs/curia.europa.eu/press-releases.yml rename to configs/curia.europa.eu/press-releases.yml diff --git a/lib/html2rss/configs/cutle.fish/index.yml b/configs/cutle.fish/index.yml similarity index 100% rename from lib/html2rss/configs/cutle.fish/index.yml rename to configs/cutle.fish/index.yml diff --git a/lib/html2rss/configs/deepmind.google/blog.yml b/configs/deepmind.google/blog.yml similarity index 100% rename from lib/html2rss/configs/deepmind.google/blog.yml rename to configs/deepmind.google/blog.yml diff --git a/lib/html2rss/configs/deraktionaer.de/meistgelesen.yml b/configs/deraktionaer.de/meistgelesen.yml similarity index 100% rename from lib/html2rss/configs/deraktionaer.de/meistgelesen.yml rename to configs/deraktionaer.de/meistgelesen.yml diff --git a/lib/html2rss/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml b/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml similarity index 100% rename from lib/html2rss/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml rename to configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml diff --git a/lib/html2rss/configs/dfs.de/pressemitteilungen.yml b/configs/dfs.de/pressemitteilungen.yml similarity index 100% rename from lib/html2rss/configs/dfs.de/pressemitteilungen.yml rename to configs/dfs.de/pressemitteilungen.yml diff --git a/lib/html2rss/configs/diis.dk/research.yml b/configs/diis.dk/research.yml similarity index 100% rename from lib/html2rss/configs/diis.dk/research.yml rename to configs/diis.dk/research.yml diff --git a/lib/html2rss/configs/dirco.gov.za/media-statements.yml b/configs/dirco.gov.za/media-statements.yml similarity index 100% rename from lib/html2rss/configs/dirco.gov.za/media-statements.yml rename to configs/dirco.gov.za/media-statements.yml diff --git a/lib/html2rss/configs/dsw-info.de/presse.yml b/configs/dsw-info.de/presse.yml similarity index 100% rename from lib/html2rss/configs/dsw-info.de/presse.yml rename to configs/dsw-info.de/presse.yml diff --git a/lib/html2rss/configs/elastic.co/elasticsearch-release-notes.yml b/configs/elastic.co/elasticsearch-release-notes.yml similarity index 100% rename from lib/html2rss/configs/elastic.co/elasticsearch-release-notes.yml rename to configs/elastic.co/elasticsearch-release-notes.yml diff --git a/lib/html2rss/configs/elysee.fr/actualites.yml b/configs/elysee.fr/actualites.yml similarity index 100% rename from lib/html2rss/configs/elysee.fr/actualites.yml rename to configs/elysee.fr/actualites.yml diff --git a/lib/html2rss/configs/equinor.com/news.yml b/configs/equinor.com/news.yml similarity index 100% rename from lib/html2rss/configs/equinor.com/news.yml rename to configs/equinor.com/news.yml diff --git a/lib/html2rss/configs/espn.com/f1.yml b/configs/espn.com/f1.yml similarity index 100% rename from lib/html2rss/configs/espn.com/f1.yml rename to configs/espn.com/f1.yml diff --git a/lib/html2rss/configs/ethz.ch/eth-news.yml b/configs/ethz.ch/eth-news.yml similarity index 100% rename from lib/html2rss/configs/ethz.ch/eth-news.yml rename to configs/ethz.ch/eth-news.yml diff --git a/lib/html2rss/configs/eures.europa.eu/news.yml b/configs/eures.europa.eu/news.yml similarity index 100% rename from lib/html2rss/configs/eures.europa.eu/news.yml rename to configs/eures.europa.eu/news.yml diff --git a/lib/html2rss/configs/faa.gov/newsroom.yml b/configs/faa.gov/newsroom.yml similarity index 100% rename from lib/html2rss/configs/faa.gov/newsroom.yml rename to configs/faa.gov/newsroom.yml diff --git a/lib/html2rss/configs/fao.org/newsroom.yml b/configs/fao.org/newsroom.yml similarity index 100% rename from lib/html2rss/configs/fao.org/newsroom.yml rename to configs/fao.org/newsroom.yml diff --git a/lib/html2rss/configs/fia.com/documents.yml b/configs/fia.com/documents.yml similarity index 100% rename from lib/html2rss/configs/fia.com/documents.yml rename to configs/fia.com/documents.yml diff --git a/lib/html2rss/configs/figma.com/blog.yml b/configs/figma.com/blog.yml similarity index 100% rename from lib/html2rss/configs/figma.com/blog.yml rename to configs/figma.com/blog.yml diff --git a/lib/html2rss/configs/formula1.com/latest.yml b/configs/formula1.com/latest.yml similarity index 100% rename from lib/html2rss/configs/formula1.com/latest.yml rename to configs/formula1.com/latest.yml diff --git a/lib/html2rss/configs/forumsec.org/publications.yml b/configs/forumsec.org/publications.yml similarity index 100% rename from lib/html2rss/configs/forumsec.org/publications.yml rename to configs/forumsec.org/publications.yml diff --git a/lib/html2rss/configs/ftc.gov/press-releases.yml b/configs/ftc.gov/press-releases.yml similarity index 100% rename from lib/html2rss/configs/ftc.gov/press-releases.yml rename to configs/ftc.gov/press-releases.yml diff --git a/lib/html2rss/configs/github.com/releases.yml b/configs/github.com/releases.yml similarity index 100% rename from lib/html2rss/configs/github.com/releases.yml rename to configs/github.com/releases.yml diff --git a/lib/html2rss/configs/go.dev/release-history.yml b/configs/go.dev/release-history.yml similarity index 100% rename from lib/html2rss/configs/go.dev/release-history.yml rename to configs/go.dev/release-history.yml diff --git a/lib/html2rss/configs/grafana.com/whatsnew.yml b/configs/grafana.com/whatsnew.yml similarity index 100% rename from lib/html2rss/configs/grafana.com/whatsnew.yml rename to configs/grafana.com/whatsnew.yml diff --git a/lib/html2rss/configs/haaretz.com/israel-news.yml b/configs/haaretz.com/israel-news.yml similarity index 100% rename from lib/html2rss/configs/haaretz.com/israel-news.yml rename to configs/haaretz.com/israel-news.yml diff --git a/lib/html2rss/configs/hai.stanford.edu/news.yml b/configs/hai.stanford.edu/news.yml similarity index 100% rename from lib/html2rss/configs/hai.stanford.edu/news.yml rename to configs/hai.stanford.edu/news.yml diff --git a/lib/html2rss/configs/iaapa.org/news.yml b/configs/iaapa.org/news.yml similarity index 100% rename from lib/html2rss/configs/iaapa.org/news.yml rename to configs/iaapa.org/news.yml diff --git a/lib/html2rss/configs/icj-cij.org/press-releases.yml b/configs/icj-cij.org/press-releases.yml similarity index 100% rename from lib/html2rss/configs/icj-cij.org/press-releases.yml rename to configs/icj-cij.org/press-releases.yml diff --git a/lib/html2rss/configs/icrc.org/news.yml b/configs/icrc.org/news.yml similarity index 100% rename from lib/html2rss/configs/icrc.org/news.yml rename to configs/icrc.org/news.yml diff --git a/lib/html2rss/configs/iea.org/news.yml b/configs/iea.org/news.yml similarity index 100% rename from lib/html2rss/configs/iea.org/news.yml rename to configs/iea.org/news.yml diff --git a/lib/html2rss/configs/iiss.org/online-analysis.yml b/configs/iiss.org/online-analysis.yml similarity index 100% rename from lib/html2rss/configs/iiss.org/online-analysis.yml rename to configs/iiss.org/online-analysis.yml diff --git a/lib/html2rss/configs/ikea.com/newsroom.yml b/configs/ikea.com/newsroom.yml similarity index 100% rename from lib/html2rss/configs/ikea.com/newsroom.yml rename to configs/ikea.com/newsroom.yml diff --git a/lib/html2rss/configs/imdb.com/ratings.yml b/configs/imdb.com/ratings.yml similarity index 100% rename from lib/html2rss/configs/imdb.com/ratings.yml rename to configs/imdb.com/ratings.yml diff --git a/lib/html2rss/configs/imo.org/press-briefings.yml b/configs/imo.org/press-briefings.yml similarity index 100% rename from lib/html2rss/configs/imo.org/press-briefings.yml rename to configs/imo.org/press-briefings.yml diff --git a/lib/html2rss/configs/imperial.ac.uk/news.yml b/configs/imperial.ac.uk/news.yml similarity index 100% rename from lib/html2rss/configs/imperial.ac.uk/news.yml rename to configs/imperial.ac.uk/news.yml diff --git a/lib/html2rss/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml b/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml similarity index 100% rename from lib/html2rss/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml rename to configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml diff --git a/lib/html2rss/configs/isro.gov.in/press.yml b/configs/isro.gov.in/press.yml similarity index 100% rename from lib/html2rss/configs/isro.gov.in/press.yml rename to configs/isro.gov.in/press.yml diff --git a/lib/html2rss/configs/issafrica.org/iss-today.yml b/configs/issafrica.org/iss-today.yml similarity index 100% rename from lib/html2rss/configs/issafrica.org/iss-today.yml rename to configs/issafrica.org/iss-today.yml diff --git a/lib/html2rss/configs/issafrica.org/press-releases.yml b/configs/issafrica.org/press-releases.yml similarity index 100% rename from lib/html2rss/configs/issafrica.org/press-releases.yml rename to configs/issafrica.org/press-releases.yml diff --git a/lib/html2rss/configs/kaist.ac.kr/news.yml b/configs/kaist.ac.kr/news.yml similarity index 100% rename from lib/html2rss/configs/kaist.ac.kr/news.yml rename to configs/kaist.ac.kr/news.yml diff --git a/lib/html2rss/configs/kinocheck.de/filmstarts.yml b/configs/kinocheck.de/filmstarts.yml similarity index 100% rename from lib/html2rss/configs/kinocheck.de/filmstarts.yml rename to configs/kinocheck.de/filmstarts.yml diff --git a/lib/html2rss/configs/kompas.com/nasional.yml b/configs/kompas.com/nasional.yml similarity index 100% rename from lib/html2rss/configs/kompas.com/nasional.yml rename to configs/kompas.com/nasional.yml diff --git a/lib/html2rss/configs/lego.com/press-releases.yml b/configs/lego.com/press-releases.yml similarity index 100% rename from lib/html2rss/configs/lego.com/press-releases.yml rename to configs/lego.com/press-releases.yml diff --git a/lib/html2rss/configs/machinelearning.apple.com/research.yml b/configs/machinelearning.apple.com/research.yml similarity index 100% rename from lib/html2rss/configs/machinelearning.apple.com/research.yml rename to configs/machinelearning.apple.com/research.yml diff --git a/lib/html2rss/configs/maersk.com/press-releases.yml b/configs/maersk.com/press-releases.yml similarity index 100% rename from lib/html2rss/configs/maersk.com/press-releases.yml rename to configs/maersk.com/press-releases.yml diff --git a/lib/html2rss/configs/microsoft.com/azure-products.yml b/configs/microsoft.com/azure-products.yml similarity index 100% rename from lib/html2rss/configs/microsoft.com/azure-products.yml rename to configs/microsoft.com/azure-products.yml diff --git a/lib/html2rss/configs/mistral.ai/news.yml b/configs/mistral.ai/news.yml similarity index 100% rename from lib/html2rss/configs/mistral.ai/news.yml rename to configs/mistral.ai/news.yml diff --git a/lib/html2rss/configs/mozilla.org/security-advisories.yml b/configs/mozilla.org/security-advisories.yml similarity index 100% rename from lib/html2rss/configs/mozilla.org/security-advisories.yml rename to configs/mozilla.org/security-advisories.yml diff --git a/lib/html2rss/configs/msf.org/latest.yml b/configs/msf.org/latest.yml similarity index 100% rename from lib/html2rss/configs/msf.org/latest.yml rename to configs/msf.org/latest.yml diff --git a/lib/html2rss/configs/nbim.no/press-releases.yml b/configs/nbim.no/press-releases.yml similarity index 100% rename from lib/html2rss/configs/nbim.no/press-releases.yml rename to configs/nbim.no/press-releases.yml diff --git a/lib/html2rss/configs/news.cn/latest.yml b/configs/news.cn/latest.yml similarity index 100% rename from lib/html2rss/configs/news.cn/latest.yml rename to configs/news.cn/latest.yml diff --git a/lib/html2rss/configs/news24.com/top-stories.yml b/configs/news24.com/top-stories.yml similarity index 100% rename from lib/html2rss/configs/news24.com/top-stories.yml rename to configs/news24.com/top-stories.yml diff --git a/lib/html2rss/configs/newyorker.com/magazine.yml b/configs/newyorker.com/magazine.yml similarity index 100% rename from lib/html2rss/configs/newyorker.com/magazine.yml rename to configs/newyorker.com/magazine.yml diff --git a/lib/html2rss/configs/nigerianstat.gov.ng/news.yml b/configs/nigerianstat.gov.ng/news.yml similarity index 100% rename from lib/html2rss/configs/nigerianstat.gov.ng/news.yml rename to configs/nigerianstat.gov.ng/news.yml diff --git a/lib/html2rss/configs/nikkei.com/asia-technology.yml b/configs/nikkei.com/asia-technology.yml similarity index 100% rename from lib/html2rss/configs/nikkei.com/asia-technology.yml rename to configs/nikkei.com/asia-technology.yml diff --git a/lib/html2rss/configs/nokia.com/newsroom.yml b/configs/nokia.com/newsroom.yml similarity index 100% rename from lib/html2rss/configs/nokia.com/newsroom.yml rename to configs/nokia.com/newsroom.yml diff --git a/lib/html2rss/configs/nomanssky.com/news.yml b/configs/nomanssky.com/news.yml similarity index 100% rename from lib/html2rss/configs/nomanssky.com/news.yml rename to configs/nomanssky.com/news.yml diff --git a/lib/html2rss/configs/notion.com/blog.yml b/configs/notion.com/blog.yml similarity index 100% rename from lib/html2rss/configs/notion.com/blog.yml rename to configs/notion.com/blog.yml diff --git a/lib/html2rss/configs/novonordisk.com/latest-news.yml b/configs/novonordisk.com/latest-news.yml similarity index 100% rename from lib/html2rss/configs/novonordisk.com/latest-news.yml rename to configs/novonordisk.com/latest-news.yml diff --git a/lib/html2rss/configs/ntu.edu.sg/news.yml b/configs/ntu.edu.sg/news.yml similarity index 100% rename from lib/html2rss/configs/ntu.edu.sg/news.yml rename to configs/ntu.edu.sg/news.yml diff --git a/lib/html2rss/configs/nupi.no/news.yml b/configs/nupi.no/news.yml similarity index 100% rename from lib/html2rss/configs/nupi.no/news.yml rename to configs/nupi.no/news.yml diff --git a/lib/html2rss/configs/occrp.org/news.yml b/configs/occrp.org/news.yml similarity index 100% rename from lib/html2rss/configs/occrp.org/news.yml rename to configs/occrp.org/news.yml diff --git a/lib/html2rss/configs/oecd.org/newsroom.yml b/configs/oecd.org/newsroom.yml similarity index 100% rename from lib/html2rss/configs/oecd.org/newsroom.yml rename to configs/oecd.org/newsroom.yml diff --git a/lib/html2rss/configs/opec.org/press-releases.yml b/configs/opec.org/press-releases.yml similarity index 100% rename from lib/html2rss/configs/opec.org/press-releases.yml rename to configs/opec.org/press-releases.yml diff --git a/lib/html2rss/configs/panapress.com/latest.yml b/configs/panapress.com/latest.yml similarity index 100% rename from lib/html2rss/configs/panapress.com/latest.yml rename to configs/panapress.com/latest.yml diff --git a/lib/html2rss/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml b/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml similarity index 100% rename from lib/html2rss/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml rename to configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml diff --git a/lib/html2rss/configs/philstar.com/nation.yml b/configs/philstar.com/nation.yml similarity index 100% rename from lib/html2rss/configs/philstar.com/nation.yml rename to configs/philstar.com/nation.yml diff --git a/lib/html2rss/configs/phys.org/weekly.yml b/configs/phys.org/weekly.yml similarity index 100% rename from lib/html2rss/configs/phys.org/weekly.yml rename to configs/phys.org/weekly.yml diff --git a/lib/html2rss/configs/pib.gov.in/press-releases.yml b/configs/pib.gov.in/press-releases.yml similarity index 100% rename from lib/html2rss/configs/pib.gov.in/press-releases.yml rename to configs/pib.gov.in/press-releases.yml diff --git a/lib/html2rss/configs/rbb24.de/meistgeklickt.yml b/configs/rbb24.de/meistgeklickt.yml similarity index 100% rename from lib/html2rss/configs/rbb24.de/meistgeklickt.yml rename to configs/rbb24.de/meistgeklickt.yml diff --git a/lib/html2rss/configs/robinwood.de/aktuelles.yml b/configs/robinwood.de/aktuelles.yml similarity index 100% rename from lib/html2rss/configs/robinwood.de/aktuelles.yml rename to configs/robinwood.de/aktuelles.yml diff --git a/lib/html2rss/configs/s3.amazonaws.com/popular_movies.yml b/configs/s3.amazonaws.com/popular_movies.yml similarity index 100% rename from lib/html2rss/configs/s3.amazonaws.com/popular_movies.yml rename to configs/s3.amazonaws.com/popular_movies.yml diff --git a/lib/html2rss/configs/sacu.int/news.yml b/configs/sacu.int/news.yml similarity index 100% rename from lib/html2rss/configs/sacu.int/news.yml rename to configs/sacu.int/news.yml diff --git a/lib/html2rss/configs/sadc.int/latest-news.yml b/configs/sadc.int/latest-news.yml similarity index 100% rename from lib/html2rss/configs/sadc.int/latest-news.yml rename to configs/sadc.int/latest-news.yml diff --git a/lib/html2rss/configs/sebastianvettel.de/news.yml b/configs/sebastianvettel.de/news.yml similarity index 100% rename from lib/html2rss/configs/sebastianvettel.de/news.yml rename to configs/sebastianvettel.de/news.yml diff --git a/lib/html2rss/configs/shopify.com/blog.yml b/configs/shopify.com/blog.yml similarity index 100% rename from lib/html2rss/configs/shopify.com/blog.yml rename to configs/shopify.com/blog.yml diff --git a/lib/html2rss/configs/sipri.org/news.yml b/configs/sipri.org/news.yml similarity index 100% rename from lib/html2rss/configs/sipri.org/news.yml rename to configs/sipri.org/news.yml diff --git a/lib/html2rss/configs/softwareleadweekly.com/issues.yml b/configs/softwareleadweekly.com/issues.yml similarity index 100% rename from lib/html2rss/configs/softwareleadweekly.com/issues.yml rename to configs/softwareleadweekly.com/issues.yml diff --git a/lib/html2rss/configs/solarthermalworld.org/news.yml b/configs/solarthermalworld.org/news.yml similarity index 100% rename from lib/html2rss/configs/solarthermalworld.org/news.yml rename to configs/solarthermalworld.org/news.yml diff --git a/lib/html2rss/configs/spektrum.de/meistgelesen.yml b/configs/spektrum.de/meistgelesen.yml similarity index 100% rename from lib/html2rss/configs/spektrum.de/meistgelesen.yml rename to configs/spektrum.de/meistgelesen.yml diff --git a/lib/html2rss/configs/spiegel.de/impressum_autor.yml b/configs/spiegel.de/impressum_autor.yml similarity index 100% rename from lib/html2rss/configs/spiegel.de/impressum_autor.yml rename to configs/spiegel.de/impressum_autor.yml diff --git a/lib/html2rss/configs/spotify.com/newsroom.yml b/configs/spotify.com/newsroom.yml similarity index 100% rename from lib/html2rss/configs/spotify.com/newsroom.yml rename to configs/spotify.com/newsroom.yml diff --git a/lib/html2rss/configs/stackoverflow.com/hot_network_questions.yml b/configs/stackoverflow.com/hot_network_questions.yml similarity index 100% rename from lib/html2rss/configs/stackoverflow.com/hot_network_questions.yml rename to configs/stackoverflow.com/hot_network_questions.yml diff --git a/lib/html2rss/configs/steuerzahler.de/news.yml b/configs/steuerzahler.de/news.yml similarity index 100% rename from lib/html2rss/configs/steuerzahler.de/news.yml rename to configs/steuerzahler.de/news.yml diff --git a/lib/html2rss/configs/stripes.com/index.yml b/configs/stripes.com/index.yml similarity index 100% rename from lib/html2rss/configs/stripes.com/index.yml rename to configs/stripes.com/index.yml diff --git a/lib/html2rss/configs/support.apple.com/en_gb_ht201222.yml b/configs/support.apple.com/en_gb_ht201222.yml similarity index 100% rename from lib/html2rss/configs/support.apple.com/en_gb_ht201222.yml rename to configs/support.apple.com/en_gb_ht201222.yml diff --git a/lib/html2rss/configs/support.apple.com/exchange_repair.yml b/configs/support.apple.com/exchange_repair.yml similarity index 100% rename from lib/html2rss/configs/support.apple.com/exchange_repair.yml rename to configs/support.apple.com/exchange_repair.yml diff --git a/lib/html2rss/configs/tap.info.tn/politics.yml b/configs/tap.info.tn/politics.yml similarity index 100% rename from lib/html2rss/configs/tap.info.tn/politics.yml rename to configs/tap.info.tn/politics.yml diff --git a/lib/html2rss/configs/telenor.com/newsroom.yml b/configs/telenor.com/newsroom.yml similarity index 100% rename from lib/html2rss/configs/telenor.com/newsroom.yml rename to configs/telenor.com/newsroom.yml diff --git a/lib/html2rss/configs/teneriffa-news.com/news.yml b/configs/teneriffa-news.com/news.yml similarity index 100% rename from lib/html2rss/configs/teneriffa-news.com/news.yml rename to configs/teneriffa-news.com/news.yml diff --git a/lib/html2rss/configs/tennet.eu/news.yml b/configs/tennet.eu/news.yml similarity index 100% rename from lib/html2rss/configs/tennet.eu/news.yml rename to configs/tennet.eu/news.yml diff --git a/lib/html2rss/configs/test.de/archiv.yml b/configs/test.de/archiv.yml similarity index 100% rename from lib/html2rss/configs/test.de/archiv.yml rename to configs/test.de/archiv.yml diff --git a/lib/html2rss/configs/theeastafrican.co.ke/news.yml b/configs/theeastafrican.co.ke/news.yml similarity index 100% rename from lib/html2rss/configs/theeastafrican.co.ke/news.yml rename to configs/theeastafrican.co.ke/news.yml diff --git a/lib/html2rss/configs/theguardian.com/international_mostpopular.yml b/configs/theguardian.com/international_mostpopular.yml similarity index 100% rename from lib/html2rss/configs/theguardian.com/international_mostpopular.yml rename to configs/theguardian.com/international_mostpopular.yml diff --git a/lib/html2rss/configs/thenationalnews.com/news.yml b/configs/thenationalnews.com/news.yml similarity index 100% rename from lib/html2rss/configs/thenationalnews.com/news.yml rename to configs/thenationalnews.com/news.yml diff --git a/lib/html2rss/configs/thoughtworks.com/insights.yml b/configs/thoughtworks.com/insights.yml similarity index 100% rename from lib/html2rss/configs/thoughtworks.com/insights.yml rename to configs/thoughtworks.com/insights.yml diff --git a/lib/html2rss/configs/tno.nl/news.yml b/configs/tno.nl/news.yml similarity index 100% rename from lib/html2rss/configs/tno.nl/news.yml rename to configs/tno.nl/news.yml diff --git a/lib/html2rss/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml b/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml similarity index 100% rename from lib/html2rss/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml rename to configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml diff --git a/lib/html2rss/configs/tralac.org/news.yml b/configs/tralac.org/news.yml similarity index 100% rename from lib/html2rss/configs/tralac.org/news.yml rename to configs/tralac.org/news.yml diff --git a/lib/html2rss/configs/tudelft.nl/news.yml b/configs/tudelft.nl/news.yml similarity index 100% rename from lib/html2rss/configs/tudelft.nl/news.yml rename to configs/tudelft.nl/news.yml diff --git a/lib/html2rss/configs/tue.nl/news.yml b/configs/tue.nl/news.yml similarity index 100% rename from lib/html2rss/configs/tue.nl/news.yml rename to configs/tue.nl/news.yml diff --git a/lib/html2rss/configs/vattenfall.com/newsroom.yml b/configs/vattenfall.com/newsroom.yml similarity index 100% rename from lib/html2rss/configs/vattenfall.com/newsroom.yml rename to configs/vattenfall.com/newsroom.yml diff --git a/lib/html2rss/configs/visitfinland.com/press-releases.yml b/configs/visitfinland.com/press-releases.yml similarity index 100% rename from lib/html2rss/configs/visitfinland.com/press-releases.yml rename to configs/visitfinland.com/press-releases.yml diff --git a/lib/html2rss/configs/webentwickler-jobs.de/in.yml b/configs/webentwickler-jobs.de/in.yml similarity index 100% rename from lib/html2rss/configs/webentwickler-jobs.de/in.yml rename to configs/webentwickler-jobs.de/in.yml diff --git a/lib/html2rss/configs/who.int/news.yml b/configs/who.int/news.yml similarity index 100% rename from lib/html2rss/configs/who.int/news.yml rename to configs/who.int/news.yml diff --git a/lib/html2rss/configs/worldbank.org/news.yml b/configs/worldbank.org/news.yml similarity index 100% rename from lib/html2rss/configs/worldbank.org/news.yml rename to configs/worldbank.org/news.yml diff --git a/lib/html2rss/configs/www.consilium.europa.eu/press-releases.yml b/configs/www.consilium.europa.eu/press-releases.yml similarity index 100% rename from lib/html2rss/configs/www.consilium.europa.eu/press-releases.yml rename to configs/www.consilium.europa.eu/press-releases.yml diff --git a/lib/html2rss/configs/www.ecdc.europa.eu/news-events.yml b/configs/www.ecdc.europa.eu/news-events.yml similarity index 100% rename from lib/html2rss/configs/www.ecdc.europa.eu/news-events.yml rename to configs/www.ecdc.europa.eu/news-events.yml diff --git a/lib/html2rss/configs/www.enisa.europa.eu/news.yml b/configs/www.enisa.europa.eu/news.yml similarity index 100% rename from lib/html2rss/configs/www.enisa.europa.eu/news.yml rename to configs/www.enisa.europa.eu/news.yml diff --git a/lib/html2rss/configs/www.europarl.europa.eu/press-room.yml b/configs/www.europarl.europa.eu/press-room.yml similarity index 100% rename from lib/html2rss/configs/www.europarl.europa.eu/press-room.yml rename to configs/www.europarl.europa.eu/press-room.yml diff --git a/lib/html2rss/configs/www.gob.mx/profeco-prensa.yml b/configs/www.gob.mx/profeco-prensa.yml similarity index 100% rename from lib/html2rss/configs/www.gob.mx/profeco-prensa.yml rename to configs/www.gob.mx/profeco-prensa.yml diff --git a/html2rss-configs.gemspec b/html2rss-configs.gemspec deleted file mode 100644 index 3e8ef82d..00000000 --- a/html2rss-configs.gemspec +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -lib = File.expand_path('lib', __dir__) -$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require 'html2rss/configs/version' - -Gem::Specification.new do |spec| - spec.name = 'html2rss-configs' - spec.version = Html2rss::Configs::VERSION - spec.authors = ['Gil Desmarais'] - spec.email = %w[html2rss-configs@desmarais.de] - - spec.summary = 'Collection of ready-to-use html2rss configs.' - spec.description = 'Configs which contain information how to generate RSS items from websites.' - spec.homepage = 'https://github.com/html2rss/html2rss-configs' - spec.license = 'MIT' - spec.required_ruby_version = '>= 3.1' - - # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host' - # to allow pushing to a single host or delete this section to allow pushing to any host. - if spec.respond_to?(:metadata) - spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'" - - spec.metadata['homepage_uri'] = spec.homepage - spec.metadata['source_code_uri'] = 'https://github.com/html2rss/html2rss-configs' - spec.metadata['rubygems_mfa_required'] = 'true' - - # spec.metadata['changelog_uri'] = "TODO: Put your gem's CHANGELOG.md URL here." - else - raise 'RubyGems 2.0 or newer is required to protect against ' \ - 'public gem pushes.' - end - - # Specify which files should be added to the gem when it is released. - # The `git ls-files -z` loads the files in the RubyGem that have been added into git. - spec.files = Dir.chdir(File.expand_path(__dir__)) do - `git ls-files -z`.split("\x0").reject do |f| - f[0] == '.' || f.match(%r{^(bin|test|spec|features|Makefile|Gemfile)/?}) - end - end - spec.bindir = 'exe' - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } - spec.require_paths = ['lib'] - - spec.add_dependency 'html2rss' -end diff --git a/lib/html2rss/configs.rb b/lib/html2rss/configs.rb deleted file mode 100644 index 23940ea4..00000000 --- a/lib/html2rss/configs.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'html2rss/configs/version' -require 'yaml' - -require_relative 'configs/catalog' -require_relative 'configs/catalog_entry' - -module Html2rss - ## - # The namespace for this gem - module Configs - class Error < Html2rss::Error; end - class ConfigNotFound < Html2rss::Configs::Error; end - - ## - # @return [Array] - def self.file_names - @file_names ||= Dir[File.join(__dir__, '**', '*.yml')].freeze - end - - ## - # @param name [String] the name of the config to find. format: `domainname/name` - # @return [Hash] the hash to create a Html2rss::Config - def self.find_by_name(name) - raise 'name must be a string' unless name.is_a?(String) - raise 'name must be in folder/file format' unless name.include?('/') - - name = "#{name}.yml" unless name.end_with?('.yml') - - file_name = file_names.find { |f| f.end_with?(name) } - - raise ConfigNotFound unless file_name - - YAML.safe_load(File.open(file_name), symbolize_names: true).freeze - end - end -end diff --git a/lib/html2rss/configs/catalog.rb b/lib/html2rss/configs/catalog.rb deleted file mode 100644 index 26754941..00000000 --- a/lib/html2rss/configs/catalog.rb +++ /dev/null @@ -1,162 +0,0 @@ -# frozen_string_literal: true - -require 'uri' -require 'yaml' - -require_relative 'catalog_entry' - -module Html2rss - module Configs - ## - # Sole owner of embedded catalog serialization from packaged YAML configs. - module Catalog - class Error < Html2rss::Error; end - class MissingDirectoryTitle < Error; end - - module_function - - ## - # @return [Array] sorted catalog entries - def entries - Html2rss::Configs.file_names.sort.filter_map { |file_name| build_entry(file_name) } - end - - ## - # @param file_name [String] - # @return [CatalogEntry] - def build_entry(file_name) - assemble_entry(file_name, load_config(file_name)) - end - - ## - # @param file_name [String] - # @param config [Hash] - # @return [CatalogEntry] - def assemble_entry(file_name, config) - title = require_directory_title!(config.fetch(:directory, {}), file_name) - id, path = entry_identifiers(file_name) - - CatalogEntry.new( - id:, path:, source: 'embedded', - directory: directory_payload(config.fetch(:directory, {}), title), - channel: channel_payload(config.fetch(:channel, {}), title), - parameters: parameters_payload(config.fetch(:channel, {}), config[:parameters]) - ) - end - - ## - # @param file_name [String] - # @return [Array(String, String)] - def entry_identifiers(file_name) - id = entry_id(file_name) - [id, "/#{id}.rss"] - end - - ## - # @param file_name [String] - # @return [Hash] - def load_config(file_name) - YAML.safe_load_file(file_name, symbolize_names: true) - end - - ## - # @param directory [Hash] - # @param file_name [String] - # @return [String] - def require_directory_title!(directory, file_name) - title = directory[:title] - raise MissingDirectoryTitle, "Missing directory.title in #{file_name}" if title.to_s.strip.empty? - - title - end - - ## - # @param directory [Hash] - # @param title [String] - # @return [Hash{Symbol => Object}] - def directory_payload(directory, title) - { - title: title.to_s, - summary: directory[:summary], - topics: Array(directory[:topics]) - }.compact - end - - ## - # @param channel [Hash] - # @param title [String] - # @return [Hash{Symbol => Object}] - def channel_payload(channel, title) - { - url: channel.fetch(:url), - language: channel[:language], - title: channel[:title] || title.to_s - }.compact - end - - ## - # @param channel [Hash] - # @param parameters_block [Hash, nil] - # @return [Hash{Symbol => Object}] - def parameters_payload(channel, parameters_block) - { - schema: parameter_schema(channel.fetch(:url), parameters_block), - defaults: default_parameters(parameters_block) - } - end - - ## - # @param file_name [String] - # @return [String] - def entry_id(file_name) - relative = file_name.sub(%r{\A.*/configs/}, '') - File.join(*relative.split('/')[0..-2], File.basename(relative, '.yml')) - end - - ## - # @param parameters [Hash, nil] - # @return [Hash{String => Object}] - def default_parameters(parameters) - return {} unless parameters.is_a?(Hash) - - parameters.each_with_object({}) do |(name, config), defaults| - next unless config.is_a?(Hash) && config.key?(:default) - - defaults[name.to_s] = config[:default] - end - end - - ## - # @param channel_url [String] - # @param parameters [Hash, nil] - # @return [Hash{String => Object}] - def parameter_schema(channel_url, parameters) - schema = url_parameter_types(channel_url) - return schema unless parameters.is_a?(Hash) - - parameters.each do |name, config| - next unless config.is_a?(Hash) - - schema[name.to_s] = config.slice(:type).transform_keys(&:to_s) - end - schema - end - - ## - # @param url [String] - # @return [Hash{String => String}] - def url_parameter_types(url) - url.to_s.scan(/%[{<](\w+)[>}](\w)?/).each_with_object({}) do |(name, format), types| - types[name] = { 'type' => numeric_format?(format) ? 'integer' : 'string' } - end - end - - ## - # @param format [String, nil] - # @return [Boolean] - def numeric_format?(format) - %w[i d u].include?(format) - end - end - end -end diff --git a/lib/html2rss/configs/catalog_entry.rb b/lib/html2rss/configs/catalog_entry.rb deleted file mode 100644 index 77744405..00000000 --- a/lib/html2rss/configs/catalog_entry.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Configs - ## - # Value object for a single embedded catalog entry exposed to consumers. - CatalogEntry = Data.define( - :id, - :path, - :source, - :directory, - :channel, - :parameters - ) do - ## - # Serializes the entry to the catalog wire shape v1. - # - # @return [Hash{Symbol => Object}] - def to_h - { - id:, - path:, - source:, - directory:, - channel:, - parameters: - } - end - end - end -end diff --git a/lib/html2rss/configs/version.rb b/lib/html2rss/configs/version.rb deleted file mode 100644 index 72d0df9a..00000000 --- a/lib/html2rss/configs/version.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Configs - VERSION = '0.2.0' - end -end diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 72917e74..00000000 --- a/package-lock.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "html2rss-configs", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "devDependencies": { - "prettier": "3.8.1" - } - }, - "node_modules/prettier": { - "version": "3.8.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", - "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", - "dev": true, - "license": "MIT", - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - } - } -} diff --git a/package.json b/package.json deleted file mode 100644 index fb8b6221..00000000 --- a/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "private": true, - "devDependencies": { - "prettier": "3.8.1" - } -} diff --git a/spec/html2rss/configs_dynamic_spec.rb b/spec/html2rss/configs_dynamic_spec.rb deleted file mode 100644 index bfb2dde6..00000000 --- a/spec/html2rss/configs_dynamic_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Html2rss::Configs do - # Get all YAML config files - config_files = Dir.glob('lib/html2rss/configs/**/*.yml') - - # Filter by environment variable if specified - config_files = config_files.select { |f| f.include?(ENV['TEST_CONFIG']) } if ENV['TEST_CONFIG'] - - # Filter by RSpec example pattern if specified - if RSpec.configuration.filter_manager.inclusions.rules[:example] - pattern = RSpec.configuration.filter_manager.inclusions.rules[:example].first - config_files = config_files.select { |f| f.include?(pattern) } - end - - config_files.each do |config_file| - config_name = config_file.sub('lib/html2rss/configs/', '').tr('/', '_').gsub('.yml', '') - domain = config_file.sub('lib/html2rss/configs/', '').split('/').first - - describe "#{config_file.sub('lib/html2rss/configs/', '')} (#{config_name})", - config: config_name, - domain: domain, - relative_path: config_file.sub('lib/html2rss/configs/', '') do - it_behaves_like 'config.yml', config_file.sub('lib/html2rss/configs/', '') - end - end -end diff --git a/spec/html2rss/configs_spec.rb b/spec/html2rss/configs_spec.rb deleted file mode 100644 index 825c441b..00000000 --- a/spec/html2rss/configs_spec.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Html2rss::Configs do - it 'has a version number' do - expect(Html2rss::Configs::VERSION).not_to be_nil - end - - describe '.file_names' do - subject(:file_names) { described_class.file_names } - - specify(:aggregate_failures) do - expect(file_names).to be_an(Array) & be_frozen - end - end - - describe '.find_by_name' do - context 'with valid name' do - subject(:find_by_name) { described_class.find_by_name('adfc.de/pressemitteilungen') } - - specify(:aggregate_failures) do - expect(find_by_name).to be_a(Hash) & be_frozen - end - end - - context 'with name not being a String' do - it 'raises ConfigNotFound error' do - expect { described_class.find_by_name(:foobar) }.to raise_error(RuntimeError) - end - end - - context 'with name not not containing a folder' do - it 'raises ConfigNotFound error' do - expect { described_class.find_by_name('foobar') }.to raise_error(RuntimeError) - end - end - - context 'with inexistent config' do - it 'raises ConfigNotFound error' do - expect { described_class.find_by_name('foobar/baz') }.to raise_error( - Html2rss::Configs::ConfigNotFound - ) - end - end - end -end diff --git a/spec/lib/html2rss/configs/catalog_spec.rb b/spec/lib/html2rss/configs/catalog_spec.rb deleted file mode 100644 index 1367b006..00000000 --- a/spec/lib/html2rss/configs/catalog_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require 'html2rss/configs' - -RSpec.describe Html2rss::Configs::Catalog do - describe '.entries' do - subject(:entries) { described_class.entries } - - it 'returns sorted embedded catalog entries', :aggregate_failures do - expect(entries).not_to be_empty - expect(entries.map(&:id)).to eq(entries.map(&:id).sort) - expect(entries).to all(be_a(Html2rss::Configs::CatalogEntry)) - end - - context 'with anthropic.com/news' do - subject(:entry) { entries.find { |candidate| candidate.id == 'anthropic.com/news' } } - - it 'maps anthropic identifiers' do - expect(entry).to have_attributes( - id: 'anthropic.com/news', - path: '/anthropic.com/news.rss', - source: 'embedded' - ) - end - - it 'maps anthropic titles and parameters', :aggregate_failures do - expect(entry.directory[:title]).to eq('Anthropic — News') - expect(entry.channel[:title]).to eq('Anthropic — News') - expect(entry.parameters).to eq(schema: {}, defaults: {}) - end - end - end - - describe '.build_entry' do - it 'maps parameterized configs to schema and defaults', :aggregate_failures do - file_name = Html2rss::Configs.file_names.find { |name| name.end_with?('cnet.com/section_sub.yml') } - entry = described_class.build_entry(file_name) - - expect(entry.id).to eq('cnet.com/section_sub') - expect(entry.parameters[:defaults]).to eq('section' => 'tech') - expect(entry.parameters[:schema]['section']).to eq('type' => 'string') - end - end -end diff --git a/spec/botasaurus_fetch_configs_spec.rb b/test/botasaurus_fetch_configs_spec.rb similarity index 85% rename from spec/botasaurus_fetch_configs_spec.rb rename to test/botasaurus_fetch_configs_spec.rb index da32f1d3..ecfa541b 100644 --- a/spec/botasaurus_fetch_configs_spec.rb +++ b/test/botasaurus_fetch_configs_spec.rb @@ -31,10 +31,10 @@ describe 'CONFIGS' do it 'includes all configs that use the botasaurus strategy' do - botasaurus_files = Dir.glob('lib/html2rss/configs/**/*.yml').select do |file| + botasaurus_files = Dir.glob('configs/**/*.yml').select do |file| YAML.load_file(file)['strategy'] == 'botasaurus' end - expected_configs = botasaurus_files.map { |file| file.sub('lib/html2rss/configs/', '') }.sort + expected_configs = botasaurus_files.map { |file| file.delete_prefix('configs/') }.sort expect(described_class::CONFIGS.sort).to eq(expected_configs) end diff --git a/test/configs_dynamic_spec.rb b/test/configs_dynamic_spec.rb new file mode 100644 index 00000000..f6351c40 --- /dev/null +++ b/test/configs_dynamic_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +RSpec.describe 'configs' do + config_files = Dir.glob('configs/**/*.yml') + + config_files = config_files.select { |path| path.include?(ENV['TEST_CONFIG']) } if ENV['TEST_CONFIG'] + + if RSpec.configuration.filter_manager.inclusions.rules[:example] + pattern = RSpec.configuration.filter_manager.inclusions.rules[:example].first + config_files = config_files.select { |path| path.include?(pattern) } + end + + config_files.each do |config_file| + config_name = config_file.delete_prefix('configs/').tr('/', '_').gsub('.yml', '') + domain = config_file.delete_prefix('configs/').split('/').first + + describe "#{config_file.delete_prefix('configs/')} (#{config_name})", + config: config_name, + domain: domain, + relative_path: config_file.delete_prefix('configs/') do + it_behaves_like 'config.yml', config_file.delete_prefix('configs/') + end + end +end diff --git a/spec/helper_spec.rb b/test/helper_spec.rb similarity index 100% rename from spec/helper_spec.rb rename to test/helper_spec.rb diff --git a/test/registry.spec.rb b/test/registry.spec.rb new file mode 100644 index 00000000..b7063fbc --- /dev/null +++ b/test/registry.spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'open3' +require 'tmpdir' + +RSpec.describe 'registry bundle' do + let(:repo_root) { File.expand_path('..', __dir__) } + + it 'builds and loads with integrity-only trust', :aggregate_failures do + Dir.mktmpdir do |tmpdir| + output = File.join(tmpdir, 'registry-bundle.tar.gz') + env = { 'BUNDLE_GEMFILE' => File.join(repo_root, 'tool/Gemfile') } + stdout, stderr, status = Open3.capture3(env, File.join(repo_root, 'tool/registry-build'), '--output', output, + chdir: repo_root) + expect(status).to eq(0), [stdout, stderr].join + + extract_dir = File.join(tmpdir, 'extracted') + FileUtils.mkdir_p(extract_dir) + File.open(output, 'rb') { |io| Html2rss::Registry::Archive.extract!(io, into: extract_dir) } + + bundle = Html2rss::Registry::Bundle.load(extract_dir, trust: :integrity_only) + + expect(bundle.manifest.registry_id).to eq('official') + expect(bundle.catalog_entries).not_to be_empty + expect(bundle.catalog_entries.map(&:id)).to eq(bundle.catalog_entries.map(&:id).sort) + + anthropic = bundle.catalog_entries.find { |entry| entry.id == 'anthropic.com/news' } + expect(anthropic).to have_attributes( + id: 'anthropic.com/news', + path: '/anthropic.com/news.rss' + ) + expect(anthropic.directory[:title]).to eq('Anthropic — News') + expect(anthropic.channel[:title]).to eq('Anthropic — News') + expect(anthropic.parameters).to eq(schema: {}, defaults: {}) + + cnet = bundle.catalog_entries.find { |entry| entry.id == 'cnet.com/section_sub' } + expect(cnet.parameters[:defaults]).to eq('section' => 'tech') + expect(cnet.parameters[:schema]['section']).to eq('type' => 'string') + end + end +end diff --git a/spec/support/botasaurus_fetch_configs.rb b/test/support/botasaurus_fetch_configs.rb similarity index 100% rename from spec/support/botasaurus_fetch_configs.rb rename to test/support/botasaurus_fetch_configs.rb diff --git a/spec/support/helper.rb b/test/support/helper.rb similarity index 100% rename from spec/support/helper.rb rename to test/support/helper.rb diff --git a/spec/support/shared_examples/config.yml_spec.rb b/test/support/shared_examples/config.yml_spec.rb similarity index 93% rename from spec/support/shared_examples/config.yml_spec.rb rename to test/support/shared_examples/config.yml_spec.rb index 404eeea9..d2b9675a 100644 --- a/spec/support/shared_examples/config.yml_spec.rb +++ b/test/support/shared_examples/config.yml_spec.rb @@ -6,16 +6,14 @@ subject(:yaml) { YAML.safe_load_file(file_path) } let!(:file_path) do - File.expand_path(File.join(__dir__, '..', '..', '..', 'lib', 'html2rss', 'configs', file_name)) + File.expand_path(File.join(__dir__, '..', '..', '..', 'configs', file_name)) end let(:config) do - feed_name = file_path.split(File::Separator)[-2..].join(File::Separator) - config = {}.merge Html2rss::Configs.find_by_name(feed_name) - # Reuse runtime browser defaults so fetch specs exercise the same header shape as production. + loaded = YAML.safe_load_file(file_path, symbolize_names: true) + config = {}.merge(loaded) config[:headers] = Html2rss::Config::RequestHeaders.browser_defaults.merge(config.fetch(:headers, {})) - # Use provided params or extract defaults from parameters section if params config[:params] = params elsif config[:parameters] diff --git a/spec/spec_helper.rb b/test/test_helper.rb similarity index 58% rename from spec/spec_helper.rb rename to test/test_helper.rb index 98fb42ea..69b2d09c 100644 --- a/spec/spec_helper.rb +++ b/test/test_helper.rb @@ -1,20 +1,16 @@ # frozen_string_literal: true +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../tool/Gemfile', __dir__) + require 'bundler/setup' +require 'rspec' require 'tzinfo' - require 'html2rss' -require 'html2rss/configs' -Dir['./spec/support/**/*.rb'].each { |f| require f } - -Zeitwerk::Loader.eager_load_all +Dir[File.join(__dir__, 'support', '**', '*.rb')].each { |path| require path } RSpec.configure do |config| - # Enable flags like --only-failures and --next-failure config.example_status_persistence_file_path = '.rspec_status' - - # Disable RSpec exposing methods globally on `Module` and `main` config.disable_monkey_patching! config.filter_run_excluding fetch: true diff --git a/spec/bin/validate_configs_spec.rb b/test/validate.spec.rb similarity index 71% rename from spec/bin/validate_configs_spec.rb rename to test/validate.spec.rb index cb8aa075..f7086261 100644 --- a/spec/bin/validate_configs_spec.rb +++ b/test/validate.spec.rb @@ -3,9 +3,9 @@ require 'fileutils' require 'tmpdir' -RSpec.describe 'bin/validate_configs' do # rubocop:disable RSpec/DescribeClass - let(:script_path) { File.expand_path('../../bin/validate_configs', __dir__) } - let(:config_path) { File.join('lib', 'html2rss', 'configs', 'example.com', 'search.yml') } +RSpec.describe 'tool/validate' do + let(:script_path) { File.expand_path('../../tool/validate', __dir__) } + let(:config_path) { File.join('configs', 'example.com', 'search.yml') } let(:success_output) do a_string_including( "ok #{config_path}", @@ -28,12 +28,16 @@ def with_temp_config end def write_config(dir) - FileUtils.mkdir_p(File.join(dir, File.dirname(config_path))) + FileUtils.mkdir_p(File.join(dir, 'configs', File.dirname(config_path.delete_prefix('configs/')))) File.write(File.join(dir, config_path), valid_config) end def run_script(dir) - Dir.chdir(dir) { load script_path } + configs_dir = File.join(dir, 'configs') + ENV['CONFIGS_DIR'] = configs_dir + load script_path + ensure + ENV.delete('CONFIGS_DIR') end def valid_config diff --git a/Gemfile b/tool/Gemfile similarity index 67% rename from Gemfile rename to tool/Gemfile index 00152190..3b0a4af8 100644 --- a/Gemfile +++ b/tool/Gemfile @@ -2,9 +2,7 @@ source 'https://rubygems.org' -git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } - -gem 'html2rss', '~> 0.27' +gem 'html2rss', path: '../../html2rss' group :development do gem 'nokogiri' @@ -14,5 +12,3 @@ group :development do gem 'rubocop-performance' gem 'rubocop-rspec' end - -gemspec diff --git a/tool/Gemfile.lock b/tool/Gemfile.lock new file mode 100644 index 00000000..5cf2d4ed --- /dev/null +++ b/tool/Gemfile.lock @@ -0,0 +1,281 @@ +PATH + remote: ../../html2rss + specs: + html2rss (0.27.1) + addressable (~> 2.7) + brotli + dry-validation + faraday (> 2.0.1, < 3.0) + faraday-follow_redirects + faraday-gzip (~> 3) + kramdown + mcp (~> 1.2) + mime-types (> 3.0) + nokogiri (>= 1.10, < 2.0) + rack (~> 3.0) + rackup (~> 2.0) + regexp_parser + reverse_markdown (~> 3.0) + rss + sanitize + thor + tzinfo + webrick (~> 1.9) + zeitwerk + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.9.0) + public_suffix (>= 2.0.2, < 8.0) + ast (2.4.3) + bigdecimal (4.1.2) + brotli (0.8.0) + concurrent-ruby (1.3.8) + crass (1.0.7) + diff-lcs (1.6.2) + dry-configurable (1.4.0) + dry-core (~> 1.0) + zeitwerk (~> 2.6) + dry-core (1.2.0) + concurrent-ruby (~> 1.0) + logger + zeitwerk (~> 2.6) + dry-inflector (1.3.1) + dry-initializer (3.2.0) + dry-logic (1.6.0) + bigdecimal + concurrent-ruby (~> 1.0) + dry-core (~> 1.1) + zeitwerk (~> 2.6) + dry-schema (1.16.0) + concurrent-ruby (~> 1.0) + dry-configurable (~> 1.0, >= 1.0.1) + dry-core (~> 1.1) + dry-initializer (~> 3.2) + dry-logic (~> 1.6) + dry-types (~> 1.9, >= 1.9.1) + zeitwerk (~> 2.6) + dry-types (1.9.1) + bigdecimal (>= 3.0) + concurrent-ruby (~> 1.0) + dry-core (~> 1.0) + dry-inflector (~> 1.0) + dry-logic (~> 1.4) + zeitwerk (~> 2.6) + dry-validation (1.11.1) + concurrent-ruby (~> 1.0) + dry-core (~> 1.1) + dry-initializer (~> 3.2) + dry-schema (~> 1.14) + zeitwerk (~> 2.6) + faraday (2.14.3) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-follow_redirects (0.5.0) + faraday (>= 1, < 3) + faraday-gzip (3.1.0) + faraday (>= 2.0, < 3) + zlib (~> 3.0) + faraday-net_http (3.4.4) + net-http (~> 0.5) + hana (1.3.7) + json (2.21.2) + json_schemer (2.5.0) + bigdecimal + hana (~> 1.3) + regexp_parser (~> 2.0) + simpleidn (~> 0.2) + kramdown (2.5.2) + rexml (>= 3.4.4) + language_server-protocol (3.17.0.6) + lint_roller (1.1.0) + logger (1.7.0) + mcp (1.3.0) + json_schemer (>= 2.4) + mime-types (3.7.0) + logger + mime-types-data (~> 3.2025, >= 3.2025.0507) + mime-types-data (3.2026.0701) + net-http (0.9.1) + uri (>= 0.11.1) + nokogiri (1.19.4-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.4-aarch64-linux-musl) + racc (~> 1.4) + nokogiri (1.19.4-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.4-arm-linux-musl) + racc (~> 1.4) + nokogiri (1.19.4-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.4-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.4-x86_64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.4-x86_64-linux-musl) + racc (~> 1.4) + parallel (2.1.0) + parser (3.3.12.0) + ast (~> 2.4.1) + racc + prism (1.9.0) + public_suffix (7.0.5) + racc (1.8.1) + rack (3.2.7) + rackup (2.3.1) + rack (>= 3) + rainbow (3.1.1) + regexp_parser (2.12.0) + reverse_markdown (3.0.2) + nokogiri + rexml (3.4.4) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.8) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.7) + rss (0.3.3) + rexml + rubocop (1.89.0) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (>= 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.49.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.50.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + rubocop-performance (1.27.0) + lint_roller (~> 1.1) + rubocop (>= 1.89.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) + rubocop-rspec (3.10.2) + lint_roller (~> 1.1) + regexp_parser (>= 2.0) + rubocop (~> 1.86, >= 1.86.2) + ruby-progressbar (1.13.0) + sanitize (7.0.0) + crass (~> 1.0.2) + nokogiri (>= 1.16.8) + simpleidn (0.2.3) + thor (1.5.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + uri (1.1.1) + webrick (1.9.2) + zeitwerk (2.8.3) + zlib (3.2.3) + +PLATFORMS + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl + +DEPENDENCIES + html2rss! + nokogiri + public_suffix + rspec (~> 3.0) + rubocop + rubocop-performance + rubocop-rspec + +CHECKSUMS + addressable (2.9.0) sha256=7fdf6ac3660f7f4e867a0838be3f6cf722ace541dd97767fa42bc6cfa980c7af + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd + brotli (0.8.0) sha256=0c5a42046b3b603fb109656881147fd76064c034b7d19c1b4fcc32a093a4d55d + bundler (4.0.17) sha256=214e21431b5665dd2f99df8a5511c6b151d7a72e8015c8b38f8b775b61cbb6c1 + concurrent-ruby (1.3.8) sha256=b2f1be836e968ccc78ccfce277ea79c72a88633f22306782c16ff23fb415d1e1 + crass (1.0.7) sha256=94868719948664c89ddcaf0a37c65048413dfcb1c869470a5f7a7ceb5390b295 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + dry-configurable (1.4.0) sha256=e35d1b5f3c081753ef361f564919db79000f32cfa6f20ee3a3ba5921b41b73ce + dry-core (1.2.0) sha256=0cc5a7da88df397f153947eeeae42e876e999c1e30900f3c536fb173854e96a1 + dry-inflector (1.3.1) sha256=7fb0c2bb04f67638f25c52e7ba39ab435d922a3a5c3cd196120f63accb682dcc + dry-initializer (3.2.0) sha256=37d59798f912dc0a1efe14a4db4a9306989007b302dcd5f25d0a2a20c166c4e3 + dry-logic (1.6.0) sha256=da6fedbc0f90fc41f9b0cc7e6f05f5d529d1efaef6c8dcc8e0733f685745cea2 + dry-schema (1.16.0) sha256=cd3aaeabc0f1af66ec82a29096d4c4fb92a0a58b9dae29a22b1bbceb78985727 + dry-types (1.9.1) sha256=baebeecdb9f8395d6c9d227b62011279440943e3ef2468fe8ccc1ba11467f178 + dry-validation (1.11.1) sha256=70900bb5a2d911c8aab566d3e360c6bff389b8bf92ea8e04885ce51c41ff8085 + faraday (2.14.3) sha256=1882247e6766615c8220b4392bf1d27f6ebb63d8e28267587cef1fb0bf37f278 + faraday-follow_redirects (0.5.0) sha256=5cde93c894b30943a5d2b93c2fe9284216a6b756f7af406a1e55f211d97d10ad + faraday-gzip (3.1.0) sha256=320783690be169f9b7ddde11598b77156951343753f66a9ab98b1f6694433ff8 + faraday-net_http (3.4.4) sha256=0e78af151747ed1b00f33e25973b4bc220d7f16c00c39676817c8b12331eb588 + hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d + html2rss (0.27.1) + json (2.21.2) sha256=1f1d3b7cf2b3ba1a69beca0bb6db13d5438b80bff3cd54cdaaa620b9b07c1c6a + json_schemer (2.5.0) sha256=2f01fb4cce721a4e08dd068fc2030cffd0702a7f333f1ea2be6e8991f00ae396 + kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa + language_server-protocol (3.17.0.6) sha256=5ef2c0c138f8267e1bc631d3328347d354f96724b0af22f2c79516120443b7f0 + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mcp (1.3.0) sha256=9395aa3a054eb8986b7714ec8abb25b533729af896f4875da4c4cb7a3024fbae + mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56 + mime-types-data (3.2026.0701) sha256=cd8811e1fb89d836499ba0582368a10ee74cef929ba956d1d5ddca045e6a730f + net-http (0.9.1) sha256=25ba0b67c63e89df626ed8fac771d0ad24ad151a858af2cc8e6a716ca4336996 + nokogiri (1.19.4-aarch64-linux-gnu) sha256=1269fb644a6de405057a53dd5c762b1209b43ca7424f839454d3dbc677c31a8f + nokogiri (1.19.4-aarch64-linux-musl) sha256=35c65b9ce72b3bb03207bdbe7067915019dc18c1b9b59139684bd6690fdd01af + nokogiri (1.19.4-arm-linux-gnu) sha256=a301313e38bb065d68239e79734bcd6f56fb6efaacebde29e9abf2a4735340ca + nokogiri (1.19.4-arm-linux-musl) sha256=588923c101bcfa78869734d247d25b598674323e7f22474fc468f6e5647311eb + nokogiri (1.19.4-arm64-darwin) sha256=a46db9853286e6597b36ebc6953817d15acf3a299583eb3f89fdc6f91dd63527 + nokogiri (1.19.4-x86_64-darwin) sha256=7fd17057d3e1f00e9954a74b3cd76595d3d4a5ef233b7ed9599047c204f70551 + nokogiri (1.19.4-x86_64-linux-gnu) sha256=379fae440b28915e3f19d752ce2dcf8465ed2b2fbefd2a7ca0dd497bc981a06a + nokogiri (1.19.4-x86_64-linux-musl) sha256=17dfb7c1fa194ae02fbf7c51a7afc8d278045ab3fdacfd86f91d02d7b274470b + parallel (2.1.0) sha256=b35258865c2e31134c5ecb708beaaf6772adf9d5efae28e93e99260877b09356 + parser (3.3.12.0) sha256=21a6d7f755d5a24dfbdc6e6b772e4e879a52e7631a88bc5a3a134606052c9828 + prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 + public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + rack (3.2.7) sha256=93e13e1c24f93556671d85d2d79fa228c3485815c50d7e2f265b5330c6528fb7 + rackup (2.3.1) sha256=6c79c26753778e90983761d677a48937ee3192b3ffef6bc963c0950f94688868 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + regexp_parser (2.12.0) sha256=35a916a1d63190ab5c9009457136ae5f3c0c7512d60291d0d1378ba18ce08ebb + reverse_markdown (3.0.2) sha256=818ebb92ce39dbb1a291690dd1ec9a6d62530d4725296b17e9c8f668f9a5b8af + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47 + rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c + rss (0.3.3) sha256=37ef1ec4d691d67edbe92159079a4e89a0965e6a6b32246009ae3d734d12d1ab + rubocop (1.89.0) sha256=4dee8e3ee9c45e474834efd9e8d6fd031e8331c8dacdff0de4ad65ae0a6faae7 + rubocop-ast (1.50.0) sha256=b9ca88300da0803ee222ad20cdb30494c0a784eed06fdc35d254b06d662788db + rubocop-performance (1.27.0) sha256=eeeb1374d062a368ee1c787b70eb0b0cc4b184cb1f8565f424760946146d61ce + rubocop-rspec (3.10.2) sha256=0b3e2ecc592cd10ecbf0095bb58d1e357905276e069643523cc19eb7495f65e2 + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + sanitize (7.0.0) sha256=269d1b9d7326e69307723af5643ec032ff86ad616e72a3b36d301ac75a273984 + simpleidn (0.2.3) sha256=08ce96f03fa1605286be22651ba0fc9c0b2d6272c9b27a260bc88be05b0d2c29 + thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73 + tzinfo (2.0.6) sha256=8daf828cc77bcf7d63b0e3bdb6caa47e2272dcfaf4fbfe46f8c3a9df087a829b + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 + webrick (1.9.2) sha256=beb4a15fc474defed24a3bda4ffd88a490d517c9e4e6118c3edce59e45864131 + zeitwerk (2.8.3) sha256=2c85125a8467ce069e20123d1e709a08955c9d29c118c25b46b7b7fafdbb92e5 + zlib (3.2.3) sha256=5bd316698b32f31a64ab910a8b6c282442ca1626a81bbd6a1674e8522e319c20 + +BUNDLED WITH + 4.0.17 diff --git a/tool/registry-build b/tool/registry-build new file mode 100755 index 00000000..34ab4ec0 --- /dev/null +++ b/tool/registry-build @@ -0,0 +1,136 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'bundler/setup' +require 'fileutils' +require 'optparse' +require 'openssl' +require 'rubygems/package' +require 'stringio' +require 'tmpdir' +require 'zlib' +require 'html2rss' + +REPO_ROOT = File.expand_path('..', __dir__) +CONFIGS_DIR = File.join(REPO_ROOT, 'configs') +DIST_DIR = File.join(REPO_ROOT, 'dist') + +## +# @param relative_paths [Array] +# @return [void] +def reject_duplicate_ids!(relative_paths) + ids = relative_paths.map { |path| Html2rss::Registry::CatalogBuilder.entry_id(path) } + duplicates = ids.group_by(&:itself).select { |_, group| group.size > 1 }.keys + return if duplicates.empty? + + abort "Duplicate config ids: #{duplicates.sort.join(', ')}" +end + +## +# @param staging [String] +# @param relative_paths [Array] +# @return [void] +def enforce_build_limits!(staging, relative_paths) + raise "Too many config files (#{relative_paths.size})" if relative_paths.size > Html2rss::Registry::Archive::MAX_FILES + + total_bytes = relative_paths.sum do |relative_path| + size = File.size(File.join(staging, relative_path)) + if size > Html2rss::Registry::Archive::MAX_FILE_BYTES + raise "Config exceeds max bytes (#{Html2rss::Registry::Archive::MAX_FILE_BYTES}): #{relative_path}" + end + + size + end + + return unless total_bytes > Html2rss::Registry::Archive::MAX_TARBALL_BYTES + + raise "Bundle exceeds max bytes (#{Html2rss::Registry::Archive::MAX_TARBALL_BYTES})" +end + +## +# @param staging [String] +# @param manifest [Html2rss::Registry::Manifest] +# @return [void] +def sign_manifest!(staging, manifest) + private_key = OpenSSL::PKey.read(ENV.fetch('REGISTRY_SIGNING_KEY')) + signature = private_key.sign(nil, manifest.canonical_bytes) + File.write( + File.join(staging, Html2rss::Registry::Manifest::SIGNATURE_FILE), + [signature].pack('m0') + ) +end + +## +# @param tar [Gem::Package::TarWriter] +# @param source_dir [String] +# @param root_dir [String] +# @return [void] +def add_tree_to_tar(tar, source_dir, root_dir) + Dir.glob(File.join(source_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |path| + next if File.directory?(path) + + relative = path.delete_prefix("#{root_dir}/") + tar.add_file(relative, 0o644) { |io| io.write(File.binread(path)) } + end +end + +## +# @param source_dir [String] +# @param output_path [String] +# @return [void] +def write_tar_gz(source_dir, output_path) + tar_data = StringIO.new + Gem::Package::TarWriter.new(tar_data) do |tar| + add_tree_to_tar(tar, source_dir, source_dir) + end + + Zlib::GzipWriter.open(output_path) { |gz| gz.write(tar_data.string) } +end + +options = { + sign: false, + output: File.join(DIST_DIR, 'registry-bundle.tar.gz'), + registry_id: ENV.fetch('REGISTRY_ID', 'official'), + version: ENV.fetch('REGISTRY_VERSION', ENV.fetch('GITHUB_REF_NAME', Time.now.utc.strftime('%Y.%m.%d'))), + public_key_id: ENV.fetch('REGISTRY_PUBLIC_KEY_ID', 'html2rss:registry:2026') +} + +OptionParser.new do |opts| + opts.banner = 'Usage: tool/registry-build [options]' + opts.on('--sign', 'Sign manifest with Ed25519 private key from REGISTRY_SIGNING_KEY') do + options[:sign] = true + end + opts.on('--output PATH', 'Output tarball path') { |value| options[:output] = value } + opts.on('--registry-id ID', 'Registry identifier') { |value| options[:registry_id] = value } + opts.on('--version VERSION', 'Registry version string') { |value| options[:version] = value } + opts.on('--public-key-id ID', 'Signing key id for manifest.public_key_id') do |value| + options[:public_key_id] = value + end +end.parse! + +abort "Missing configs directory: #{CONFIGS_DIR}" unless File.directory?(CONFIGS_DIR) + +Dir.mktmpdir('registry-build') do |staging| + FileUtils.cp_r(File.join(CONFIGS_DIR, '.'), File.join(staging, 'configs')) + relative_paths = Html2rss::Registry::CatalogBuilder.config_paths(staging).sort + abort 'No config files found under configs/' if relative_paths.empty? + + reject_duplicate_ids!(relative_paths) + enforce_build_limits!(staging, relative_paths) + + file_index = Html2rss::Registry::Manifest.file_index(staging, relative_paths) + manifest = Html2rss::Registry::Manifest.build( + file_index:, + registry_id: options[:registry_id], + version: options[:version], + public_key_id: options[:public_key_id] + ) + + File.write(File.join(staging, Html2rss::Registry::Manifest::MANIFEST_FILE), manifest.to_json) + sign_manifest!(staging, manifest) if options[:sign] + + FileUtils.mkdir_p(File.dirname(options[:output])) + write_tar_gz(staging, options[:output]) +end + +puts "Built #{options[:output]}" diff --git a/tool/validate b/tool/validate new file mode 100755 index 00000000..26f8924a --- /dev/null +++ b/tool/validate @@ -0,0 +1,58 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'bundler/setup' +require 'html2rss' + +REPO_ROOT = File.expand_path('..', __dir__) +CONFIGS_DIR = ENV.fetch('CONFIGS_DIR', File.join(REPO_ROOT, 'configs')) +CONFIGS_GLOB = File.join(CONFIGS_DIR, '**', '*.yml') + +unless Html2rss::Config.respond_to?(:validate_yaml) + warn 'Error: installed html2rss gem does not support Config.validate_yaml.' + warn 'Run: bundle update html2rss' + exit 1 +end + +## +# @param file [String] +# @return [String] +def display_path(file) + prefix = "#{CONFIGS_DIR}/" + return file.delete_prefix(prefix) if file.start_with?(prefix) + + file.delete_prefix("#{REPO_ROOT}/") +end + +files = Dir[CONFIGS_GLOB] +failed = [] + +files.each do |file| + result = Html2rss::Config.validate_yaml(file) + + if result.success? + puts "ok #{display_path(file)}" + else + puts "FAIL #{display_path(file)}" + result.errors.to_h.each do |key, messages| + Array(messages).each { |msg| warn " #{key}: #{msg}" } + end + failed << file + end +rescue Psych::Exception => error + puts "FAIL #{display_path(file)}" + warn " parse: #{error.message}" + failed << file +rescue StandardError => error + puts "FAIL #{display_path(file)}" + warn " validation: #{error.class}: #{error.message}" + failed << file +end + +puts +if failed.empty? + puts "#{files.size} configs validated successfully." +else + warn "#{failed.size}/#{files.size} configs failed validation." + exit 1 +end From b9d93e53aeb986e7bd175ba20e3b1dd27f69f38c Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 14:57:20 +0200 Subject: [PATCH 02/11] docs: document signed registry bundles and registry.id rules --- .../html2rss-config/reference/catalog.md | 51 ++++++-- AGENTS.md | 45 +++++-- README.md | 123 +++--------------- 3 files changed, 91 insertions(+), 128 deletions(-) diff --git a/.agents/skills/html2rss-config/reference/catalog.md b/.agents/skills/html2rss-config/reference/catalog.md index 4b129029..6dc7e829 100644 --- a/.agents/skills/html2rss-config/reference/catalog.md +++ b/.agents/skills/html2rss-config/reference/catalog.md @@ -1,11 +1,32 @@ # Catalog metadata (Feed Directory) -Every YAML in this repo is a catalog entry. Registry bundle serialization is owned by `Html2rss::Registry::CatalogBuilder` in the core `html2rss` gem — build with `make registry-build`; do not re-walk YAML in other repos. +Every YAML in this repo is a catalog entry. Declare feed identity explicitly — do not rely on filesystem path. + +## Feed identity (`registry.id`) + +```yaml +registry: + id: europa.eu/europarl/press-room + aliases: [] # optional; previous ids after a rename +directory: + title: "European Parliament — Press room" + topics: [civic] +``` + +| Field | Rule | +| --- | --- | +| `registry.id` | Unique within the bundle; slug `org/surface[/variant]`; lowercase; `[a-z0-9._/-]`; no `www.` prefix | +| `registry.aliases` | Optional previous ids (same bundle); resolved at feed lookup; **not** listed in catalog API | + +Pick a sensible path under `configs/` (group by organization). Path is contributor ergonomics only. + +**Rename workflow:** change `registry.id`; add the old id to `aliases`; remove the alias in a later release when comfortable. ## Required fields | Field | Rule | | ------------------ | -------------------------------------------------------------- | +| `registry.id` | Explicit feed identity (see above) | | `directory.topics` | 1–2 values from controlled vocabulary — [topics.md](topics.md) | | `directory.title` | Human label: `{Organization} — {Feed surface}` (em dash) | | `channel.title` | Same string as `directory.title` (RSS and OPML) | @@ -22,9 +43,9 @@ Every YAML in this repo is a catalog entry. Registry bundle serialization is own | Config path | `directory.summary` | | ------------------------- | --------------------------------------------------------------------- | -| `who.int/news.yml` | Official news and statements from the World Health Organization. | -| `apple.com/newsroom.yml` | Product announcements and press releases from Apple. | -| `github.com/releases.yml` | Release notes for a GitHub repository (owner and name as parameters). | +| `configs/who.int/news.yml` | Official news and statements from the World Health Organization. | +| `configs/apple/newsroom.yml` | Product announcements and press releases from Apple. | +| `configs/github.com/releases.yml` | Release notes for a GitHub repository (owner and name as parameters). | ## Parameterized configs @@ -37,9 +58,9 @@ Describe feed intent in `directory.title`, not the template URL. | Config path | `directory.title` | | ------------------------ | -------------------------------- | -| `anthropic.com/news.yml` | Anthropic — News | -| `who.int/news.yml` | World Health Organization — News | -| `apnews.com/hub.yml` | AP News — Top stories | +| `configs/anthropic.com/news.yml` | Anthropic — News | +| `configs/who.int/news.yml` | World Health Organization — News | +| `configs/apnews.com/hub.yml` | AP News — Top stories | ## Verification @@ -49,13 +70,15 @@ make validate make registry-build ``` -`CatalogBuilder.build_entry` raises `MissingDirectoryTitle` when `directory.title` is absent. +`make registry-build` emits a signed `registry.v1` bundle; `Html2rss::Registry::CatalogBuilder.build_entry` raises `MissingDirectoryTitle` when `directory.title` is absent. -## Downstream consumers +## Downstream pipeline -| Consumer | How it reads catalog data | -| -------------------------- | ------------------------------------------------------------------------------------------------------------ | -| `html2rss-web` | `GET /api/v1/configs` merges registry bundles with local `feeds.yml` entries that include `directory.title` | -| Feed Directory (docs site) | Fetches catalog JSON from a running `html2rss-web` instance | +| Layer | Owner | Responsibility | +| --- | --- | --- | +| YAML + bundle | `html2rss-configs/` | `configs/`; `tool/registry-build` → signed tarball on release | +| Catalog rows | `html2rss/` | `Html2rss::Registry::CatalogBuilder` from verified bundle manifest | +| Sync + API | `html2rss-web/` | `config/registries.yml`, `Registry::Sync`, `Registry::Index`, `GET /api/v1/configs` | +| Browse UI | `html2rss.github.io/` | Fetches catalog JSON from a running instance | -Do not duplicate catalog expansion logic outside `Html2rss::Registry::CatalogBuilder`. +Wire rows add `source: registry` and `registry: ` in the web handler. Do not duplicate catalog expansion outside `CatalogBuilder`. diff --git a/AGENTS.md b/AGENTS.md index 373b9d38..23b33d93 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,13 +6,31 @@ Primary goal: add or repair configs that are stable, shippable, and easy to veri ## Scope -- Source of truth here: `lib/html2rss/configs/`. -- Do not hand-edit generated schema output. +- Source of truth here: `configs/` (flat repo layout: `configs/`, `tool/`, `test/`). +- Do not hand-edit generated schema output or signed bundle artifacts (`dist/`). - Keep config work separate from downstream docs, web, or example changes unless the task explicitly includes them. -## Defaults +## Feed identity (`registry.id`) + +Every config **must** declare identity explicitly — never rely on filesystem path: + +```yaml +registry: + id: europa.eu/europarl/press-room + aliases: [] # optional; previous ids after a rename +directory: + title: "European Parliament — Press room" + topics: [civic] +``` + +Rules: -- Use the registrable domain folder, not a subdomain folder, unless there is a strong existing reason. +- **`registry.id`** — unique within the bundle; slug path `org/surface[/variant]`; lowercase; `[a-z0-9._/-]`; no `www.` prefix. +- **`registry.aliases`** — optional list of previous ids (same bundle only); resolved by `html2rss-web` at feed lookup; **not** listed in the catalog API. +- **Filesystem path** — pick a sensible path under `configs/` (group by organization, not raw hostname folklore). Path does not define identity. +- **Rename workflow** — change `registry.id` to the new slug; add the old id to `aliases`; remove the alias in a later release when comfortable. + +## Defaults - Start from the cleanest article list the site offers, not the marketing homepage by default. - Prefer stable list/detail extraction over extracting every possible field. - If the site only becomes reliable on a narrower path, use that narrower path. @@ -28,10 +46,16 @@ Primary goal: add or repair configs that are stable, shippable, and easy to veri ## Feed Directory catalog -- **Serialization owner:** `Html2rss::Configs::Catalog` in `lib/html2rss/configs/catalog.rb` builds wire-ready entries from packaged YAML. Do not duplicate YAML walking in `html2rss-web` or the docs site. -- **Entry type:** `Html2rss::Configs::CatalogEntry` (`Data.define`) — use `#to_h` for the v1 wire shape (`id`, `path`, `source`, `directory`, `channel`, `parameters`). +- **Bundle build:** `make registry-build` (`tool/registry-build`) walks YAML, validates `registry.id`, and emits a signed `registry.v1` bundle (`manifest.json`, `manifest.sig`, `configs/`). +- **Catalog serialization owner:** `Html2rss::Registry::CatalogBuilder` in the core `html2rss` gem — do not duplicate YAML walking in `html2rss-web` or the docs site. +- **Wire shape:** domain entries from `CatalogBuilder`; `html2rss-web` adds `source: registry` and `registry: ` at the HTTP layer. - **Agent reference:** [.agents/skills/html2rss-config/reference/catalog.md](.agents/skills/html2rss-config/reference/catalog.md). +## Release + +- Tag push runs `.github/workflows/release.yml`: `make registry-build -- --sign` uploads `dist/registry-bundle.tar.gz` to GitHub Releases. +- Instances sync the official bundle via `html2rss-web` `Registry::Sync` (default channel `html2rss-official`). + ## Surface Selection Prefer these surfaces first: @@ -141,10 +165,11 @@ cd ../html2rss html2rss feed /abs/path/to/config.yml ``` -3. Catalog serialization for changed configs: +3. Registry bundle verification for changed configs: ```bash -bundle exec rspec spec/lib/html2rss/configs/catalog_spec.rb +bundle exec rspec test/registry.spec.rb +make registry-build ``` 4. Repo-wide validation in this repo: @@ -164,14 +189,14 @@ make test - Faraday-backed candidate: ```bash -bundle exec rspec --tag fetch --example 'example.com/feed.yml' spec/html2rss/configs_dynamic_spec.rb +bundle exec rspec --tag fetch --example 'example.com/feed.yml' test/configs_dynamic_spec.rb ``` - Botasaurus-backed candidate: ```bash BOTASAURUS_SCRAPER_URL=http://localhost:4010 \ -bundle exec rspec --tag fetch --example 'example.com/feed.yml' spec/html2rss/configs_dynamic_spec.rb +bundle exec rspec --tag fetch --example 'example.com/feed.yml' test/configs_dynamic_spec.rb ``` 7. If fetch still fails, decide explicitly whether: diff --git a/README.md b/README.md index 782a7f7c..84414030 100644 --- a/README.md +++ b/README.md @@ -2,121 +2,36 @@ # html2rss-configs -This repository contains `html2rss` feed configurations for many websites. +Curated feed YAML for the [html2rss](https://html2rss.github.io) Feed Directory. Configs live under `configs/`; releases ship as signed `registry.v1` bundles. -## 🌐 Community & Resources +## Quick links -| Resource | Description | Link | -| ------------------------------------- | ----------------------------------------------------------- | ------------------------------------------------------------------ | -| **📚 Documentation & Feed Directory** | Complete guides, tutorials, and browse 100+ pre-built feeds | [html2rss.github.io](https://html2rss.github.io) | -| **💬 Community Discussions** | Get help, share ideas, and connect with other users | [GitHub Discussions](https://github.com/orgs/html2rss/discussions) | -| **📋 Project Board** | Track development progress and upcoming features | [View Project Board](https://github.com/orgs/html2rss/projects) | -| **💖 Support Development** | Help fund ongoing development and maintenance | [Sponsor on GitHub](https://github.com/sponsors/gildesmarais) | +| Resource | Link | +| --- | --- | +| **Feed Directory** | [Browse pre-built feeds](https://html2rss.github.io/feed-directory) | +| **Contributing** | [AGENTS.md](AGENTS.md) and [catalog reference](.agents/skills/html2rss-config/reference/catalog.md) | +| **Documentation** | [html2rss.github.io](https://html2rss.github.io) | -**Quick Start Options:** - -- **Need a specific feed?** → Browse the [feed directory](https://html2rss.github.io/feed-directory) -- **Want to create feeds?** → Use the [web application](https://html2rss.github.io/web-application) -- **Ruby Developer?** → Check out the [Ruby gem documentation](https://html2rss.github.io/ruby-gem) -- **Want to contribute?** → See our [contributing guide](https://html2rss.github.io/get-involved/contributing) - -## Dynamic Parameters - -Parameterized configs should include a `parameters` section with default values: - -```yaml -parameters: - query: - type: string - default: "technology" - category: - type: string - default: "news" - -channel: - url: https://example.com/search?q=%s&cat=%s - # ... rest of config -``` - -The `type` field specifies the parameter type (currently only `string` is supported), and `default` provides the default value used by this repository's validation and fetch tests. - -Notes: - -- Only configs that use `%s` placeholders need a `parameters` section. -- Callers can still override those defaults at runtime with `html2rss feed ... --params ...`. -- Dynamic substitution applies to `channel` and `headers`; selectors are not parameterized by this feature. - -## Validation - -Use both schema-aware editing and runtime validation before committing: +## Verify locally ```bash -# Validate all configs with the runtime validator -make validate - -# Validate a single config directly -bundle exec html2rss validate lib/html2rss/configs/github.com/releases.yml - -# Export the current JSON Schema locally for editor use -make schema +make ready # lint, validate all configs, run tests +make registry-build # build unsigned bundle to dist/ ``` -The JSON Schema is useful for editor autocompletion and basic structural checks. Runtime validation remains authoritative for merged defaults and cross-field rules. - -## Testing - -Uses **dynamic test generation** - no individual spec files needed! +Single-file validation uses the core CLI from the sibling `html2rss` checkout: ```bash -# Test all configs -bundle exec rspec spec/html2rss/configs_dynamic_spec.rb - -# Test specific config -make test-config CONFIG=github.com/releases.yml - -# Test domain -make test-domain DOMAIN=github.com - -# Run live fetch tests for the full corpus -make test-fetch-all-configs - -# Run the Botasaurus-backed fetch subset -BOTASAURUS_SCRAPER_URL=http://localhost:4010 \ -make test-fetch-botasaurus-configs +html2rss validate configs/github.com/releases.yml +html2rss feed configs/github.com/releases.yml ``` -**Adding new configs**: Create the YAML file with top-level `directory.topics` (controlled vocabulary), run `make validate`, then run the generated tests. No dedicated spec file is needed. - -The fetch suite has two lanes: - -- `make test-fetch-all-configs` runs all `:fetch` examples. Configs marked as Botasaurus-backed are skipped unless `BOTASAURUS_SCRAPER_URL` is configured. -- `make test-fetch-botasaurus-configs` runs only the Botasaurus-backed config subset and requires `BOTASAURUS_SCRAPER_URL`. - -**Config folder convention**: Place configs under the registrable domain folder (e.g., `example.com/` or `bbc.co.uk/`). Legacy subdomain folders (e.g., `news.example.com/`) are allowed but not preferred. +## Layout -## Editor Setup (JSON Schema) - -Get inline validation and autocompletion when editing configs in your IDE. -All config files already carry the schema modeline at the top: - -```yaml -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +``` +configs/ # feed YAML (each file declares registry.id) +tool/ # validate, registry-build +test/ # dynamic config specs + registry bundle spec ``` -Any editor with [yaml-language-server](https://github.com/redhat-developer/yaml-language-server) -support (VS Code + [YAML extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml), -Neovim, Helix, …) will automatically pick up the schema when opening a config file. - -The included `.vscode/settings.json` additionally associates the schema with all -configs via a glob pattern, so new files get validation before the modeline is added. - -> `make schema` writes `schema/html2rss-config.schema.json` locally — useful for -> offline editing or when you want to pin against the currently installed gem version. - -## Documentation - -- [Selectors Reference](https://html2rss.github.io/ruby-gem/reference/selectors/) -- [Dynamic Parameters](https://html2rss.github.io/ruby-gem/how-to/dynamic-parameters/) -- [CLI Reference](https://html2rss.github.io/ruby-gem/reference/cli-reference/) -- [Contributing Guide](https://html2rss.github.io/get-involved/contributing) -- [Sponsorship Page](https://html2rss.github.io/get-involved/sponsoring) +Release tags trigger CI to sign and publish `registry-bundle.tar.gz`. Running `html2rss-web` instances sync that artifact (or mount a local bundle via `path:` in `config/registries.yml`). From b7394d542f6230a5015452b1db8d1f02636f5695 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 15:22:52 +0200 Subject: [PATCH 03/11] fix(ci): resolve tool/Gemfile path for setup-ruby Remove working-directory from setup-ruby so BUNDLE_GEMFILE=tool/Gemfile resolves from the repo root instead of tool/tool/Gemfile. --- .github/workflows/ruby.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index ccb910c6..9b4982ac 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -24,7 +24,6 @@ jobs: with: ruby-version: 3.3 bundler-cache: true - working-directory: tool - name: Run quality gate run: make ready @@ -43,7 +42,6 @@ jobs: with: ruby-version: 3.3 bundler-cache: true - working-directory: tool - name: setup remote origin run: >- From ce1ba0cb51533fa341e4d02e3c25f74bd49474b8 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 16:22:01 +0200 Subject: [PATCH 04/11] feat(registry): flat configs repo with signed release governance Add explicit registry.id to all bundled configs, replace legacy bins with tool/registry-build, and harden release CI with registry-release env, draft tags, and signed bundle verification on Ruby 4.0. --- .../html2rss-config/reference/catalog.md | 2 +- .github/workflows/release.yml | 31 ++++++- .github/workflows/ruby.yml | 4 +- .gitignore | 1 + .rubocop.yml | 11 ++- .ruby-version | 1 + AGENTS.md | 16 +++- Makefile | 2 +- bin/setup | 33 ------- bin/validate_configs | 44 --------- configs/aalto.fi/news.yml | 2 + configs/academictransfer.com/jobs.yml | 2 + configs/adfc.de/pressemitteilungen.yml | 2 + configs/ahram.org.eg/egypt.yml | 2 + configs/ai.meta.com/blog.yml | 2 + configs/akorda.kz/events.yml | 2 + configs/angop.ao/noticias.yml | 2 + configs/anthropic.com/news.yml | 2 + configs/anu.edu.au/news.yml | 2 + configs/apnews.com/hub.yml | 2 + configs/apple.com/newsroom.yml | 2 + configs/aps.dz/national-news.yml | 2 + configs/asml.com/press-releases.yml | 2 + configs/aspi.org.au/strategist.yml | 2 + configs/bbc.co.uk/available_episodes.yml | 2 + configs/bbc.com/mundo.yml | 2 + configs/blog.mondediplo.net/feed.yml | 2 + configs/brookings.edu/articles.yml | 2 + .../bundesregierung.de/pressemitteilungen.yml | 2 + configs/canarianweekly.com/front.yml | 2 + configs/caricom.org/news.yml | 2 + configs/carnegieendowment.org/research.yml | 2 + configs/cepal.org/press-releases.yml | 2 + configs/chalmers.se/news.yml | 2 + .../research-publications.yml | 2 + configs/cinemascore.com/index.yml | 2 + configs/cleanenergywire.org/news.yml | 2 + configs/cnet.com/section_sub.yml | 2 + configs/cohere.com/blog.yml | 2 + configs/computerbase.de/meistgelesen.yml | 2 + configs/cplp.org/noticias.yml | 2 + configs/csiro.au/news.yml | 2 + configs/curia.europa.eu/press-releases.yml | 2 + configs/cutle.fish/index.yml | 2 + configs/deepmind.google/blog.yml | 2 + configs/deraktionaer.de/meistgelesen.yml | 2 + ...ials_data_documentation_technotes_json.yml | 2 + configs/dfs.de/pressemitteilungen.yml | 2 + configs/diis.dk/research.yml | 2 + configs/dirco.gov.za/media-statements.yml | 2 + configs/dsw-info.de/presse.yml | 2 + .../elasticsearch-release-notes.yml | 2 + configs/elysee.fr/actualites.yml | 2 + configs/equinor.com/news.yml | 2 + configs/espn.com/f1.yml | 2 + configs/ethz.ch/eth-news.yml | 2 + configs/eures.europa.eu/news.yml | 2 + configs/faa.gov/newsroom.yml | 2 + configs/fao.org/newsroom.yml | 2 + configs/fia.com/documents.yml | 2 + configs/figma.com/blog.yml | 2 + configs/formula1.com/latest.yml | 2 + configs/forumsec.org/publications.yml | 2 + configs/ftc.gov/press-releases.yml | 2 + configs/github.com/releases.yml | 2 + configs/go.dev/release-history.yml | 2 + configs/grafana.com/whatsnew.yml | 2 + configs/haaretz.com/israel-news.yml | 2 + configs/hai.stanford.edu/news.yml | 2 + configs/iaapa.org/news.yml | 2 + configs/icj-cij.org/press-releases.yml | 2 + configs/icrc.org/news.yml | 2 + configs/iea.org/news.yml | 2 + configs/iiss.org/online-analysis.yml | 2 + configs/ikea.com/newsroom.yml | 2 + configs/imdb.com/ratings.yml | 2 + configs/imo.org/press-briefings.yml | 2 + configs/imperial.ac.uk/news.yml | 2 + .../karriere_arbeitsleben_heiko_mell.yml | 2 + configs/isro.gov.in/press.yml | 2 + configs/issafrica.org/iss-today.yml | 2 + configs/issafrica.org/press-releases.yml | 2 + configs/kaist.ac.kr/news.yml | 2 + configs/kinocheck.de/filmstarts.yml | 2 + configs/kompas.com/nasional.yml | 2 + configs/lego.com/press-releases.yml | 2 + .../machinelearning.apple.com/research.yml | 2 + configs/maersk.com/press-releases.yml | 2 + configs/microsoft.com/azure-products.yml | 2 + configs/mistral.ai/news.yml | 2 + configs/mozilla.org/security-advisories.yml | 2 + configs/msf.org/latest.yml | 2 + configs/nbim.no/press-releases.yml | 2 + configs/news.cn/latest.yml | 2 + configs/news24.com/top-stories.yml | 2 + configs/newyorker.com/magazine.yml | 2 + configs/nigerianstat.gov.ng/news.yml | 2 + configs/nikkei.com/asia-technology.yml | 2 + configs/nokia.com/newsroom.yml | 2 + configs/nomanssky.com/news.yml | 2 + configs/notion.com/blog.yml | 2 + configs/novonordisk.com/latest-news.yml | 2 + configs/ntu.edu.sg/news.yml | 2 + configs/nupi.no/news.yml | 2 + configs/occrp.org/news.yml | 2 + configs/oecd.org/newsroom.yml | 2 + configs/opec.org/press-releases.yml | 2 + configs/panapress.com/latest.yml | 2 + .../search.yml | 2 + configs/philstar.com/nation.yml | 2 + configs/phys.org/weekly.yml | 2 + configs/pib.gov.in/press-releases.yml | 2 + configs/rbb24.de/meistgeklickt.yml | 2 + configs/robinwood.de/aktuelles.yml | 2 + configs/s3.amazonaws.com/popular_movies.yml | 2 + configs/sacu.int/news.yml | 2 + configs/sadc.int/latest-news.yml | 2 + configs/sebastianvettel.de/news.yml | 2 + configs/shopify.com/blog.yml | 2 + configs/sipri.org/news.yml | 2 + configs/softwareleadweekly.com/issues.yml | 2 + configs/solarthermalworld.org/news.yml | 2 + configs/spektrum.de/meistgelesen.yml | 2 + configs/spiegel.de/impressum_autor.yml | 2 + configs/spotify.com/newsroom.yml | 2 + .../hot_network_questions.yml | 2 + configs/steuerzahler.de/news.yml | 2 + configs/stripes.com/index.yml | 2 + configs/support.apple.com/en_gb_ht201222.yml | 2 + configs/support.apple.com/exchange_repair.yml | 2 + configs/tap.info.tn/politics.yml | 2 + configs/telenor.com/newsroom.yml | 2 + configs/teneriffa-news.com/news.yml | 2 + configs/tennet.eu/news.yml | 2 + configs/test.de/archiv.yml | 2 + configs/theeastafrican.co.ke/news.yml | 2 + .../international_mostpopular.yml | 2 + configs/thenationalnews.com/news.yml | 2 + configs/thoughtworks.com/insights.yml | 2 + configs/tno.nl/news.yml | 2 + .../aktuelle_nachrichten.yml | 2 + configs/tralac.org/news.yml | 2 + configs/tudelft.nl/news.yml | 2 + configs/tue.nl/news.yml | 2 + configs/vattenfall.com/newsroom.yml | 2 + configs/visitfinland.com/press-releases.yml | 2 + configs/webentwickler-jobs.de/in.yml | 2 + configs/who.int/news.yml | 2 + configs/worldbank.org/news.yml | 2 + .../press-releases.yml | 2 + configs/www.ecdc.europa.eu/news-events.yml | 2 + configs/www.enisa.europa.eu/news.yml | 2 + configs/www.europarl.europa.eu/press-room.yml | 2 + configs/www.gob.mx/profeco-prensa.yml | 2 + test/registry.spec.rb | 42 --------- test/registry_build_spec.rb | 75 +++++++++++++++ tool/Gemfile | 2 +- tool/Gemfile.lock | 6 +- tool/registry-build | 93 ++++++++----------- 159 files changed, 463 insertions(+), 188 deletions(-) create mode 100644 .ruby-version delete mode 100755 bin/setup delete mode 100755 bin/validate_configs delete mode 100644 test/registry.spec.rb create mode 100644 test/registry_build_spec.rb diff --git a/.agents/skills/html2rss-config/reference/catalog.md b/.agents/skills/html2rss-config/reference/catalog.md index 6dc7e829..8b67e9a0 100644 --- a/.agents/skills/html2rss-config/reference/catalog.md +++ b/.agents/skills/html2rss-config/reference/catalog.md @@ -65,7 +65,7 @@ Describe feed intent in `directory.title`, not the template URL. ## Verification ```bash -bundle exec rspec test/registry.spec.rb +bundle exec rspec test/registry_build_spec.rb make validate make registry-build ``` diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6bd13dd4..bf0e375b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Release "on": push: tags: - - '*' + - 'v*' env: BUNDLE_GEMFILE: tool/Gemfile @@ -11,18 +11,24 @@ env: jobs: release: runs-on: ubuntu-latest + environment: registry-release permissions: contents: write steps: - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get install -yqq yamllint + - name: Set up Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: 3.3 + ruby-version: '4.0' bundler-cache: true - working-directory: tool + + - name: Run quality gate + run: make ready - name: Build signed registry bundle env: @@ -30,8 +36,27 @@ jobs: REGISTRY_SIGNING_KEY: ${{ secrets.REGISTRY_SIGNING_KEY }} run: make registry-build -- --sign + - name: Verify release artifact + env: + REGISTRY_PUBLIC_KEY_ID: html2rss:registry:2026 + REGISTRY_PUBLIC_KEY_PEM: ${{ secrets.REGISTRY_PUBLIC_KEY_PEM }} + run: | + bundle exec ruby -rhtml2rss -ropenssl -rtmpdir -e " + Dir.mktmpdir do |dir| + File.open('dist/registry-bundle.tar.gz', 'rb') do |io| + Html2rss::Registry::Archive.extract!(io, into: dir) + end + public_key = OpenSSL::PKey.read(ENV.fetch('REGISTRY_PUBLIC_KEY_PEM')) + public_keys = { ENV.fetch('REGISTRY_PUBLIC_KEY_ID') => public_key } + bundle = Html2rss::Registry::Bundle.load(dir, trust: :signed, public_keys:) + abort 'empty catalog' if bundle.catalog_entries.empty? + puts \"Verified #{bundle.catalog_entries.size} catalog entries\" + end + " + - name: Upload release artifact uses: softprops/action-gh-release@v2 with: files: dist/registry-bundle.tar.gz generate_release_notes: true + draft: true diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 9b4982ac..352302f8 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: 3.3 + ruby-version: '4.0' bundler-cache: true - name: Run quality gate @@ -40,7 +40,7 @@ jobs: - name: Set up Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: 3.3 + ruby-version: '4.0' bundler-cache: true - name: setup remote origin diff --git a/.gitignore b/.gitignore index 797943fb..bbb31c7f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /.bundle/ +/tool/.bundle/ /.yardoc /_yardoc/ /coverage/ diff --git a/.rubocop.yml b/.rubocop.yml index 290e8ea7..72c91cad 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -5,7 +5,7 @@ plugins: AllCops: DisplayCopName: true NewCops: enable - TargetRubyVersion: 3.3 + TargetRubyVersion: 4.0 Exclude: - vendor/**/* - tool/Gemfile.lock @@ -17,13 +17,18 @@ RSpec/DescribeClass: Exclude: - "test/**/*.yml_spec.rb" - "test/configs_dynamic_spec.rb" - - "test/registry.spec.rb" + - "test/registry_build_spec.rb" - "test/validate.spec.rb" Metrics/BlockLength: Exclude: - - "test/registry.spec.rb" + - "test/registry_build_spec.rb" - "test/validate.spec.rb" + - "tool/registry-build" + +RSpec/ExampleLength: + Exclude: + - "test/registry_build_spec.rb" Naming/RescuedExceptionsVariableName: PreferredName: error diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 00000000..5186d070 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +4.0 diff --git a/AGENTS.md b/AGENTS.md index 23b33d93..a0d2ce5f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,9 @@ Rules: - **Rename workflow** — change `registry.id` to the new slug; add the old id to `aliases`; remove the alias in a later release when comfortable. ## Defaults + +Local development expects **Ruby 4.0** (`mise exec -- …` or `.ruby-version` when present). CI and release workflows run on Ruby 4.0 only. + - Start from the cleanest article list the site offers, not the marketing homepage by default. - Prefer stable list/detail extraction over extracting every possible field. - If the site only becomes reliable on a narrower path, use that narrower path. @@ -53,8 +56,15 @@ Rules: ## Release -- Tag push runs `.github/workflows/release.yml`: `make registry-build -- --sign` uploads `dist/registry-bundle.tar.gz` to GitHub Releases. -- Instances sync the official bundle via `html2rss-web` `Registry::Sync` (default channel `html2rss-official`). +Publisher flow (`.github/workflows/release.yml`): + +1. Push a `v*` tag — the release job runs in the GitHub **`registry-release`** environment (requires maintainer approval before signing secrets are used). +2. CI runs `make ready`, then `make registry-build -- --sign` with `REGISTRY_SIGNING_KEY` from environment secrets. +3. The workflow verifies the tarball with `REGISTRY_PUBLIC_KEY_PEM` from the **`registry-release`** environment (operator-supplied public key PEM; must match `html2rss-web/config/registries.yml`), not the private signing key. +4. `softprops/action-gh-release@v2` uploads `dist/registry-bundle.tar.gz` as a **draft** GitHub Release with generated notes. +5. A maintainer reviews the draft release and clicks **Publish release** — instances only pick up published releases via `html2rss-web` `Registry::Sync` (default channel `html2rss-official`). + +Local bundle build (unsigned): `make registry-build`. Signing locally requires `REGISTRY_SIGNING_KEY` in the environment. ## Surface Selection @@ -168,7 +178,7 @@ html2rss feed /abs/path/to/config.yml 3. Registry bundle verification for changed configs: ```bash -bundle exec rspec test/registry.spec.rb +bundle exec rspec test/registry_build_spec.rb make registry-build ``` diff --git a/Makefile b/Makefile index b78eeec1..41d766eb 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ validate: test: $(BUNDLE) rspec test -registry-build: +registry-build: validate $(BUNDLE_RUBY) tool/registry-build test-fetch-changed-configs: diff --git a/bin/setup b/bin/setup deleted file mode 100755 index 243eb575..00000000 --- a/bin/setup +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -IFS=$'\n\t' - -echo "==> Installing Ruby dependencies..." -bundle install - -echo -echo "==> Installing Node dependencies (required for make lint)..." -npm ci --ignore-scripts --no-fund --no-audit - -echo -echo "==> Checking system tools..." -if command -v yamllint &>/dev/null; then - echo " yamllint: ok ($(yamllint --version))" -else - echo " yamllint: not found" - echo " Install with one of:" - echo " brew install yamllint" - echo " pip install yamllint" -fi - -echo -echo "==> Setup complete. Common commands:" -echo -echo " make # lint + validate + test" -echo " make lint # yamllint, rubocop, prettier" -echo " make validate # validate all configs against the html2rss schema" -echo " make test # run rspec" -echo -echo " make test-config CONFIG=github.com/releases.yml # test a single config" -echo " make test-domain DOMAIN=github.com # test all configs for a domain" -echo " make schema # write schema locally (optional)" diff --git a/bin/validate_configs b/bin/validate_configs deleted file mode 100755 index 13442e96..00000000 --- a/bin/validate_configs +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'bundler/setup' -require 'html2rss' - -unless Html2rss::Config.respond_to?(:validate_yaml) - warn 'Error: installed html2rss gem does not support Config.validate_yaml.' - warn 'Run: bundle update html2rss' - exit 1 -end - -files = Dir['lib/html2rss/configs/**/*.yml'] -failed = [] - -files.each do |file| - result = Html2rss::Config.validate_yaml(file) - - if result.success? - puts "ok #{file}" - else - puts "FAIL #{file}" - result.errors.to_h.each do |key, messages| - Array(messages).each { |msg| warn " #{key}: #{msg}" } - end - failed << file - end -rescue Psych::Exception => error - puts "FAIL #{file}" - warn " parse: #{error.message}" - failed << file -rescue StandardError => error - puts "FAIL #{file}" - warn " validation: #{error.class}: #{error.message}" - failed << file -end - -puts -if failed.empty? - puts "#{files.size} configs validated successfully." -else - warn "#{failed.size}/#{files.size} configs failed validation." - exit 1 -end diff --git a/configs/aalto.fi/news.yml b/configs/aalto.fi/news.yml index 5b970fa5..841a424d 100644 --- a/configs/aalto.fi/news.yml +++ b/configs/aalto.fi/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: aalto.fi/news directory: topics: - science diff --git a/configs/academictransfer.com/jobs.yml b/configs/academictransfer.com/jobs.yml index 616f483d..60beb2fa 100644 --- a/configs/academictransfer.com/jobs.yml +++ b/configs/academictransfer.com/jobs.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: academictransfer.com/jobs directory: topics: - jobs diff --git a/configs/adfc.de/pressemitteilungen.yml b/configs/adfc.de/pressemitteilungen.yml index f3455448..27a18262 100644 --- a/configs/adfc.de/pressemitteilungen.yml +++ b/configs/adfc.de/pressemitteilungen.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: adfc.de/pressemitteilungen directory: topics: - civic diff --git a/configs/ahram.org.eg/egypt.yml b/configs/ahram.org.eg/egypt.yml index d424a15f..518e156f 100644 --- a/configs/ahram.org.eg/egypt.yml +++ b/configs/ahram.org.eg/egypt.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ahram.org.eg/egypt directory: topics: - news diff --git a/configs/ai.meta.com/blog.yml b/configs/ai.meta.com/blog.yml index bcb60de2..36640067 100644 --- a/configs/ai.meta.com/blog.yml +++ b/configs/ai.meta.com/blog.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ai.meta.com/blog directory: topics: - tech diff --git a/configs/akorda.kz/events.yml b/configs/akorda.kz/events.yml index 5f7443b1..c77ac858 100644 --- a/configs/akorda.kz/events.yml +++ b/configs/akorda.kz/events.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: akorda.kz/events directory: topics: - civic diff --git a/configs/angop.ao/noticias.yml b/configs/angop.ao/noticias.yml index 7426cc87..4eafef1a 100644 --- a/configs/angop.ao/noticias.yml +++ b/configs/angop.ao/noticias.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: angop.ao/noticias directory: topics: - news diff --git a/configs/anthropic.com/news.yml b/configs/anthropic.com/news.yml index 143101e9..af1cbc81 100644 --- a/configs/anthropic.com/news.yml +++ b/configs/anthropic.com/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: anthropic.com/news directory: topics: - tech diff --git a/configs/anu.edu.au/news.yml b/configs/anu.edu.au/news.yml index a640e45e..2e4f5aa4 100644 --- a/configs/anu.edu.au/news.yml +++ b/configs/anu.edu.au/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: anu.edu.au/news directory: topics: - science diff --git a/configs/apnews.com/hub.yml b/configs/apnews.com/hub.yml index 940e3f5a..69a22acd 100644 --- a/configs/apnews.com/hub.yml +++ b/configs/apnews.com/hub.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: apnews.com/hub directory: topics: - news diff --git a/configs/apple.com/newsroom.yml b/configs/apple.com/newsroom.yml index a1ebc7ff..7b301e61 100644 --- a/configs/apple.com/newsroom.yml +++ b/configs/apple.com/newsroom.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: apple.com/newsroom directory: topics: - tech diff --git a/configs/aps.dz/national-news.yml b/configs/aps.dz/national-news.yml index e5acd115..7a0bfa88 100644 --- a/configs/aps.dz/national-news.yml +++ b/configs/aps.dz/national-news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: aps.dz/national-news directory: topics: - news diff --git a/configs/asml.com/press-releases.yml b/configs/asml.com/press-releases.yml index 5dcc32fe..5550871e 100644 --- a/configs/asml.com/press-releases.yml +++ b/configs/asml.com/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: asml.com/press-releases directory: topics: - tech diff --git a/configs/aspi.org.au/strategist.yml b/configs/aspi.org.au/strategist.yml index 7b804fdc..c4954622 100644 --- a/configs/aspi.org.au/strategist.yml +++ b/configs/aspi.org.au/strategist.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: aspi.org.au/strategist directory: topics: - civic diff --git a/configs/bbc.co.uk/available_episodes.yml b/configs/bbc.co.uk/available_episodes.yml index 9f09fc42..d0154422 100644 --- a/configs/bbc.co.uk/available_episodes.yml +++ b/configs/bbc.co.uk/available_episodes.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: bbc.co.uk/available_episodes directory: topics: - entertainment diff --git a/configs/bbc.com/mundo.yml b/configs/bbc.com/mundo.yml index 865254d2..3cde3ea9 100644 --- a/configs/bbc.com/mundo.yml +++ b/configs/bbc.com/mundo.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: bbc.com/mundo directory: topics: - news diff --git a/configs/blog.mondediplo.net/feed.yml b/configs/blog.mondediplo.net/feed.yml index d1d84cfb..7d723b9a 100644 --- a/configs/blog.mondediplo.net/feed.yml +++ b/configs/blog.mondediplo.net/feed.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: blog.mondediplo.net/feed directory: topics: - news diff --git a/configs/brookings.edu/articles.yml b/configs/brookings.edu/articles.yml index d3fbc1e4..5dd51ef8 100644 --- a/configs/brookings.edu/articles.yml +++ b/configs/brookings.edu/articles.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: brookings.edu/articles directory: topics: - civic diff --git a/configs/bundesregierung.de/pressemitteilungen.yml b/configs/bundesregierung.de/pressemitteilungen.yml index 4fe0f5c7..b7673581 100644 --- a/configs/bundesregierung.de/pressemitteilungen.yml +++ b/configs/bundesregierung.de/pressemitteilungen.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: bundesregierung.de/pressemitteilungen directory: topics: - civic diff --git a/configs/canarianweekly.com/front.yml b/configs/canarianweekly.com/front.yml index 0249b697..395ed541 100644 --- a/configs/canarianweekly.com/front.yml +++ b/configs/canarianweekly.com/front.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: canarianweekly.com/front directory: topics: - news diff --git a/configs/caricom.org/news.yml b/configs/caricom.org/news.yml index 3ddae5ef..f7dead8a 100644 --- a/configs/caricom.org/news.yml +++ b/configs/caricom.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: caricom.org/news directory: topics: - civic diff --git a/configs/carnegieendowment.org/research.yml b/configs/carnegieendowment.org/research.yml index 6ae11985..9fc241ff 100644 --- a/configs/carnegieendowment.org/research.yml +++ b/configs/carnegieendowment.org/research.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: carnegieendowment.org/research directory: topics: - civic diff --git a/configs/cepal.org/press-releases.yml b/configs/cepal.org/press-releases.yml index 865e1b14..918ff530 100644 --- a/configs/cepal.org/press-releases.yml +++ b/configs/cepal.org/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cepal.org/press-releases directory: topics: - civic diff --git a/configs/chalmers.se/news.yml b/configs/chalmers.se/news.yml index 18546827..224010b8 100644 --- a/configs/chalmers.se/news.yml +++ b/configs/chalmers.se/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: chalmers.se/news directory: topics: - science diff --git a/configs/chathamhouse.org/research-publications.yml b/configs/chathamhouse.org/research-publications.yml index bda48a18..2f4f1fcf 100644 --- a/configs/chathamhouse.org/research-publications.yml +++ b/configs/chathamhouse.org/research-publications.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: chathamhouse.org/research-publications directory: topics: - civic diff --git a/configs/cinemascore.com/index.yml b/configs/cinemascore.com/index.yml index 64797052..1e011302 100644 --- a/configs/cinemascore.com/index.yml +++ b/configs/cinemascore.com/index.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cinemascore.com/index directory: topics: - entertainment diff --git a/configs/cleanenergywire.org/news.yml b/configs/cleanenergywire.org/news.yml index 3d2547e8..18fb1956 100644 --- a/configs/cleanenergywire.org/news.yml +++ b/configs/cleanenergywire.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cleanenergywire.org/news directory: topics: - energy diff --git a/configs/cnet.com/section_sub.yml b/configs/cnet.com/section_sub.yml index 7fda12f6..f8184813 100644 --- a/configs/cnet.com/section_sub.yml +++ b/configs/cnet.com/section_sub.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: cnet.com/section_sub directory: topics: - tech diff --git a/configs/cohere.com/blog.yml b/configs/cohere.com/blog.yml index bb1a9f07..8e206d84 100644 --- a/configs/cohere.com/blog.yml +++ b/configs/cohere.com/blog.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cohere.com/blog directory: topics: - tech diff --git a/configs/computerbase.de/meistgelesen.yml b/configs/computerbase.de/meistgelesen.yml index cab24b95..7af302e3 100644 --- a/configs/computerbase.de/meistgelesen.yml +++ b/configs/computerbase.de/meistgelesen.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: computerbase.de/meistgelesen directory: topics: - tech diff --git a/configs/cplp.org/noticias.yml b/configs/cplp.org/noticias.yml index 92e2f8d3..bc5d577a 100644 --- a/configs/cplp.org/noticias.yml +++ b/configs/cplp.org/noticias.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cplp.org/noticias directory: topics: - civic diff --git a/configs/csiro.au/news.yml b/configs/csiro.au/news.yml index 69ff1421..30bdb368 100644 --- a/configs/csiro.au/news.yml +++ b/configs/csiro.au/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: csiro.au/news directory: topics: - science diff --git a/configs/curia.europa.eu/press-releases.yml b/configs/curia.europa.eu/press-releases.yml index af189872..612e548f 100644 --- a/configs/curia.europa.eu/press-releases.yml +++ b/configs/curia.europa.eu/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: curia.europa.eu/press-releases directory: topics: - civic diff --git a/configs/cutle.fish/index.yml b/configs/cutle.fish/index.yml index 2f8420f6..4224b718 100644 --- a/configs/cutle.fish/index.yml +++ b/configs/cutle.fish/index.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: cutle.fish/index directory: topics: - tech diff --git a/configs/deepmind.google/blog.yml b/configs/deepmind.google/blog.yml index b89549b7..6d3733c1 100644 --- a/configs/deepmind.google/blog.yml +++ b/configs/deepmind.google/blog.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: deepmind.google/blog directory: topics: - science diff --git a/configs/deraktionaer.de/meistgelesen.yml b/configs/deraktionaer.de/meistgelesen.yml index ff5f3595..14c09373 100644 --- a/configs/deraktionaer.de/meistgelesen.yml +++ b/configs/deraktionaer.de/meistgelesen.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: deraktionaer.de/meistgelesen directory: topics: - finance diff --git a/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml b/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml index 0de4b516..4026f9ab 100644 --- a/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml +++ b/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: developer.apple.com/tutorials_data_documentation_technotes_json directory: topics: - tech diff --git a/configs/dfs.de/pressemitteilungen.yml b/configs/dfs.de/pressemitteilungen.yml index bb073e0f..00dca1f8 100644 --- a/configs/dfs.de/pressemitteilungen.yml +++ b/configs/dfs.de/pressemitteilungen.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: dfs.de/pressemitteilungen directory: topics: - civic diff --git a/configs/diis.dk/research.yml b/configs/diis.dk/research.yml index 3331929a..2ba60c1c 100644 --- a/configs/diis.dk/research.yml +++ b/configs/diis.dk/research.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: diis.dk/research directory: topics: - civic diff --git a/configs/dirco.gov.za/media-statements.yml b/configs/dirco.gov.za/media-statements.yml index b3472c34..bb999f73 100644 --- a/configs/dirco.gov.za/media-statements.yml +++ b/configs/dirco.gov.za/media-statements.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: dirco.gov.za/media-statements directory: topics: - civic diff --git a/configs/dsw-info.de/presse.yml b/configs/dsw-info.de/presse.yml index 1764f066..69f870c9 100644 --- a/configs/dsw-info.de/presse.yml +++ b/configs/dsw-info.de/presse.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: dsw-info.de/presse directory: topics: - civic diff --git a/configs/elastic.co/elasticsearch-release-notes.yml b/configs/elastic.co/elasticsearch-release-notes.yml index 7ddb0408..e73169f5 100644 --- a/configs/elastic.co/elasticsearch-release-notes.yml +++ b/configs/elastic.co/elasticsearch-release-notes.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: elastic.co/elasticsearch-release-notes directory: topics: - tech diff --git a/configs/elysee.fr/actualites.yml b/configs/elysee.fr/actualites.yml index 51d61091..11380d0a 100644 --- a/configs/elysee.fr/actualites.yml +++ b/configs/elysee.fr/actualites.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: elysee.fr/actualites directory: topics: - civic diff --git a/configs/equinor.com/news.yml b/configs/equinor.com/news.yml index b4e038db..76b365e4 100644 --- a/configs/equinor.com/news.yml +++ b/configs/equinor.com/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: equinor.com/news directory: topics: - energy diff --git a/configs/espn.com/f1.yml b/configs/espn.com/f1.yml index b868f591..88f38c62 100644 --- a/configs/espn.com/f1.yml +++ b/configs/espn.com/f1.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: espn.com/f1 directory: topics: - sports diff --git a/configs/ethz.ch/eth-news.yml b/configs/ethz.ch/eth-news.yml index 0cd77390..2093c116 100644 --- a/configs/ethz.ch/eth-news.yml +++ b/configs/ethz.ch/eth-news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ethz.ch/eth-news directory: topics: - science diff --git a/configs/eures.europa.eu/news.yml b/configs/eures.europa.eu/news.yml index a3243440..e8f877f9 100644 --- a/configs/eures.europa.eu/news.yml +++ b/configs/eures.europa.eu/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: eures.europa.eu/news directory: topics: - jobs diff --git a/configs/faa.gov/newsroom.yml b/configs/faa.gov/newsroom.yml index d6b82cdc..faf42747 100644 --- a/configs/faa.gov/newsroom.yml +++ b/configs/faa.gov/newsroom.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: faa.gov/newsroom directory: topics: - civic diff --git a/configs/fao.org/newsroom.yml b/configs/fao.org/newsroom.yml index aa800f03..22b58ea1 100644 --- a/configs/fao.org/newsroom.yml +++ b/configs/fao.org/newsroom.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: fao.org/newsroom directory: topics: - environment diff --git a/configs/fia.com/documents.yml b/configs/fia.com/documents.yml index eacf5218..d0d3be30 100644 --- a/configs/fia.com/documents.yml +++ b/configs/fia.com/documents.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: fia.com/documents directory: topics: - sports diff --git a/configs/figma.com/blog.yml b/configs/figma.com/blog.yml index 6cce2bf9..f9501bcc 100644 --- a/configs/figma.com/blog.yml +++ b/configs/figma.com/blog.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: figma.com/blog directory: topics: - tech diff --git a/configs/formula1.com/latest.yml b/configs/formula1.com/latest.yml index 273b1de5..be1a8de1 100644 --- a/configs/formula1.com/latest.yml +++ b/configs/formula1.com/latest.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: formula1.com/latest directory: topics: - sports diff --git a/configs/forumsec.org/publications.yml b/configs/forumsec.org/publications.yml index 45f3c1ef..9abd9402 100644 --- a/configs/forumsec.org/publications.yml +++ b/configs/forumsec.org/publications.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: forumsec.org/publications directory: topics: - civic diff --git a/configs/ftc.gov/press-releases.yml b/configs/ftc.gov/press-releases.yml index c507edf6..1d78e5bb 100644 --- a/configs/ftc.gov/press-releases.yml +++ b/configs/ftc.gov/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ftc.gov/press-releases directory: topics: - security diff --git a/configs/github.com/releases.yml b/configs/github.com/releases.yml index ac74465d..cab0a89a 100644 --- a/configs/github.com/releases.yml +++ b/configs/github.com/releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: github.com/releases directory: topics: - tech diff --git a/configs/go.dev/release-history.yml b/configs/go.dev/release-history.yml index 1ff45996..77c58366 100644 --- a/configs/go.dev/release-history.yml +++ b/configs/go.dev/release-history.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: go.dev/release-history directory: topics: - tech diff --git a/configs/grafana.com/whatsnew.yml b/configs/grafana.com/whatsnew.yml index 8d656ad6..d903cbc6 100644 --- a/configs/grafana.com/whatsnew.yml +++ b/configs/grafana.com/whatsnew.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: grafana.com/whatsnew directory: topics: - tech diff --git a/configs/haaretz.com/israel-news.yml b/configs/haaretz.com/israel-news.yml index 2992e568..5c856dae 100644 --- a/configs/haaretz.com/israel-news.yml +++ b/configs/haaretz.com/israel-news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: haaretz.com/israel-news directory: topics: - news diff --git a/configs/hai.stanford.edu/news.yml b/configs/hai.stanford.edu/news.yml index 44c04e5e..81a28b3b 100644 --- a/configs/hai.stanford.edu/news.yml +++ b/configs/hai.stanford.edu/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: hai.stanford.edu/news directory: topics: - science diff --git a/configs/iaapa.org/news.yml b/configs/iaapa.org/news.yml index be53b07d..d03cb0e3 100644 --- a/configs/iaapa.org/news.yml +++ b/configs/iaapa.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: iaapa.org/news directory: topics: - entertainment diff --git a/configs/icj-cij.org/press-releases.yml b/configs/icj-cij.org/press-releases.yml index a6815c41..45068aaf 100644 --- a/configs/icj-cij.org/press-releases.yml +++ b/configs/icj-cij.org/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: icj-cij.org/press-releases directory: topics: - civic diff --git a/configs/icrc.org/news.yml b/configs/icrc.org/news.yml index c3d38f81..2516bf9d 100644 --- a/configs/icrc.org/news.yml +++ b/configs/icrc.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: icrc.org/news directory: topics: - civic diff --git a/configs/iea.org/news.yml b/configs/iea.org/news.yml index 93ddee92..3e854eca 100644 --- a/configs/iea.org/news.yml +++ b/configs/iea.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: iea.org/news directory: topics: - energy diff --git a/configs/iiss.org/online-analysis.yml b/configs/iiss.org/online-analysis.yml index e9d0bbf9..0e9b8651 100644 --- a/configs/iiss.org/online-analysis.yml +++ b/configs/iiss.org/online-analysis.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: iiss.org/online-analysis directory: topics: - civic diff --git a/configs/ikea.com/newsroom.yml b/configs/ikea.com/newsroom.yml index 33234a83..3a4daa1a 100644 --- a/configs/ikea.com/newsroom.yml +++ b/configs/ikea.com/newsroom.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ikea.com/newsroom directory: topics: - consumer diff --git a/configs/imdb.com/ratings.yml b/configs/imdb.com/ratings.yml index 2ea3c786..32936b7e 100644 --- a/configs/imdb.com/ratings.yml +++ b/configs/imdb.com/ratings.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: imdb.com/ratings directory: topics: - entertainment diff --git a/configs/imo.org/press-briefings.yml b/configs/imo.org/press-briefings.yml index 1ebdc144..0857264a 100644 --- a/configs/imo.org/press-briefings.yml +++ b/configs/imo.org/press-briefings.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: imo.org/press-briefings directory: topics: - civic diff --git a/configs/imperial.ac.uk/news.yml b/configs/imperial.ac.uk/news.yml index 1fe03e75..fbeed688 100644 --- a/configs/imperial.ac.uk/news.yml +++ b/configs/imperial.ac.uk/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: imperial.ac.uk/news directory: topics: - science diff --git a/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml b/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml index 5606ef01..8df62e40 100644 --- a/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml +++ b/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: ingenieur.de/karriere_arbeitsleben_heiko_mell directory: topics: - jobs diff --git a/configs/isro.gov.in/press.yml b/configs/isro.gov.in/press.yml index 12c4432b..ffd5162e 100644 --- a/configs/isro.gov.in/press.yml +++ b/configs/isro.gov.in/press.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: isro.gov.in/press directory: topics: - science diff --git a/configs/issafrica.org/iss-today.yml b/configs/issafrica.org/iss-today.yml index a67c1cc6..ec530484 100644 --- a/configs/issafrica.org/iss-today.yml +++ b/configs/issafrica.org/iss-today.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: issafrica.org/iss-today directory: topics: - civic diff --git a/configs/issafrica.org/press-releases.yml b/configs/issafrica.org/press-releases.yml index 0a617b05..79d21204 100644 --- a/configs/issafrica.org/press-releases.yml +++ b/configs/issafrica.org/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: issafrica.org/press-releases directory: topics: - civic diff --git a/configs/kaist.ac.kr/news.yml b/configs/kaist.ac.kr/news.yml index fc449b19..9805ed93 100644 --- a/configs/kaist.ac.kr/news.yml +++ b/configs/kaist.ac.kr/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: kaist.ac.kr/news directory: topics: - science diff --git a/configs/kinocheck.de/filmstarts.yml b/configs/kinocheck.de/filmstarts.yml index 72c98647..558eecf2 100644 --- a/configs/kinocheck.de/filmstarts.yml +++ b/configs/kinocheck.de/filmstarts.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: kinocheck.de/filmstarts directory: topics: - entertainment diff --git a/configs/kompas.com/nasional.yml b/configs/kompas.com/nasional.yml index 332d8abc..e5655866 100644 --- a/configs/kompas.com/nasional.yml +++ b/configs/kompas.com/nasional.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: kompas.com/nasional directory: topics: - news diff --git a/configs/lego.com/press-releases.yml b/configs/lego.com/press-releases.yml index ba29e71e..f2cdc9b2 100644 --- a/configs/lego.com/press-releases.yml +++ b/configs/lego.com/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: lego.com/press-releases directory: topics: - entertainment diff --git a/configs/machinelearning.apple.com/research.yml b/configs/machinelearning.apple.com/research.yml index 00c199af..785f7f8c 100644 --- a/configs/machinelearning.apple.com/research.yml +++ b/configs/machinelearning.apple.com/research.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: machinelearning.apple.com/research directory: topics: - tech diff --git a/configs/maersk.com/press-releases.yml b/configs/maersk.com/press-releases.yml index f8681fab..5a8f6ebe 100644 --- a/configs/maersk.com/press-releases.yml +++ b/configs/maersk.com/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: maersk.com/press-releases directory: topics: - finance diff --git a/configs/microsoft.com/azure-products.yml b/configs/microsoft.com/azure-products.yml index f86abf66..4cfc4c0e 100644 --- a/configs/microsoft.com/azure-products.yml +++ b/configs/microsoft.com/azure-products.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: microsoft.com/azure-products directory: topics: - tech diff --git a/configs/mistral.ai/news.yml b/configs/mistral.ai/news.yml index 4cac43fa..38a07065 100644 --- a/configs/mistral.ai/news.yml +++ b/configs/mistral.ai/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: mistral.ai/news directory: topics: - tech diff --git a/configs/mozilla.org/security-advisories.yml b/configs/mozilla.org/security-advisories.yml index 7db44b2d..2e0d5f3c 100644 --- a/configs/mozilla.org/security-advisories.yml +++ b/configs/mozilla.org/security-advisories.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: mozilla.org/security-advisories directory: topics: - security diff --git a/configs/msf.org/latest.yml b/configs/msf.org/latest.yml index 2c878dca..f60f1345 100644 --- a/configs/msf.org/latest.yml +++ b/configs/msf.org/latest.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: msf.org/latest directory: topics: - news diff --git a/configs/nbim.no/press-releases.yml b/configs/nbim.no/press-releases.yml index 6d62877f..f286fed7 100644 --- a/configs/nbim.no/press-releases.yml +++ b/configs/nbim.no/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nbim.no/press-releases directory: topics: - finance diff --git a/configs/news.cn/latest.yml b/configs/news.cn/latest.yml index e7b2d56d..9312eb83 100644 --- a/configs/news.cn/latest.yml +++ b/configs/news.cn/latest.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: news.cn/latest directory: topics: - news diff --git a/configs/news24.com/top-stories.yml b/configs/news24.com/top-stories.yml index 7f014f68..e68f8f17 100644 --- a/configs/news24.com/top-stories.yml +++ b/configs/news24.com/top-stories.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: news24.com/top-stories directory: topics: - news diff --git a/configs/newyorker.com/magazine.yml b/configs/newyorker.com/magazine.yml index cd04c8ff..6cc171dd 100644 --- a/configs/newyorker.com/magazine.yml +++ b/configs/newyorker.com/magazine.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: newyorker.com/magazine directory: topics: - news diff --git a/configs/nigerianstat.gov.ng/news.yml b/configs/nigerianstat.gov.ng/news.yml index bb746c62..7c635c88 100644 --- a/configs/nigerianstat.gov.ng/news.yml +++ b/configs/nigerianstat.gov.ng/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nigerianstat.gov.ng/news directory: topics: - civic diff --git a/configs/nikkei.com/asia-technology.yml b/configs/nikkei.com/asia-technology.yml index 704bfd16..20502656 100644 --- a/configs/nikkei.com/asia-technology.yml +++ b/configs/nikkei.com/asia-technology.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nikkei.com/asia-technology directory: topics: - news diff --git a/configs/nokia.com/newsroom.yml b/configs/nokia.com/newsroom.yml index 90ee2d4e..d9a25f17 100644 --- a/configs/nokia.com/newsroom.yml +++ b/configs/nokia.com/newsroom.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nokia.com/newsroom directory: topics: - tech diff --git a/configs/nomanssky.com/news.yml b/configs/nomanssky.com/news.yml index 85b45d1e..b3fc8257 100644 --- a/configs/nomanssky.com/news.yml +++ b/configs/nomanssky.com/news.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: nomanssky.com/news directory: topics: - entertainment diff --git a/configs/notion.com/blog.yml b/configs/notion.com/blog.yml index 1779e7fa..6e4efc34 100644 --- a/configs/notion.com/blog.yml +++ b/configs/notion.com/blog.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: notion.com/blog directory: topics: - tech diff --git a/configs/novonordisk.com/latest-news.yml b/configs/novonordisk.com/latest-news.yml index b3f0bb3a..e9cba5eb 100644 --- a/configs/novonordisk.com/latest-news.yml +++ b/configs/novonordisk.com/latest-news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: novonordisk.com/latest-news directory: topics: - science diff --git a/configs/ntu.edu.sg/news.yml b/configs/ntu.edu.sg/news.yml index 5f14c627..130ad35a 100644 --- a/configs/ntu.edu.sg/news.yml +++ b/configs/ntu.edu.sg/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ntu.edu.sg/news directory: topics: - science diff --git a/configs/nupi.no/news.yml b/configs/nupi.no/news.yml index 63bf334a..223a5327 100644 --- a/configs/nupi.no/news.yml +++ b/configs/nupi.no/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nupi.no/news directory: topics: - civic diff --git a/configs/occrp.org/news.yml b/configs/occrp.org/news.yml index f3262f69..6ec01fa8 100644 --- a/configs/occrp.org/news.yml +++ b/configs/occrp.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: occrp.org/news directory: topics: - news diff --git a/configs/oecd.org/newsroom.yml b/configs/oecd.org/newsroom.yml index d6590a22..43194b04 100644 --- a/configs/oecd.org/newsroom.yml +++ b/configs/oecd.org/newsroom.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: oecd.org/newsroom directory: topics: - finance diff --git a/configs/opec.org/press-releases.yml b/configs/opec.org/press-releases.yml index 81eb6c1f..d89fa2eb 100644 --- a/configs/opec.org/press-releases.yml +++ b/configs/opec.org/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: opec.org/press-releases directory: topics: - energy diff --git a/configs/panapress.com/latest.yml b/configs/panapress.com/latest.yml index 0d7386ff..4124a4f9 100644 --- a/configs/panapress.com/latest.yml +++ b/configs/panapress.com/latest.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: panapress.com/latest directory: topics: - news diff --git a/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml b/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml index cf3a8b07..ca3a888c 100644 --- a/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml +++ b/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: pankow.lebensmittel-kontrollergebnisse.de/search directory: topics: - consumer diff --git a/configs/philstar.com/nation.yml b/configs/philstar.com/nation.yml index e15e6030..87071a14 100644 --- a/configs/philstar.com/nation.yml +++ b/configs/philstar.com/nation.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: philstar.com/nation directory: topics: - news diff --git a/configs/phys.org/weekly.yml b/configs/phys.org/weekly.yml index 3db0727f..a40b0486 100644 --- a/configs/phys.org/weekly.yml +++ b/configs/phys.org/weekly.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: phys.org/weekly directory: topics: - science diff --git a/configs/pib.gov.in/press-releases.yml b/configs/pib.gov.in/press-releases.yml index 8079fcc1..21ae23d4 100644 --- a/configs/pib.gov.in/press-releases.yml +++ b/configs/pib.gov.in/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: pib.gov.in/press-releases directory: topics: - civic diff --git a/configs/rbb24.de/meistgeklickt.yml b/configs/rbb24.de/meistgeklickt.yml index 32f9cf38..b28744a8 100644 --- a/configs/rbb24.de/meistgeklickt.yml +++ b/configs/rbb24.de/meistgeklickt.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: rbb24.de/meistgeklickt directory: topics: - news diff --git a/configs/robinwood.de/aktuelles.yml b/configs/robinwood.de/aktuelles.yml index ea5cde3c..58a03e08 100644 --- a/configs/robinwood.de/aktuelles.yml +++ b/configs/robinwood.de/aktuelles.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: robinwood.de/aktuelles directory: topics: - environment diff --git a/configs/s3.amazonaws.com/popular_movies.yml b/configs/s3.amazonaws.com/popular_movies.yml index 2bdf8f33..e109bf43 100644 --- a/configs/s3.amazonaws.com/popular_movies.yml +++ b/configs/s3.amazonaws.com/popular_movies.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: s3.amazonaws.com/popular_movies directory: topics: - entertainment diff --git a/configs/sacu.int/news.yml b/configs/sacu.int/news.yml index 540db6c8..e1de83dc 100644 --- a/configs/sacu.int/news.yml +++ b/configs/sacu.int/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: sacu.int/news directory: topics: - civic diff --git a/configs/sadc.int/latest-news.yml b/configs/sadc.int/latest-news.yml index 039f2702..2af67d5f 100644 --- a/configs/sadc.int/latest-news.yml +++ b/configs/sadc.int/latest-news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: sadc.int/latest-news directory: topics: - civic diff --git a/configs/sebastianvettel.de/news.yml b/configs/sebastianvettel.de/news.yml index 8d1b3dd8..226563b2 100644 --- a/configs/sebastianvettel.de/news.yml +++ b/configs/sebastianvettel.de/news.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: sebastianvettel.de/news directory: topics: - sports diff --git a/configs/shopify.com/blog.yml b/configs/shopify.com/blog.yml index 161715ef..834e736f 100644 --- a/configs/shopify.com/blog.yml +++ b/configs/shopify.com/blog.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: shopify.com/blog directory: topics: - tech diff --git a/configs/sipri.org/news.yml b/configs/sipri.org/news.yml index d0045aee..6778a907 100644 --- a/configs/sipri.org/news.yml +++ b/configs/sipri.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: sipri.org/news directory: topics: - civic diff --git a/configs/softwareleadweekly.com/issues.yml b/configs/softwareleadweekly.com/issues.yml index a0d5d5e5..d0a8c49a 100644 --- a/configs/softwareleadweekly.com/issues.yml +++ b/configs/softwareleadweekly.com/issues.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: softwareleadweekly.com/issues directory: topics: - tech diff --git a/configs/solarthermalworld.org/news.yml b/configs/solarthermalworld.org/news.yml index 1a7cb20f..c25d2fa0 100644 --- a/configs/solarthermalworld.org/news.yml +++ b/configs/solarthermalworld.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: solarthermalworld.org/news directory: topics: - energy diff --git a/configs/spektrum.de/meistgelesen.yml b/configs/spektrum.de/meistgelesen.yml index 9d2a18c0..db7788f7 100644 --- a/configs/spektrum.de/meistgelesen.yml +++ b/configs/spektrum.de/meistgelesen.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: spektrum.de/meistgelesen directory: topics: - science diff --git a/configs/spiegel.de/impressum_autor.yml b/configs/spiegel.de/impressum_autor.yml index 140a1a10..e6aa1346 100644 --- a/configs/spiegel.de/impressum_autor.yml +++ b/configs/spiegel.de/impressum_autor.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: spiegel.de/impressum_autor directory: topics: - news diff --git a/configs/spotify.com/newsroom.yml b/configs/spotify.com/newsroom.yml index 277ad3a3..b9ef9fbc 100644 --- a/configs/spotify.com/newsroom.yml +++ b/configs/spotify.com/newsroom.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: spotify.com/newsroom directory: topics: - entertainment diff --git a/configs/stackoverflow.com/hot_network_questions.yml b/configs/stackoverflow.com/hot_network_questions.yml index a3519e92..2b031bb2 100644 --- a/configs/stackoverflow.com/hot_network_questions.yml +++ b/configs/stackoverflow.com/hot_network_questions.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: stackoverflow.com/hot_network_questions directory: topics: - tech diff --git a/configs/steuerzahler.de/news.yml b/configs/steuerzahler.de/news.yml index 738add41..07443d87 100644 --- a/configs/steuerzahler.de/news.yml +++ b/configs/steuerzahler.de/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: steuerzahler.de/news directory: topics: - civic diff --git a/configs/stripes.com/index.yml b/configs/stripes.com/index.yml index 7fd5ed37..fae6cef8 100644 --- a/configs/stripes.com/index.yml +++ b/configs/stripes.com/index.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: stripes.com/index directory: topics: - news diff --git a/configs/support.apple.com/en_gb_ht201222.yml b/configs/support.apple.com/en_gb_ht201222.yml index fe2cc334..f97e60f6 100644 --- a/configs/support.apple.com/en_gb_ht201222.yml +++ b/configs/support.apple.com/en_gb_ht201222.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: support.apple.com/en_gb_ht201222 directory: topics: - tech diff --git a/configs/support.apple.com/exchange_repair.yml b/configs/support.apple.com/exchange_repair.yml index 8fb71cd7..5321603f 100644 --- a/configs/support.apple.com/exchange_repair.yml +++ b/configs/support.apple.com/exchange_repair.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: support.apple.com/exchange_repair directory: topics: - tech diff --git a/configs/tap.info.tn/politics.yml b/configs/tap.info.tn/politics.yml index d7890c0a..c460912c 100644 --- a/configs/tap.info.tn/politics.yml +++ b/configs/tap.info.tn/politics.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tap.info.tn/politics directory: topics: - news diff --git a/configs/telenor.com/newsroom.yml b/configs/telenor.com/newsroom.yml index 23a120eb..030b8ea3 100644 --- a/configs/telenor.com/newsroom.yml +++ b/configs/telenor.com/newsroom.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: telenor.com/newsroom directory: topics: - tech diff --git a/configs/teneriffa-news.com/news.yml b/configs/teneriffa-news.com/news.yml index 82c74304..0ff7ec08 100644 --- a/configs/teneriffa-news.com/news.yml +++ b/configs/teneriffa-news.com/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: teneriffa-news.com/news directory: topics: - news diff --git a/configs/tennet.eu/news.yml b/configs/tennet.eu/news.yml index 55926b1b..99df43cc 100644 --- a/configs/tennet.eu/news.yml +++ b/configs/tennet.eu/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tennet.eu/news directory: topics: - energy diff --git a/configs/test.de/archiv.yml b/configs/test.de/archiv.yml index c942f7f2..e95ad41f 100644 --- a/configs/test.de/archiv.yml +++ b/configs/test.de/archiv.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: test.de/archiv directory: topics: - consumer diff --git a/configs/theeastafrican.co.ke/news.yml b/configs/theeastafrican.co.ke/news.yml index d4957bb5..db9e5187 100644 --- a/configs/theeastafrican.co.ke/news.yml +++ b/configs/theeastafrican.co.ke/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: theeastafrican.co.ke/news directory: topics: - news diff --git a/configs/theguardian.com/international_mostpopular.yml b/configs/theguardian.com/international_mostpopular.yml index de229e2c..333b3ea1 100644 --- a/configs/theguardian.com/international_mostpopular.yml +++ b/configs/theguardian.com/international_mostpopular.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: theguardian.com/international_mostpopular directory: topics: - news diff --git a/configs/thenationalnews.com/news.yml b/configs/thenationalnews.com/news.yml index a614935f..911d873a 100644 --- a/configs/thenationalnews.com/news.yml +++ b/configs/thenationalnews.com/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: thenationalnews.com/news directory: topics: - news diff --git a/configs/thoughtworks.com/insights.yml b/configs/thoughtworks.com/insights.yml index a596a157..85ac3231 100644 --- a/configs/thoughtworks.com/insights.yml +++ b/configs/thoughtworks.com/insights.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: thoughtworks.com/insights directory: topics: - tech diff --git a/configs/tno.nl/news.yml b/configs/tno.nl/news.yml index 8d864bd3..232b56ef 100644 --- a/configs/tno.nl/news.yml +++ b/configs/tno.nl/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tno.nl/news directory: topics: - research diff --git a/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml b/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml index 0458f164..0d383e87 100644 --- a/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml +++ b/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten directory: topics: - travel diff --git a/configs/tralac.org/news.yml b/configs/tralac.org/news.yml index f33e32f4..0a1dbc09 100644 --- a/configs/tralac.org/news.yml +++ b/configs/tralac.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tralac.org/news directory: topics: - civic diff --git a/configs/tudelft.nl/news.yml b/configs/tudelft.nl/news.yml index 010d0364..d0786ce3 100644 --- a/configs/tudelft.nl/news.yml +++ b/configs/tudelft.nl/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tudelft.nl/news directory: topics: - science diff --git a/configs/tue.nl/news.yml b/configs/tue.nl/news.yml index 7d068801..d5cecc03 100644 --- a/configs/tue.nl/news.yml +++ b/configs/tue.nl/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tue.nl/news directory: topics: - science diff --git a/configs/vattenfall.com/newsroom.yml b/configs/vattenfall.com/newsroom.yml index 3e04eb49..9bcfbf55 100644 --- a/configs/vattenfall.com/newsroom.yml +++ b/configs/vattenfall.com/newsroom.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: vattenfall.com/newsroom directory: topics: - energy diff --git a/configs/visitfinland.com/press-releases.yml b/configs/visitfinland.com/press-releases.yml index 6db70542..14778890 100644 --- a/configs/visitfinland.com/press-releases.yml +++ b/configs/visitfinland.com/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: visitfinland.com/press-releases directory: topics: - travel diff --git a/configs/webentwickler-jobs.de/in.yml b/configs/webentwickler-jobs.de/in.yml index 922a385f..cc35d7f4 100644 --- a/configs/webentwickler-jobs.de/in.yml +++ b/configs/webentwickler-jobs.de/in.yml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json --- +registry: + id: webentwickler-jobs.de/in directory: topics: - jobs diff --git a/configs/who.int/news.yml b/configs/who.int/news.yml index 08520de1..c100d520 100644 --- a/configs/who.int/news.yml +++ b/configs/who.int/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: who.int/news directory: topics: - science diff --git a/configs/worldbank.org/news.yml b/configs/worldbank.org/news.yml index 80cfa45f..b183bd71 100644 --- a/configs/worldbank.org/news.yml +++ b/configs/worldbank.org/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: worldbank.org/news directory: topics: - finance diff --git a/configs/www.consilium.europa.eu/press-releases.yml b/configs/www.consilium.europa.eu/press-releases.yml index a7e9aeea..b458bd2a 100644 --- a/configs/www.consilium.europa.eu/press-releases.yml +++ b/configs/www.consilium.europa.eu/press-releases.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: consilium.europa.eu/press-releases directory: topics: - civic diff --git a/configs/www.ecdc.europa.eu/news-events.yml b/configs/www.ecdc.europa.eu/news-events.yml index 54e4f16e..0835868f 100644 --- a/configs/www.ecdc.europa.eu/news-events.yml +++ b/configs/www.ecdc.europa.eu/news-events.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ecdc.europa.eu/news-events directory: topics: - science diff --git a/configs/www.enisa.europa.eu/news.yml b/configs/www.enisa.europa.eu/news.yml index 434e01f3..228824c9 100644 --- a/configs/www.enisa.europa.eu/news.yml +++ b/configs/www.enisa.europa.eu/news.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: enisa.europa.eu/news directory: topics: - security diff --git a/configs/www.europarl.europa.eu/press-room.yml b/configs/www.europarl.europa.eu/press-room.yml index 063297ba..5da553bb 100644 --- a/configs/www.europarl.europa.eu/press-room.yml +++ b/configs/www.europarl.europa.eu/press-room.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: europarl.europa.eu/press-room directory: topics: - civic diff --git a/configs/www.gob.mx/profeco-prensa.yml b/configs/www.gob.mx/profeco-prensa.yml index f1e72062..79fec148 100644 --- a/configs/www.gob.mx/profeco-prensa.yml +++ b/configs/www.gob.mx/profeco-prensa.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: gob.mx/profeco-prensa directory: topics: - consumer diff --git a/test/registry.spec.rb b/test/registry.spec.rb deleted file mode 100644 index b7063fbc..00000000 --- a/test/registry.spec.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -require 'fileutils' -require 'open3' -require 'tmpdir' - -RSpec.describe 'registry bundle' do - let(:repo_root) { File.expand_path('..', __dir__) } - - it 'builds and loads with integrity-only trust', :aggregate_failures do - Dir.mktmpdir do |tmpdir| - output = File.join(tmpdir, 'registry-bundle.tar.gz') - env = { 'BUNDLE_GEMFILE' => File.join(repo_root, 'tool/Gemfile') } - stdout, stderr, status = Open3.capture3(env, File.join(repo_root, 'tool/registry-build'), '--output', output, - chdir: repo_root) - expect(status).to eq(0), [stdout, stderr].join - - extract_dir = File.join(tmpdir, 'extracted') - FileUtils.mkdir_p(extract_dir) - File.open(output, 'rb') { |io| Html2rss::Registry::Archive.extract!(io, into: extract_dir) } - - bundle = Html2rss::Registry::Bundle.load(extract_dir, trust: :integrity_only) - - expect(bundle.manifest.registry_id).to eq('official') - expect(bundle.catalog_entries).not_to be_empty - expect(bundle.catalog_entries.map(&:id)).to eq(bundle.catalog_entries.map(&:id).sort) - - anthropic = bundle.catalog_entries.find { |entry| entry.id == 'anthropic.com/news' } - expect(anthropic).to have_attributes( - id: 'anthropic.com/news', - path: '/anthropic.com/news.rss' - ) - expect(anthropic.directory[:title]).to eq('Anthropic — News') - expect(anthropic.channel[:title]).to eq('Anthropic — News') - expect(anthropic.parameters).to eq(schema: {}, defaults: {}) - - cnet = bundle.catalog_entries.find { |entry| entry.id == 'cnet.com/section_sub' } - expect(cnet.parameters[:defaults]).to eq('section' => 'tech') - expect(cnet.parameters[:schema]['section']).to eq('type' => 'string') - end - end -end diff --git a/test/registry_build_spec.rb b/test/registry_build_spec.rb new file mode 100644 index 00000000..9d62da38 --- /dev/null +++ b/test/registry_build_spec.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'open3' +require 'tmpdir' + +RSpec.describe 'registry bundle' do + let(:repo_root) { File.expand_path('..', __dir__) } + + def build_bundle(output_path) + env = { 'BUNDLE_GEMFILE' => File.join(repo_root, 'tool/Gemfile') } + stdout, stderr, status = Open3.capture3( + env, + File.join(repo_root, 'tool/registry-build'), + '--output', + output_path, + chdir: repo_root + ) + expect(status).to eq(0), [stdout, stderr].join + end + + def load_bundle(output_path) + extract_dir = File.join(File.dirname(output_path), 'extracted') + FileUtils.mkdir_p(extract_dir) + File.open(output_path, 'rb') { |io| Html2rss::Registry::Archive.extract!(io, into: extract_dir) } + Html2rss::Registry::Bundle.load(extract_dir, trust: :integrity_only) + end + + it 'builds a tarball' do + Dir.mktmpdir do |tmpdir| + output = File.join(tmpdir, 'registry-bundle.tar.gz') + build_bundle(output) + expect(File).to be_file(output) + end + end + + it 'loads manifest and sorted catalog entries', :aggregate_failures do + Dir.mktmpdir do |tmpdir| + output = File.join(tmpdir, 'registry-bundle.tar.gz') + build_bundle(output) + bundle = load_bundle(output) + + expect(bundle.manifest.registry_id).to eq('official') + expect(bundle.catalog_entries).not_to be_empty + expect(bundle.catalog_entries.map(&:id)).to eq(bundle.catalog_entries.map(&:id).sort) + end + end + + it 'maps anthropic.com/news catalog metadata', :aggregate_failures do + Dir.mktmpdir do |tmpdir| + bundle = load_bundle(build_and_return(File.join(tmpdir, 'registry-bundle.tar.gz'))) + + anthropic = bundle.catalog_entries.find { |entry| entry.id == 'anthropic.com/news' } + expect(anthropic).to have_attributes(id: 'anthropic.com/news', path: '/anthropic.com/news.rss') + expect(anthropic.directory[:title]).to eq('Anthropic — News') + expect(anthropic.channel[:title]).to eq('Anthropic — News') + expect(anthropic.parameters).to eq(schema: {}, defaults: {}) + end + end + + it 'extracts cnet parameter schema and defaults', :aggregate_failures do + Dir.mktmpdir do |tmpdir| + bundle = load_bundle(build_and_return(File.join(tmpdir, 'registry-bundle.tar.gz'))) + + cnet = bundle.catalog_entries.find { |entry| entry.id == 'cnet.com/section_sub' } + expect(cnet.parameters[:defaults]).to eq('section' => 'tech') + expect(cnet.parameters[:schema]['section']).to eq('type' => 'string') + end + end + + def build_and_return(output_path) + build_bundle(output_path) + output_path + end +end diff --git a/tool/Gemfile b/tool/Gemfile index 3b0a4af8..11133d50 100644 --- a/tool/Gemfile +++ b/tool/Gemfile @@ -2,7 +2,7 @@ source 'https://rubygems.org' -gem 'html2rss', path: '../../html2rss' +gem 'html2rss', git: 'https://github.com/html2rss/html2rss.git', branch: 'feat/registry-v1' group :development do gem 'nokogiri' diff --git a/tool/Gemfile.lock b/tool/Gemfile.lock index 5cf2d4ed..949bbaa5 100644 --- a/tool/Gemfile.lock +++ b/tool/Gemfile.lock @@ -1,5 +1,7 @@ -PATH - remote: ../../html2rss +GIT + remote: https://github.com/html2rss/html2rss.git + revision: e32014ac089679cac7e91fd55f3bd85a0489771e + branch: feat/registry-v1 specs: html2rss (0.27.1) addressable (~> 2.7) diff --git a/tool/registry-build b/tool/registry-build index 34ab4ec0..b25b2138 100755 --- a/tool/registry-build +++ b/tool/registry-build @@ -4,11 +4,8 @@ require 'bundler/setup' require 'fileutils' require 'optparse' -require 'openssl' -require 'rubygems/package' -require 'stringio' require 'tmpdir' -require 'zlib' +require 'yaml' require 'html2rss' REPO_ROOT = File.expand_path('..', __dir__) @@ -16,14 +13,33 @@ CONFIGS_DIR = File.join(REPO_ROOT, 'configs') DIST_DIR = File.join(REPO_ROOT, 'dist') ## +# @param staging [String] +# @param relative_path [String] +# @return [void] +def validate_config!(staging, relative_path) + absolute_path = File.join(staging, relative_path) + result = Html2rss::Config.validate_yaml(absolute_path) + return if result.success? + + messages = result.errors.to_h.flat_map do |key, values| + Array(values).map { |message| "#{key}: #{message}" } + end + abort "Validation failed for #{relative_path}:\n #{messages.join("\n ")}" +end + +## +# @param staging [String] # @param relative_paths [Array] # @return [void] -def reject_duplicate_ids!(relative_paths) - ids = relative_paths.map { |path| Html2rss::Registry::CatalogBuilder.entry_id(path) } +def reject_duplicate_ids!(staging, relative_paths) + ids = relative_paths.map do |relative_path| + config = YAML.safe_load_file(File.join(staging, relative_path), symbolize_names: true) + Html2rss::Registry::CatalogBuilder.entry_id(config, relative_path) + end duplicates = ids.group_by(&:itself).select { |_, group| group.size > 1 }.keys return if duplicates.empty? - abort "Duplicate config ids: #{duplicates.sort.join(', ')}" + abort "Duplicate registry.id values: #{duplicates.sort.join(', ')}" end ## @@ -42,49 +58,9 @@ def enforce_build_limits!(staging, relative_paths) size end - return unless total_bytes > Html2rss::Registry::Archive::MAX_TARBALL_BYTES - - raise "Bundle exceeds max bytes (#{Html2rss::Registry::Archive::MAX_TARBALL_BYTES})" -end + return unless total_bytes > Html2rss::Registry::Archive::MAX_EXTRACT_BYTES -## -# @param staging [String] -# @param manifest [Html2rss::Registry::Manifest] -# @return [void] -def sign_manifest!(staging, manifest) - private_key = OpenSSL::PKey.read(ENV.fetch('REGISTRY_SIGNING_KEY')) - signature = private_key.sign(nil, manifest.canonical_bytes) - File.write( - File.join(staging, Html2rss::Registry::Manifest::SIGNATURE_FILE), - [signature].pack('m0') - ) -end - -## -# @param tar [Gem::Package::TarWriter] -# @param source_dir [String] -# @param root_dir [String] -# @return [void] -def add_tree_to_tar(tar, source_dir, root_dir) - Dir.glob(File.join(source_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |path| - next if File.directory?(path) - - relative = path.delete_prefix("#{root_dir}/") - tar.add_file(relative, 0o644) { |io| io.write(File.binread(path)) } - end -end - -## -# @param source_dir [String] -# @param output_path [String] -# @return [void] -def write_tar_gz(source_dir, output_path) - tar_data = StringIO.new - Gem::Package::TarWriter.new(tar_data) do |tar| - add_tree_to_tar(tar, source_dir, source_dir) - end - - Zlib::GzipWriter.open(output_path) { |gz| gz.write(tar_data.string) } + raise "Bundle exceeds max bytes (#{Html2rss::Registry::Archive::MAX_EXTRACT_BYTES})" end options = { @@ -112,10 +88,13 @@ abort "Missing configs directory: #{CONFIGS_DIR}" unless File.directory?(CONFIGS Dir.mktmpdir('registry-build') do |staging| FileUtils.cp_r(File.join(CONFIGS_DIR, '.'), File.join(staging, 'configs')) - relative_paths = Html2rss::Registry::CatalogBuilder.config_paths(staging).sort + relative_paths = Dir.glob(File.join(staging, 'configs', '**', '*.yml')).map do |absolute_path| + absolute_path.delete_prefix("#{staging}/") + end.sort abort 'No config files found under configs/' if relative_paths.empty? - reject_duplicate_ids!(relative_paths) + relative_paths.each { |relative_path| validate_config!(staging, relative_path) } + reject_duplicate_ids!(staging, relative_paths) enforce_build_limits!(staging, relative_paths) file_index = Html2rss::Registry::Manifest.file_index(staging, relative_paths) @@ -127,10 +106,18 @@ Dir.mktmpdir('registry-build') do |staging| ) File.write(File.join(staging, Html2rss::Registry::Manifest::MANIFEST_FILE), manifest.to_json) - sign_manifest!(staging, manifest) if options[:sign] + if options[:sign] + Html2rss::Registry::TestSupport.sign!( + manifest, + key_pem: ENV.fetch('REGISTRY_SIGNING_KEY'), + bundle_dir: staging + ) + end FileUtils.mkdir_p(File.dirname(options[:output])) - write_tar_gz(staging, options[:output]) + File.open(options[:output], 'wb') do |io| + Html2rss::Registry::Archive.pack_dir(staging, io:) + end end puts "Built #{options[:output]}" From 125f580166b7540208190c913ebe3a1d50cb8208 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 16:22:41 +0200 Subject: [PATCH 05/11] chore(deps): pin html2rss to registry hardening commit Align tool/Gemfile.lock with feat/registry-v1 b32ec095 so CI registry-build uses manifest-only catalog and BundleRelativePath validation. --- tool/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/Gemfile.lock b/tool/Gemfile.lock index 949bbaa5..aeb746ea 100644 --- a/tool/Gemfile.lock +++ b/tool/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss.git - revision: e32014ac089679cac7e91fd55f3bd85a0489771e + revision: b32ec095bd366075297f9b7f59cf55db6b665d2a branch: feat/registry-v1 specs: html2rss (0.27.1) From c37d9187dedbff8f9524a2a36bc10697c7353030 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:16:03 +0200 Subject: [PATCH 06/11] fix(ci): repair changed-config fetch runner and html2rss git pin Set BUNDLE_GEMFILE around system calls in rspec_changed_configs and pin html2rss from feat/registry-v1 for CI until 0.28.0 is on RubyGems. --- bin/rspec_changed_configs | 12 ++++++++++-- tool/Gemfile | 3 ++- tool/Gemfile.lock | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/bin/rspec_changed_configs b/bin/rspec_changed_configs index a1b50c28..537b642b 100755 --- a/bin/rspec_changed_configs +++ b/bin/rspec_changed_configs @@ -28,8 +28,16 @@ failed = false test_files.each do |file| config_name = file.delete_prefix('configs/') puts "Testing changed config: #{config_name}" - failed ||= !system('bundle', 'exec', 'rspec', '--example', config_name, '--tag', 'fetch', - 'test/configs_dynamic_spec.rb', env: { 'BUNDLE_GEMFILE' => 'tool/Gemfile' }) + bundle_gemfile = ENV.fetch('BUNDLE_GEMFILE', nil) + ENV['BUNDLE_GEMFILE'] = 'tool/Gemfile' + ok = system('bundle', 'exec', 'rspec', '--example', config_name, '--tag', 'fetch', + 'test/configs_dynamic_spec.rb') + if bundle_gemfile.nil? + ENV.delete('BUNDLE_GEMFILE') + else + ENV['BUNDLE_GEMFILE'] = bundle_gemfile + end + failed ||= !ok end exit 1 if failed diff --git a/tool/Gemfile b/tool/Gemfile index 11133d50..8cac9458 100644 --- a/tool/Gemfile +++ b/tool/Gemfile @@ -2,7 +2,8 @@ source 'https://rubygems.org' -gem 'html2rss', git: 'https://github.com/html2rss/html2rss.git', branch: 'feat/registry-v1' +# CI uses git until 0.28.0 is on RubyGems; local monorepo: BUNDLE_LOCAL__HTML2RSS=/path/to/html2rss +gem 'html2rss', github: 'html2rss/html2rss', branch: 'feat/registry-v1' group :development do gem 'nokogiri' diff --git a/tool/Gemfile.lock b/tool/Gemfile.lock index aeb746ea..5c0ad8d2 100644 --- a/tool/Gemfile.lock +++ b/tool/Gemfile.lock @@ -3,7 +3,7 @@ GIT revision: b32ec095bd366075297f9b7f59cf55db6b665d2a branch: feat/registry-v1 specs: - html2rss (0.27.1) + html2rss (0.28.0) addressable (~> 2.7) brotli dry-validation @@ -227,7 +227,7 @@ CHECKSUMS faraday-gzip (3.1.0) sha256=320783690be169f9b7ddde11598b77156951343753f66a9ab98b1f6694433ff8 faraday-net_http (3.4.4) sha256=0e78af151747ed1b00f33e25973b4bc220d7f16c00c39676817c8b12331eb588 hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d - html2rss (0.27.1) + html2rss (0.28.0) json (2.21.2) sha256=1f1d3b7cf2b3ba1a69beca0bb6db13d5438b80bff3cd54cdaaa620b9b07c1c6a json_schemer (2.5.0) sha256=2f01fb4cce721a4e08dd068fc2030cffd0702a7f333f1ea2be6e8991f00ae396 kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa From 7270ba291cad32807e50133aa6ebc29ac5ae9d4c Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:26:18 +0200 Subject: [PATCH 07/11] docs: clarify local registry-build is unsigned Align AGENTS, catalog reference, and README with release CI signing via --sign so local make registry-build is not described as signed output. --- .agents/skills/html2rss-config/reference/catalog.md | 2 +- AGENTS.md | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.agents/skills/html2rss-config/reference/catalog.md b/.agents/skills/html2rss-config/reference/catalog.md index 8b67e9a0..d51044e8 100644 --- a/.agents/skills/html2rss-config/reference/catalog.md +++ b/.agents/skills/html2rss-config/reference/catalog.md @@ -70,7 +70,7 @@ make validate make registry-build ``` -`make registry-build` emits a signed `registry.v1` bundle; `Html2rss::Registry::CatalogBuilder.build_entry` raises `MissingDirectoryTitle` when `directory.title` is absent. +`make registry-build` emits a `registry.v1` bundle (release CI signs with `--sign`); `Html2rss::Registry::CatalogBuilder.build_entry` raises `MissingDirectoryTitle` when `directory.title` is absent. ## Downstream pipeline diff --git a/AGENTS.md b/AGENTS.md index a0d2ce5f..3a3e6c32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,7 +49,7 @@ Local development expects **Ruby 4.0** (`mise exec -- …` or `.ruby-version` wh ## Feed Directory catalog -- **Bundle build:** `make registry-build` (`tool/registry-build`) walks YAML, validates `registry.id`, and emits a signed `registry.v1` bundle (`manifest.json`, `manifest.sig`, `configs/`). +- **Bundle build:** `make registry-build` (`tool/registry-build`) walks YAML, validates `registry.id`, and emits a `registry.v1` bundle (`manifest.json`, optional `manifest.sig`, `configs/`). Local builds are unsigned; release CI signs with `--sign`. - **Catalog serialization owner:** `Html2rss::Registry::CatalogBuilder` in the core `html2rss` gem — do not duplicate YAML walking in `html2rss-web` or the docs site. - **Wire shape:** domain entries from `CatalogBuilder`; `html2rss-web` adds `source: registry` and `registry: ` at the HTTP layer. - **Agent reference:** [.agents/skills/html2rss-config/reference/catalog.md](.agents/skills/html2rss-config/reference/catalog.md). diff --git a/README.md b/README.md index 84414030..a9341cc6 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Curated feed YAML for the [html2rss](https://html2rss.github.io) Feed Directory. ```bash make ready # lint, validate all configs, run tests -make registry-build # build unsigned bundle to dist/ +make registry-build # build local registry.v1 bundle to dist/ (unsigned; release CI signs with --sign) ``` Single-file validation uses the core CLI from the sibling `html2rss` checkout: From d896de05e8166b44f756e5df14b8b69f7b3f5545 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 23 Aug 2026 12:03:45 +0200 Subject: [PATCH 08/11] chore(deps): update html2rss gem ref to latest feat/registry-v1 --- tool/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/Gemfile.lock b/tool/Gemfile.lock index 5c0ad8d2..8f8ce5ab 100644 --- a/tool/Gemfile.lock +++ b/tool/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss.git - revision: b32ec095bd366075297f9b7f59cf55db6b665d2a + revision: 5d767b3973449e44682b0e2c127981b26ed5036a branch: feat/registry-v1 specs: html2rss (0.28.0) From 9a61fb047fb4ab16d545cbe023cdc19cd6a4ed59 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 10:33:39 +0200 Subject: [PATCH 09/11] refactor(registry-build): use Html2rss::Registry::Signer --- tool/Gemfile.lock | 2 +- tool/registry-build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/Gemfile.lock b/tool/Gemfile.lock index 8f8ce5ab..702fa79a 100644 --- a/tool/Gemfile.lock +++ b/tool/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss.git - revision: 5d767b3973449e44682b0e2c127981b26ed5036a + revision: 879bcdf1cd220825089dccc648b9b03ecc8d1d60 branch: feat/registry-v1 specs: html2rss (0.28.0) diff --git a/tool/registry-build b/tool/registry-build index b25b2138..86a8c4bc 100755 --- a/tool/registry-build +++ b/tool/registry-build @@ -107,7 +107,7 @@ Dir.mktmpdir('registry-build') do |staging| File.write(File.join(staging, Html2rss::Registry::Manifest::MANIFEST_FILE), manifest.to_json) if options[:sign] - Html2rss::Registry::TestSupport.sign!( + Html2rss::Registry::Signer.sign!( manifest, key_pem: ENV.fetch('REGISTRY_SIGNING_KEY'), bundle_dir: staging From 66e429a131644cadefd01a0388028935e799b72f Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 15:05:41 +0200 Subject: [PATCH 10/11] docs(registry): update release artifact consumption details --- AGENTS.md | 2 +- README.md | 3 ++- tool/Gemfile.lock | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3a3e6c32..e428fca4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ Publisher flow (`.github/workflows/release.yml`): 2. CI runs `make ready`, then `make registry-build -- --sign` with `REGISTRY_SIGNING_KEY` from environment secrets. 3. The workflow verifies the tarball with `REGISTRY_PUBLIC_KEY_PEM` from the **`registry-release`** environment (operator-supplied public key PEM; must match `html2rss-web/config/registries.yml`), not the private signing key. 4. `softprops/action-gh-release@v2` uploads `dist/registry-bundle.tar.gz` as a **draft** GitHub Release with generated notes. -5. A maintainer reviews the draft release and clicks **Publish release** — instances only pick up published releases via `html2rss-web` `Registry::Sync` (default channel `html2rss-official`). +5. A maintainer reviews the draft release and clicks **Publish release** — web container builds bake the published release artifact, and live instances pick up updates via `html2rss-web` `Registry::Sync` (default channel `html2rss-official`). Local bundle build (unsigned): `make registry-build`. Signing locally requires `REGISTRY_SIGNING_KEY` in the environment. diff --git a/README.md b/README.md index a9341cc6..0773d7f1 100644 --- a/README.md +++ b/README.md @@ -34,4 +34,5 @@ tool/ # validate, registry-build test/ # dynamic config specs + registry bundle spec ``` -Release tags trigger CI to sign and publish `registry-bundle.tar.gz`. Running `html2rss-web` instances sync that artifact (or mount a local bundle via `path:` in `config/registries.yml`). +Release tags trigger CI to sign and publish `registry-bundle.tar.gz`. `html2rss-web` Docker images bake this artifact directly at build time, and live instances sync updates via network sync or mount local bundles via `path:`. + diff --git a/tool/Gemfile.lock b/tool/Gemfile.lock index 702fa79a..fe3eb93b 100644 --- a/tool/Gemfile.lock +++ b/tool/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss.git - revision: 879bcdf1cd220825089dccc648b9b03ecc8d1d60 + revision: e1fbff732b143274a7a41b5fd150c32ebffc38bb branch: feat/registry-v1 specs: html2rss (0.28.0) From d13cf25ce58c4dd8ca596f7b2290d98a7da6312f Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 18:26:15 +0200 Subject: [PATCH 11/11] chore(bundle): update html2rss git revision in tool/Gemfile.lock --- tool/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/Gemfile.lock b/tool/Gemfile.lock index fe3eb93b..42fcf81e 100644 --- a/tool/Gemfile.lock +++ b/tool/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss.git - revision: e1fbff732b143274a7a41b5fd150c32ebffc38bb + revision: 5044272b9dcf404a1ef550e0496508cd6ce5687e branch: feat/registry-v1 specs: html2rss (0.28.0)