From 0bb4b605375bab0757cc2ed2fb012e023f543b64 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Tue, 25 Aug 2026 09:19:26 +0200 Subject: [PATCH 01/37] Add the adaptors data model, Postgres store, source strategies, and refresh scheduler - Postgres-backed data model and store for adaptors/versions - Local-directory and npm source strategies - Supervision tree and periodic refresh scheduler --- config/dev.exs | 12 + config/test.exs | 14 + lib/lightning/adaptors.ex | 100 ++ lib/lightning/adaptors/channel_broadcaster.ex | 91 ++ lib/lightning/adaptors/config.ex | 102 ++ lib/lightning/adaptors/icon_cache.ex | 96 ++ lib/lightning/adaptors/invalidator.ex | 45 + lib/lightning/adaptors/local.ex | 229 +++++ lib/lightning/adaptors/node_monitor.ex | 48 + lib/lightning/adaptors/npm.ex | 108 ++ lib/lightning/adaptors/npm/github.ex | 302 ++++++ lib/lightning/adaptors/npm/registry.ex | 195 ++++ lib/lightning/adaptors/npm/schema.ex | 83 ++ lib/lightning/adaptors/repo.ex | 339 +++++++ lib/lightning/adaptors/repo_adaptor.ex | 163 +++ .../adaptors/repo_adaptor_version.ex | 73 ++ lib/lightning/adaptors/scheduler.ex | 597 +++++++++++ lib/lightning/adaptors/store.ex | 347 +++++++ lib/lightning/adaptors/strategy.ex | 148 +++ lib/lightning/adaptors/supervisor.ex | 242 +++++ lib/lightning/application.ex | 1 + mix.exs | 5 +- mix.lock | 1 + .../20260514150000_create_adaptors.exs | 52 + .../adaptors/channel_broadcaster_test.exs | 238 +++++ test/lightning/adaptors/config_test.exs | 114 +++ .../adaptors/end_to_end_broadcast_test.exs | 48 + .../adaptors/highlander_integration_test.exs | 107 ++ test/lightning/adaptors/icon_cache_test.exs | 156 +++ test/lightning/adaptors/invalidator_test.exs | 122 +++ test/lightning/adaptors/local_test.exs | 360 +++++++ test/lightning/adaptors/node_monitor_test.exs | 160 +++ test/lightning/adaptors/npm/github_test.exs | 390 +++++++ test/lightning/adaptors/npm/registry_test.exs | 199 ++++ test/lightning/adaptors/npm/schema_test.exs | 86 ++ test/lightning/adaptors/npm_test.exs | 270 +++++ test/lightning/adaptors/repo_adaptor_test.exs | 315 ++++++ .../adaptors/repo_adaptor_version_test.exs | 172 ++++ test/lightning/adaptors/repo_test.exs | 485 +++++++++ test/lightning/adaptors/scheduler_test.exs | 959 ++++++++++++++++++ test/lightning/adaptors/store_test.exs | 552 ++++++++++ .../adaptors/supervisor_integration_test.exs | 191 ++++ test/lightning/adaptors/supervisor_test.exs | 178 ++++ test/lightning/adaptors_test.exs | 279 +++++ test/support/factories.ex | 10 + test/test_helper.exs | 28 + 46 files changed, 8811 insertions(+), 1 deletion(-) create mode 100644 lib/lightning/adaptors.ex create mode 100644 lib/lightning/adaptors/channel_broadcaster.ex create mode 100644 lib/lightning/adaptors/config.ex create mode 100644 lib/lightning/adaptors/icon_cache.ex create mode 100644 lib/lightning/adaptors/invalidator.ex create mode 100644 lib/lightning/adaptors/local.ex create mode 100644 lib/lightning/adaptors/node_monitor.ex create mode 100644 lib/lightning/adaptors/npm.ex create mode 100644 lib/lightning/adaptors/npm/github.ex create mode 100644 lib/lightning/adaptors/npm/registry.ex create mode 100644 lib/lightning/adaptors/npm/schema.ex create mode 100644 lib/lightning/adaptors/repo.ex create mode 100644 lib/lightning/adaptors/repo_adaptor.ex create mode 100644 lib/lightning/adaptors/repo_adaptor_version.ex create mode 100644 lib/lightning/adaptors/scheduler.ex create mode 100644 lib/lightning/adaptors/store.ex create mode 100644 lib/lightning/adaptors/strategy.ex create mode 100644 lib/lightning/adaptors/supervisor.ex create mode 100644 priv/repo/migrations/20260514150000_create_adaptors.exs create mode 100644 test/lightning/adaptors/channel_broadcaster_test.exs create mode 100644 test/lightning/adaptors/config_test.exs create mode 100644 test/lightning/adaptors/end_to_end_broadcast_test.exs create mode 100644 test/lightning/adaptors/highlander_integration_test.exs create mode 100644 test/lightning/adaptors/icon_cache_test.exs create mode 100644 test/lightning/adaptors/invalidator_test.exs create mode 100644 test/lightning/adaptors/local_test.exs create mode 100644 test/lightning/adaptors/node_monitor_test.exs create mode 100644 test/lightning/adaptors/npm/github_test.exs create mode 100644 test/lightning/adaptors/npm/registry_test.exs create mode 100644 test/lightning/adaptors/npm/schema_test.exs create mode 100644 test/lightning/adaptors/npm_test.exs create mode 100644 test/lightning/adaptors/repo_adaptor_test.exs create mode 100644 test/lightning/adaptors/repo_adaptor_version_test.exs create mode 100644 test/lightning/adaptors/repo_test.exs create mode 100644 test/lightning/adaptors/scheduler_test.exs create mode 100644 test/lightning/adaptors/store_test.exs create mode 100644 test/lightning/adaptors/supervisor_integration_test.exs create mode 100644 test/lightning/adaptors/supervisor_test.exs create mode 100644 test/lightning/adaptors_test.exs diff --git a/config/dev.exs b/config/dev.exs index 4d55304f67c..3fd698f53dd 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -147,6 +147,18 @@ config :philter, allowed_hosts: ["localhost"] config :lightning, Lightning.AuthProviders.OauthHTTPClient.PinnedAdapter, allowed_hosts: ["localhost"] +# Lightning.Adaptors.NPM upstream URLs — explicit override for clarity in dev. +# Each key is read by a single sub-module: +# * registry_url → NPM.Registry (npm search + packument) +# * jsdelivr_url → NPM.Schema (configuration-schema.json fetch) +# * github_url → NPM.GitHub (raw icon GETs) +# * github_ref → NPM.GitHub (git ref under OpenFn/adaptors) +config :lightning, Lightning.Adaptors.NPM, + registry_url: "https://registry.npmjs.org", + github_url: "https://raw.githubusercontent.com", + github_ref: "main", + jsdelivr_url: "https://cdn.jsdelivr.net" + config :git_hooks, # In local dev (with a real .git repo) we auto-install hooks. # In Docker builds the .git directory is not present (or incomplete), diff --git a/config/test.exs b/config/test.exs index 96259eaf2da..301e472d4b5 100644 --- a/config/test.exs +++ b/config/test.exs @@ -101,6 +101,20 @@ config :lightning, Lightning.Mailer, adapter: Swoosh.Adapters.Test config :lightning, Lightning.AdaptorRegistry, use_cache: "test/fixtures/adaptor_registry_cache.json" +# Phase A Adaptors.Supervisor config for test boot. +# +# - `:strategy` — the production `Lightning.Adaptors.Supervisor` mounted in +# `application.ex` would default to `Lightning.Adaptors.NPM` and try to +# hit the network on the first Scheduler tick. Replace it with the +# Mox-backed `StrategyMock` so the application-level supervisor (under +# the production name `Lightning.Adaptors`) is a no-op for tests that +# exercise the facade directly. +# - `:refresh_interval` — `0` disables Scheduler tick scheduling entirely. +# Per-test isolated supervisors set their own interval as needed. +config :lightning, Lightning.Adaptors, + strategy: Lightning.Adaptors.StrategyMock, + refresh_interval: 0 + config :hammer, backend: {Hammer.Backend.ETS, diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex new file mode 100644 index 00000000000..fd69d2eb9bb --- /dev/null +++ b/lib/lightning/adaptors.ex @@ -0,0 +1,100 @@ +defmodule Lightning.Adaptors do + @moduledoc """ + Public facade for all adaptor metadata. + + Delegates reads to `Lightning.Adaptors.Store`, refresh calls to + `Lightning.Adaptors.Scheduler`, and version resolution to + `Lightning.Adaptors.Repo`. No logic lives here. + + All functions come in a dual-arity shape: the zero-/single-arg form + passes the compile-time default supervisor name `@sup`; the extra-arity + form accepts an explicit supervisor name for test isolation. + `resolve_version/2` is the single exception — it has no sup arity because + it reads the global Repo directly. + """ + + alias Lightning.Adaptors.Config + alias Lightning.Adaptors.Repo + alias Lightning.Adaptors.Scheduler + alias Lightning.Adaptors.Store + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + @sup Lightning.Adaptors + + @type package_meta :: Store.package_meta() + @type version_meta :: Store.version_meta() + + @spec packages() :: {:ok, [package_meta()]} | {:error, :timeout | term()} + def packages, do: packages(@sup) + + @spec packages(atom()) :: {:ok, [package_meta()]} | {:error, :timeout | term()} + def packages(sup), do: Store.packages(sup) + + @spec versions(String.t()) :: {:ok, [version_meta()]} | {:error, term()} + def versions(pkg), do: versions(@sup, pkg) + + @spec versions(atom(), String.t()) :: + {:ok, [version_meta()]} | {:error, term()} + def versions(sup, pkg), do: Store.versions(sup, pkg) + + @spec schema(String.t()) :: {:ok, String.t()} | {:error, term()} + def schema(pkg), do: schema(@sup, pkg) + + @spec schema(atom(), String.t()) :: {:ok, String.t()} | {:error, term()} + def schema(sup, pkg), do: Store.schema(sup, pkg) + + @spec icon(String.t(), :square | :rectangle) :: + {:ok, Path.t()} | {:error, term()} + def icon(pkg, shape), do: icon(@sup, pkg, shape) + + @spec icon(atom(), String.t(), :square | :rectangle) :: + {:ok, Path.t()} | {:error, term()} + def icon(sup, pkg, shape), do: Store.icon(sup, pkg, shape) + + @spec resolve_version(String.t(), String.t()) :: + {:ok, String.t()} | {:error, :not_found} + def resolve_version(name, requested) when requested in ["latest", "local"] do + case Repo.get_adaptor(name, Config.current_source()) do + %{latest_version: v} -> {:ok, v} + nil -> {:error, :not_found} + end + end + + def resolve_version(_name, version), do: {:ok, version} + + @spec refresh_now() :: :ok | {:error, term()} + def refresh_now, do: refresh_now(@sup) + + @spec refresh_now(atom()) :: :ok | {:error, term()} + def refresh_now(sup), + do: Scheduler.refresh_now(AdaptorsSupervisor.global_scheduler_name(sup)) + + @spec refresh_package(String.t()) :: :ok | {:error, :not_found | term()} + def refresh_package(name) when is_binary(name), do: refresh_package(@sup, name) + + @spec refresh_package(atom(), String.t()) :: + :ok | {:error, :not_found | term()} + def refresh_package(sup, name) when is_binary(name), + do: + Scheduler.refresh_package( + AdaptorsSupervisor.global_scheduler_name(sup), + name + ) + + @spec refresh_icons() :: + {:ok, %{updated: non_neg_integer(), unchanged: non_neg_integer()}} + | {:error, term()} + def refresh_icons, do: refresh_icons(@sup) + + @spec refresh_icons(atom()) :: + {:ok, %{updated: non_neg_integer(), unchanged: non_neg_integer()}} + | {:error, term()} + def refresh_icons(sup), + do: Scheduler.refresh_icons(AdaptorsSupervisor.global_scheduler_name(sup)) + + @doc false + def icon_meta(name), do: icon_meta(@sup, name) + + @doc false + def icon_meta(sup, name), do: Store.icon_meta(sup, name) +end diff --git a/lib/lightning/adaptors/channel_broadcaster.ex b/lib/lightning/adaptors/channel_broadcaster.ex new file mode 100644 index 00000000000..85b598374c0 --- /dev/null +++ b/lib/lightning/adaptors/channel_broadcaster.ex @@ -0,0 +1,91 @@ +defmodule Lightning.Adaptors.ChannelBroadcaster do + @moduledoc """ + Burst-coalesced fan-out of adaptor changes to connected sessions. + + Subscribes to `:source_topic` (the cache-coherence topic shared with + `Lightning.Adaptors.Invalidator`) and republishes a single pre-rendered + envelope to `:client_topic` at most once per 250ms leading-edge window. + + Two-topic separation: the source topic is the cache-coherence audience; + the client topic is the display-freshness audience (`WorkflowChannel` + subscribers). This bridges them: `Lightning.Adaptors.packages/1` is + rendered once per burst and fanned out by PubSub rather than once per + session (§6.5c). + + No within-callback fan-out in `:flush` — `Phoenix.PubSub.broadcast/3` + is a single call that reaches all subscribers in one hop (§10 #19). + """ + + use GenServer + + @debounce_ms 250 + + @doc """ + Leading-edge coalesce window in milliseconds. + + Exposed so integration tests can compute receive timeouts off the + authoritative value rather than hard-coding a duplicate. + """ + @spec debounce_ms() :: pos_integer() + def debounce_ms, do: @debounce_ms + + @doc """ + Start the ChannelBroadcaster linked to the calling process. + + Required opts: + * `:name` — registered GenServer name. + * `:source_topic` — PubSub topic to subscribe to (cache-coherence). + * `:client_topic` — PubSub topic to broadcast the rendered envelope to. + * `:sup` — supervisor instance name; forwarded to + `Lightning.Adaptors.packages/1` for per-instance isolation. + """ + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts) do + name = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts) do + :ok = + Phoenix.PubSub.subscribe( + Lightning.PubSub, + Keyword.fetch!(opts, :source_topic) + ) + + {:ok, + %{ + client_topic: Keyword.fetch!(opts, :client_topic), + sup: Keyword.fetch!(opts, :sup), + timer: nil + }} + end + + @impl true + # First message of a burst: arm the leading-edge timer. + def handle_info({:changed, _name, _source}, %{timer: nil} = state) do + timer = Process.send_after(self(), :flush, @debounce_ms) + {:noreply, %{state | timer: timer}} + end + + # Subsequent messages within the debounce window: drop on the floor. + def handle_info({:changed, _name, _source}, state) do + {:noreply, state} + end + + def handle_info(:flush, %{client_topic: topic, sup: sup} = state) do + case Lightning.Adaptors.packages(sup) do + {:ok, pkgs} -> + Phoenix.PubSub.broadcast( + Lightning.PubSub, + topic, + %{event: "adaptors_updated", payload: %{adaptors: pkgs}} + ) + + {:error, _} -> + :ok + end + + {:noreply, %{state | timer: nil}} + end +end diff --git a/lib/lightning/adaptors/config.ex b/lib/lightning/adaptors/config.ex new file mode 100644 index 00000000000..8f45fd8cfcf --- /dev/null +++ b/lib/lightning/adaptors/config.ex @@ -0,0 +1,102 @@ +defmodule Lightning.Adaptors.Config do + @moduledoc """ + Stateless runtime configuration for the `Lightning.Adaptors.*` subsystem. + + Every helper is a thin wrapper around `Application.get_env/3`. It is the + single runtime source of truth for which strategy is active, how often + the scheduler ticks, the per-call cache fetch deadline, the icon cache + root, and per-strategy opt blocks. + + ## Application key layout + + Two-tier: + + * `:lightning, Lightning.Adaptors` — subsystem-wide knobs + (`:strategy`, `:refresh_interval`, `:cache_timeout_ms`, `:icon_path`). + * `:lightning, ` — each strategy owns its own + Application key for its own knobs; read via `strategy_opts/1`. + + No GenServer, no ETS, no `:persistent_term` — every call is a fresh + `Application.get_env/3`. + """ + + @parent_key Lightning.Adaptors + + @default_strategy Lightning.Adaptors.NPM + @default_refresh_interval :timer.hours(1) + @default_cache_timeout_ms 15_000 + @default_icon_path {:tmp, "lightning/adaptor_icons"} + + @doc """ + The active strategy module. Defaults to `Lightning.Adaptors.NPM`. + """ + @spec strategy() :: module() + def strategy do + get(:strategy, @default_strategy) + end + + @doc """ + Atom mapping of `strategy/0`: `:local` for `Lightning.Adaptors.Local`, + `:npm` for any other strategy module. + """ + @spec current_source() :: :local | :npm + def current_source do + case strategy() do + Lightning.Adaptors.Local -> :local + _other -> :npm + end + end + + @doc """ + Scheduler tick interval in milliseconds. Defaults to one hour. + """ + @spec refresh_interval() :: non_neg_integer() + def refresh_interval do + get(:refresh_interval, @default_refresh_interval) + end + + @doc """ + Per-`Cachex.fetch` courier deadline in milliseconds. Defaults to 15s. + """ + @spec cache_timeout_ms() :: non_neg_integer() + def cache_timeout_ms do + get(:cache_timeout_ms, @default_cache_timeout_ms) + end + + @doc """ + Resolved filesystem path for the icon cache. + + Accepts either: + + * `{:tmp, suffix}` — resolved against `System.tmp_dir!/0` at call + time so the default does not bake a container-specific tmp path + into a compiled release. + * a plain binary path — returned verbatim. + + Defaults to `{:tmp, "lightning/adaptor_icons"}`. + """ + @spec icon_path() :: Path.t() + def icon_path do + case get(:icon_path, @default_icon_path) do + {:tmp, suffix} -> Path.join(System.tmp_dir!(), suffix) + path when is_binary(path) -> path + end + end + + @doc """ + Per-strategy keyword opts. Parameterised on the strategy module — each + strategy is its own Application key, not nested under the parent. + Returns `[]` when the strategy's Application key is unset. + """ + @spec strategy_opts(module()) :: keyword() + def strategy_opts(strategy_mod) when is_atom(strategy_mod) do + Application.get_env(:lightning, strategy_mod, []) + end + + @spec get(atom(), term()) :: term() + defp get(key, default) do + :lightning + |> Application.get_env(@parent_key, []) + |> Keyword.get(key, default) + end +end diff --git a/lib/lightning/adaptors/icon_cache.ex b/lib/lightning/adaptors/icon_cache.ex new file mode 100644 index 00000000000..22147f68139 --- /dev/null +++ b/lib/lightning/adaptors/icon_cache.ex @@ -0,0 +1,96 @@ +defmodule Lightning.Adaptors.IconCache do + @moduledoc """ + Pure filesystem helper owning the on-disk adaptor icon cache. + + Not a GenServer. Three stateless functions over + `Lightning.Adaptors.Config.icon_path/0`, which resolves the + `{:tmp, suffix}` default at call time. + + Disk layout is **source-partitioned** and **latest-only**: + + ///. + + Source partitioning means flipping `LOCAL_ADAPTORS` between restarts + cannot accidentally serve `:npm` bytes from a row that's now resolved + via `:local` (or vice versa). Latest-only means a subsequent + `write!/5` for the same key overwrites — content-addressable URLs + carry the sha8 prefix, so cache invalidation is intrinsic and we + don't need to keep old versions on disk. + + Concurrent first-request fetchers are coalesced upstream by Cachex's + courier on `{:icon_bytes, source, name, shape}` inside + `Lightning.Adaptors.Store.icon/3` — the courier returns `{:ignore, _}` + so no entry is committed, but all in-flight peers receive the courier's + result for free. The temp-then-rename in `write!/5` is the belt-and- + braces guarantee for the file-write step itself: readers never observe + a half-written file. + """ + + alias Lightning.Adaptors.Config + + @type source :: :npm | :local + @type name :: String.t() + @type shape :: :square | :rectangle + @type ext :: String.t() + + @doc """ + Disk path for an icon. Pure — nothing is checked or created. + + `name` may contain a `/` (scoped npm packages like + `@openfn/language-foo`); `Path.join/1` preserves the slash so the + scope becomes a real subdirectory. + """ + @spec path(source(), name(), shape(), ext()) :: Path.t() + def path(source, name, shape, ext) do + Path.join([Config.icon_path(), to_string(source), name, "#{shape}.#{ext}"]) + end + + @doc """ + Whether the icon at `path(source, name, shape, ext)` exists on disk. + """ + @spec cached?(source(), name(), shape(), ext()) :: boolean() + def cached?(source, name, shape, ext) do + File.exists?(path(source, name, shape, ext)) + end + + @doc """ + Atomically write `bytes` to `path(source, name, shape, ext)` and + return the sha256 of the supplied bytes as a 32-byte binary. + + The write is staged in a sibling temp file and then renamed into + place, so concurrent readers never observe a half-written file. + + The caller (Strategy / Scheduler) persists the returned sha on the + adaptor row and is responsible for verifying it matches the expected + sha from the upstream `adaptor_record` — defence against tarball or + filesystem corruption. + """ + @spec write!(source(), name(), shape(), ext(), binary()) :: + {:ok, binary()} + def write!(source, name, shape, ext, bytes) when is_binary(bytes) do + final_path = path(source, name, shape, ext) + dir = Path.dirname(final_path) + File.mkdir_p!(dir) + + sha = :crypto.hash(:sha256, bytes) + + temp_path = + Path.join(dir, ".#{Path.basename(final_path)}.#{random_suffix()}.tmp") + + try do + File.write!(temp_path, bytes) + File.rename!(temp_path, final_path) + rescue + e -> + _ = File.rm(temp_path) + reraise e, __STACKTRACE__ + end + + {:ok, sha} + end + + @spec random_suffix() :: String.t() + defp random_suffix do + 8 |> :crypto.strong_rand_bytes() |> Base.encode16(case: :lower) + end +end diff --git a/lib/lightning/adaptors/invalidator.ex b/lib/lightning/adaptors/invalidator.ex new file mode 100644 index 00000000000..a548c23ee13 --- /dev/null +++ b/lib/lightning/adaptors/invalidator.ex @@ -0,0 +1,45 @@ +defmodule Lightning.Adaptors.Invalidator do + @moduledoc """ + Subscribes to cluster adaptor-change broadcasts and evicts matching + local Cachex entries, keeping each node coherent with Postgres. + + Subscribes to `opts[:source_topic]` on `Lightning.PubSub` at init. + On `{:changed, name, source}`, deletes the four per-adaptor cache keys + written by `Lightning.Adaptors.Store`. No source filtering on the hot + path — a broadcast for a source that isn't active on this node is a + no-op because those keys simply don't exist in Cachex. + """ + + use GenServer + + @doc """ + Start the Invalidator linked to the calling process. + + Required opts: + * `:name` — registered process name. + * `:source_topic` — `Phoenix.PubSub` topic to subscribe to. + * `:cache` — Cachex table atom (from `Lightning.Adaptors.Supervisor.cache_name/1`). + """ + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts) do + name = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts) do + topic = Keyword.fetch!(opts, :source_topic) + cache = Keyword.fetch!(opts, :cache) + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, topic) + {:ok, %{cache: cache}} + end + + @impl true + def handle_info({:changed, name, source}, state) do + Cachex.del(state.cache, {:schema, name, source}) + Cachex.del(state.cache, {:versions, name, source}) + Cachex.del(state.cache, {:icon_meta, name, source}) + Cachex.del(state.cache, {:packages, source}) + {:noreply, state} + end +end diff --git a/lib/lightning/adaptors/local.ex b/lib/lightning/adaptors/local.ex new file mode 100644 index 00000000000..c47a942af6f --- /dev/null +++ b/lib/lightning/adaptors/local.ex @@ -0,0 +1,229 @@ +defmodule Lightning.Adaptors.Local do + @moduledoc """ + Filesystem implementation of `Lightning.Adaptors.Strategy`. + + Serves adaptor metadata, schemas, and icons from an on-disk OpenFn + adaptors monorepo checkout. Gated by `LOCAL_ADAPTORS=true` and + `OPENFN_ADAPTORS_REPO=/path/to/adaptors` at the runtime-config layer; + this module only reads the resolved path via + `Lightning.Adaptors.Config.strategy_opts(__MODULE__)[:path]`. + + Each callback walks the filesystem afresh — caching is the Store's + responsibility. The module is stateless; no GenServer, no ETS. + + ## Layout + + Walks `$path/packages/*/`, reads each subdirectory's `package.json` + for the authoritative `name` and `version`. Directories with missing + or unparseable `package.json` are skipped with `Logger.warning` so a + malformed entry never crashes boot. Multiple directories sharing the + same `name` are collapsed into one record: `latest_version` is the + highest semver and `versions` lists every on-disk path. + + `source: :local` is **not** set here — the Store stamps it before + upsert. No network calls anywhere in this module. + """ + + @behaviour Lightning.Adaptors.Strategy + + alias Lightning.Adaptors.Config + + require Logger + + @schema_filename "configuration-schema.json" + @icon_exts ~w(png svg) + + @impl Lightning.Adaptors.Strategy + def list_adaptors do + with {:ok, records} <- discover() do + {:ok, + Enum.map(records, fn %{name: name, latest_version: v} -> + %{name: name, latest_version: v} + end)} + end + end + + @impl Lightning.Adaptors.Strategy + def fetch_adaptor(name) when is_binary(name) do + with {:ok, records} <- discover() do + case Enum.find(records, &(&1.name == name)) do + nil -> {:error, :not_found} + record -> {:ok, build_adaptor_record(record)} + end + end + end + + @impl Lightning.Adaptors.Strategy + def fetch_icon(name, shape) + when is_binary(name) and shape in [:square, :rectangle] do + with {:ok, records} <- discover() do + case Enum.find(records, &(&1.name == name)) do + nil -> {:error, :not_found} + %{latest_path: path} -> read_icon(path, shape) + end + end + end + + @impl Lightning.Adaptors.Strategy + def fetch_icons(_opts \\ []) do + with {:ok, records} <- discover() do + icons = + Enum.reduce(records, %{}, fn record, acc -> + Enum.reduce([:square, :rectangle], acc, fn shape, inner -> + case read_icon(record.latest_path, shape) do + {:ok, %{data: bytes, ext: ext}} -> + entry = %{ + data: bytes, + ext: ext, + sha256: :crypto.hash(:sha256, bytes) + } + + Map.update( + inner, + record.name, + %{shape => entry}, + &Map.put(&1, shape, entry) + ) + + {:error, _} -> + inner + end + end) + end) + + {:ok, icons} + end + end + + defp discover do + case Config.strategy_opts(__MODULE__)[:path] do + nil -> + Logger.warning( + "Lightning.Adaptors.Local: :path is not configured " <> + "(set OPENFN_ADAPTORS_REPO or :lightning, Lightning.Adaptors.Local, path:)" + ) + + {:error, :no_repo_path} + + path -> + records = + path + |> Path.join("packages") + |> Path.join("*") + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + |> Enum.flat_map(&read_package_dir/1) + |> group_by_name() + + {:ok, records} + end + end + + defp read_package_dir(dir) do + pkg_json_path = Path.join(dir, "package.json") + + with {:ok, body} <- File.read(pkg_json_path), + {:ok, %{"name" => name, "version" => version} = parsed} + when is_binary(name) and is_binary(version) <- Jason.decode(body) do + [%{name: name, version: version, path: dir, package_json: parsed}] + else + other -> + Logger.warning( + "Lightning.Adaptors.Local: skipping #{inspect(dir)}: " <> + "missing or unparseable package.json (#{inspect(other)})" + ) + + [] + end + end + + defp group_by_name(entries) do + entries + |> Enum.group_by(& &1.name) + |> Enum.map(fn {name, versions} -> + sorted = Enum.sort_by(versions, & &1.version, &version_descending/2) + latest = List.first(sorted) + + %{ + name: name, + latest_version: latest.version, + latest_path: latest.path, + latest_package_json: latest.package_json, + versions: sorted + } + end) + end + + defp version_descending(a, b) do + case {Version.parse(a), Version.parse(b)} do + {{:ok, va}, {:ok, vb}} -> Version.compare(va, vb) != :lt + _ -> a >= b + end + end + + defp build_adaptor_record(record) do + pkg = record.latest_package_json + {schema_data, schema_sha256} = read_schema(record.latest_path) + + %{ + name: record.name, + description: pkg["description"], + homepage: pkg["homepage"], + repository: extract_repository(pkg["repository"]), + license: pkg["license"], + latest_version: record.latest_version, + deprecated: false, + schema_data: schema_data, + schema_sha256: schema_sha256, + versions: Enum.map(record.versions, &build_version_record/1) + } + end + + defp build_version_record(%{version: v, package_json: pkg}) do + %{ + version: v, + integrity: nil, + tarball_url: nil, + size_bytes: nil, + dependencies: Map.get(pkg, "dependencies", %{}), + peer_dependencies: Map.get(pkg, "peerDependencies", %{}), + published_at: nil, + deprecated: false + } + end + + defp read_schema(dir) do + case File.read(Path.join(dir, @schema_filename)) do + {:ok, body} -> + # Validate JSON, but keep the raw binary so credential-form + # rendering can re-engage ordered_objects decoding downstream. + case Jason.decode(body) do + {:ok, _data} -> + sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) + {body, sha} + + {:error, _} -> + {nil, nil} + end + + {:error, _} -> + {nil, nil} + end + end + + defp read_icon(dir, shape) do + Enum.find_value(@icon_exts, {:error, :not_found}, fn ext -> + case File.read(icon_path(dir, shape, ext)) do + {:ok, bytes} -> {:ok, %{data: bytes, ext: ext}} + {:error, _} -> nil + end + end) + end + + defp icon_path(dir, shape, ext), + do: Path.join([dir, "assets", "#{shape}.#{ext}"]) + + defp extract_repository(repo) when is_binary(repo), do: repo + defp extract_repository(%{"url" => url}) when is_binary(url), do: url + defp extract_repository(_), do: nil +end diff --git a/lib/lightning/adaptors/node_monitor.ex b/lib/lightning/adaptors/node_monitor.ex new file mode 100644 index 00000000000..3ecc0d3ab8f --- /dev/null +++ b/lib/lightning/adaptors/node_monitor.ex @@ -0,0 +1,48 @@ +defmodule Lightning.Adaptors.NodeMonitor do + @moduledoc """ + Partition-recovery companion to `Lightning.Adaptors.Invalidator`. + + On `:nodeup`, re-warms the Cachex table from Postgres so a reconnecting + peer never serves stale data until the 24-hour TTL expires. Steady-state + invalidation belongs to `Lightning.Adaptors.Invalidator`. + + `:nodedown` is a deliberate no-op. The worst case on a silent departure is + one stale-URL redirect per client, backstopped by 302-on-stale-sha. + """ + + use GenServer + + alias Lightning.Adaptors.Store + + @doc """ + Start a NodeMonitor for the given supervisor instance. + + Required opts: + * `:name` — registered GenServer name (§6.11 async-test rule). + * `:sup` — supervisor instance name, forwarded to `Store.warm_from_repo/1`. + """ + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts) do + name = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts) do + sup = Keyword.fetch!(opts, :sup) + :net_kernel.monitor_nodes(true, node_type: :visible) + {:ok, %{sup: sup}} + end + + @impl true + def handle_info({:nodeup, _node, _info}, state) do + Store.warm_from_repo(state.sup) + {:noreply, state} + end + + # Deliberate no-op: nodedown does not trigger a re-warm. The 24h Cachex TTL + # backstops any staleness; 302-on-stale-sha handles already-issued URLs. + def handle_info({:nodedown, _node, _info}, state) do + {:noreply, state} + end +end diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex new file mode 100644 index 00000000000..d5ed5e44c29 --- /dev/null +++ b/lib/lightning/adaptors/npm.ex @@ -0,0 +1,108 @@ +defmodule Lightning.Adaptors.NPM do + @moduledoc """ + Production implementation of `Lightning.Adaptors.Strategy` that talks + to the public NPM registry and the OpenFn adaptors monorepo on GitHub. + + Consolidates the legacy `Lightning.AdaptorRegistry`, + `Mix.Tasks.Lightning.InstallSchemas`, and + `Mix.Tasks.Lightning.InstallAdaptorIcons` into one stateless module: + + * `c:list_adaptors/0` — single search-API call returning + `name + latest_version` for every `@openfn/language-*` package. + * `c:fetch_adaptor/1` — packument fetch + per-version decode and + latest-version schema retrieval via jsDelivr. Icon fields are + **not** stamped here; the Scheduler joins them on after a bulk + `c:fetch_icons/1` pass. + * `c:fetch_icon/2` — single icon raw GET against + `raw.githubusercontent.com`, used by the Store's rare lazy-miss + fallback. + * `c:fetch_icons/1` — bulk fan-out over the search listing, one + HTTP request per `(name, shape)`. Threads `:prior_etags` from + the caller down into the per-request `If-None-Match` headers. + + ## HTTP + + This module is a thin orchestrator. The actual HTTP work is delegated + to three sub-modules, each of which owns its own Tesla client and + upstream base URL: + + * `Lightning.Adaptors.NPM.Registry` — npm registry search + packument. + * `Lightning.Adaptors.NPM.Schema` — jsDelivr `configuration-schema.json`. + * `Lightning.Adaptors.NPM.GitHub` — `raw.githubusercontent.com` + icon fetches (one GET per `(name, shape)`). + + Each sub-module issues at most a handful of single-shot Tesla requests + bounded by `http_timeout`. No retry, no backoff, no circuit-breaker — + transient failures (5xx, timeout, nxdomain) of the *primary* request + (`packument` for `fetch_adaptor/1`, `search` for `list_adaptors/0` and + `fetch_icons/1`) surface as `{:error, term()}` unchanged. Schema and + icon fetches inside `fetch_adaptor/1` and `fetch_icons/1` are + best-effort: a single icon miss degrades that entry to absence rather + than failing the whole record. + + ## Configuration + + Each sub-module reads `:registry_url`, `:jsdelivr_url`, `:github_url`, + `:github_ref`, and `:http_timeout` via + `Lightning.Adaptors.Config.strategy_opts(__MODULE__)`, with defaults + baked in so the module works even when no Application env block is + set. + """ + + @behaviour Lightning.Adaptors.Strategy + + alias Lightning.Adaptors.NPM.GitHub + alias Lightning.Adaptors.NPM.Registry + alias Lightning.Adaptors.NPM.Schema + + @impl Lightning.Adaptors.Strategy + def list_adaptors, do: Registry.list_adaptors() + + @impl Lightning.Adaptors.Strategy + def fetch_adaptor(name) when is_binary(name) do + with {:ok, packument} <- Registry.get_packument(name), + {:ok, latest_version} <- Registry.latest_version(packument) do + {schema_data, schema_sha} = Schema.schema(name, latest_version) + + {:ok, + %{ + name: Map.get(packument, "name", name), + description: Map.get(packument, "description"), + homepage: Map.get(packument, "homepage"), + repository: Registry.repository_url(Map.get(packument, "repository")), + license: Map.get(packument, "license"), + latest_version: latest_version, + deprecated: Registry.deprecated?(packument, latest_version), + schema_data: encode_schema(schema_data), + schema_sha256: schema_sha, + versions: Registry.build_versions(packument) + }} + end + end + + # Strategy boundary: re-encode the decoded schema map to a JSON binary + # so the row is persisted as text and `Jason.decode!(_, + # objects: :ordered_objects)` re-engages downstream. NPM's upstream + # Schema sub-module already decoded into a regular map, so field order + # is whatever map iteration yields — the round-trip preserves it for + # the Local strategy (raw binary in) and is a no-op for NPM data. + defp encode_schema(nil), do: nil + defp encode_schema(data) when is_binary(data), do: data + defp encode_schema(data) when is_map(data), do: Jason.encode!(data) + + @impl Lightning.Adaptors.Strategy + def fetch_icon(name, shape) + when is_binary(name) and shape in [:square, :rectangle] do + GitHub.fetch_one(name, shape) + end + + @impl Lightning.Adaptors.Strategy + def fetch_icons(opts \\ []) when is_list(opts) do + prior_etags = Keyword.get(opts, :prior_etags, %{}) + + with {:ok, listing} <- Registry.list_adaptors() do + names = Enum.map(listing, & &1.name) + GitHub.fetch_all(names, prior_etags) + end + end +end diff --git a/lib/lightning/adaptors/npm/github.ex b/lib/lightning/adaptors/npm/github.ex new file mode 100644 index 00000000000..1e2a8096cd4 --- /dev/null +++ b/lib/lightning/adaptors/npm/github.ex @@ -0,0 +1,302 @@ +defmodule Lightning.Adaptors.NPM.GitHub do + @moduledoc """ + Raw `raw.githubusercontent.com` client for adaptor icons. + + Icons aren't published inside npm tarballs — they live in the + `OpenFn/adaptors` monorepo. This module fetches them directly via the + raw GitHub content host, one icon per HTTP GET, no tarball walking. + + ## URL pattern + + /OpenFn/adaptors//packages//assets/. + + where `` strips the `@openfn/` scope from the package name. + Each `(name, shape)` is probed `png` first then `svg` — matching the + ext order used by `Lightning.Adaptors.Local`. + + ## Configuration + + Both `:github_url` (default `https://raw.githubusercontent.com`) and + `:github_ref` (default `main`) are read via + `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)`, + symmetric with the existing `:registry_url`, `:jsdelivr_url`, and + `:http_timeout` keys. + """ + + alias Lightning.Adaptors.Config + + require Logger + + @default_github_url "https://raw.githubusercontent.com" + @default_github_ref "main" + @default_http_timeout :timer.seconds(30) + + @default_max_concurrency 20 + + @icon_exts ~w(png svg) + @scope_prefix "@openfn/" + @language_prefix "language-" + + @doc """ + Fetch a single icon for `(name, shape)`. + + Tries `png` then `svg`. No conditional GET — this entry point is used + by the Store's lazy-miss fallback, where no prior etag is in scope. + Returns: + + * `{:ok, %{data: binary(), ext: String.t(), etag: String.t() | nil}}` + on success. + * `{:error, :not_found}` when neither ext yields a 200. + * `{:error, term()}` on transport-level failure (timeout, nxdomain). + """ + @spec fetch_one(String.t(), :square | :rectangle) :: + {:ok, %{data: binary(), ext: String.t(), etag: String.t() | nil}} + | {:error, :not_found | term()} + def fetch_one(name, shape) + when is_binary(name) and shape in [:square, :rectangle] do + client = raw_client() + do_fetch_one(client, name, shape, nil) + end + + @doc """ + Fetch icons for every `(name, shape)` pair across `names`. + + Returns `{:ok, partial_map}` where each entry is keyed by the + package name and contains zero, one, or two shape keys. Absence + is **not** an error — packages with no upstream icon simply do not + appear (or appear with a missing shape). + + When `prior_etags` is supplied as `%{name => %{shape => etag}}`, the + corresponding `If-None-Match` header is sent per `(name, shape)`. A + 304 response is surfaced as a `:not_modified` sentinel in the + per-shape slot — distinct from "absent" which means upstream had no + such shape at all. + + Fans out via `Task.async_stream` with a bounded concurrency. Transport + failures for a single `(name, shape)` are dropped silently — the whole + pipeline only fails if every fetch crashes the supervisor, which is + not surfaced here. + """ + @spec fetch_all([String.t()], %{ + optional(String.t()) => %{ + optional(:square) => String.t(), + optional(:rectangle) => String.t() + } + }) :: + {:ok, + %{ + required(String.t()) => %{ + optional(:square) => + %{ + data: binary(), + ext: String.t(), + sha256: binary(), + etag: String.t() | nil + } + | :not_modified, + optional(:rectangle) => + %{ + data: binary(), + ext: String.t(), + sha256: binary(), + etag: String.t() | nil + } + | :not_modified + } + }} + def fetch_all(names, prior_etags) + when is_list(names) and is_map(prior_etags) do + client = raw_client() + + work = + for name <- names, shape <- [:square, :rectangle], do: {name, shape} + + {results, counts} = + work + |> Task.async_stream( + fn {name, shape} -> + prior = prior_etag_for(prior_etags, name, shape) + + case do_fetch_one(client, name, shape, prior) do + {:ok, %{data: bytes, ext: ext, etag: etag}} -> + {name, shape, + {:ok, + %{ + data: bytes, + ext: ext, + sha256: :crypto.hash(:sha256, bytes), + etag: etag + }}} + + :not_modified -> + {name, shape, :not_modified} + + {:error, reason} -> + {name, shape, {:error, reason}} + end + end, + max_concurrency: max_concurrency(), + timeout: max(http_timeout() * 2, 5_000), + on_timeout: :kill_task, + ordered: false + ) + |> Enum.reduce( + {%{}, %{fetched: 0, not_modified: 0, not_found: 0, error: 0}}, + fn + {:ok, {name, shape, {:ok, entry}}}, {acc, c} -> + {put_entry(acc, name, shape, entry), + Map.update!(c, :fetched, &(&1 + 1))} + + {:ok, {name, shape, :not_modified}}, {acc, c} -> + {put_entry(acc, name, shape, :not_modified), + Map.update!(c, :not_modified, &(&1 + 1))} + + {:ok, {_name, _shape, {:error, :not_found}}}, {acc, c} -> + {acc, Map.update!(c, :not_found, &(&1 + 1))} + + {:ok, {_name, _shape, {:error, _reason}}}, {acc, c} -> + {acc, Map.update!(c, :error, &(&1 + 1))} + + {:exit, _reason}, {acc, c} -> + {acc, Map.update!(c, :error, &(&1 + 1))} + end + ) + + Logger.info( + "NPM.GitHub: fetch_all names=#{length(names)} pairs=#{length(work)} " <> + "fetched=#{counts.fetched} not_modified=#{counts.not_modified} " <> + "not_found=#{counts.not_found} errors=#{counts.error}" + ) + + {:ok, results} + end + + defp prior_etag_for(prior_etags, name, shape) do + case Map.get(prior_etags, name) do + %{} = shapes -> Map.get(shapes, shape) + _ -> nil + end + end + + defp put_entry(acc, name, shape, entry) do + Map.update(acc, name, %{shape => entry}, &Map.put(&1, shape, entry)) + end + + # GitHub raw does not gzip-compress PNG/SVG bodies, so no + # `Tesla.Middleware.DecompressResponse` is in play. The sha256 computed + # in `fetch_all/2` is therefore over the raw response bytes (which equals + # the decompressed bytes in our case — there is no compression layer to + # speak of, and no `accept-encoding` middleware is configured). If GitHub + # ever starts compressing 200 bodies, Tesla/Finch would need an + # accept-encoding/decompress middleware to preserve this invariant. + defp do_fetch_one(client, name, shape, prior_etag) do + suffix = strip_scope(name) + cond_get? = is_binary(prior_etag) + + headers = + if cond_get?, do: [{"if-none-match", prior_etag}], else: [] + + Enum.reduce_while(@icon_exts, {:error, :not_found}, fn ext, _acc -> + path = build_path(suffix, shape, ext) + + case Tesla.get(client, path, headers: headers) do + {:ok, %Tesla.Env{status: 200, body: body} = env} when is_binary(body) -> + etag = response_etag(env) + + Logger.debug(fn -> + "NPM.GitHub: GET #{path} → 200 (#{byte_size(body)}B) " <> + "name=#{name} shape=#{shape} cond_get=#{cond_get?}" + end) + + {:halt, {:ok, %{data: body, ext: ext, etag: etag}}} + + {:ok, %Tesla.Env{status: 304}} -> + Logger.debug(fn -> + "NPM.GitHub: GET #{path} → 304 name=#{name} shape=#{shape} " <> + "cond_get=#{cond_get?}" + end) + + {:halt, :not_modified} + + {:ok, %Tesla.Env{status: 404}} -> + Logger.debug(fn -> + "NPM.GitHub: GET #{path} → 404 name=#{name} shape=#{shape} " <> + "cond_get=#{cond_get?}" + end) + + {:cont, {:error, :not_found}} + + {:ok, %Tesla.Env{status: status}} -> + Logger.debug(fn -> + "NPM.GitHub: GET #{path} → #{status} name=#{name} shape=#{shape} " <> + "cond_get=#{cond_get?}" + end) + + {:halt, {:error, {:http_status, status}}} + + {:error, reason} -> + Logger.debug(fn -> + "NPM.GitHub: GET #{path} → transport error #{inspect(reason)} " <> + "name=#{name} shape=#{shape} cond_get=#{cond_get?}" + end) + + {:halt, {:error, reason}} + end + end) + end + + defp response_etag(%Tesla.Env{headers: headers}) do + Enum.find_value(headers, fn {k, v} -> + if is_binary(k) and String.downcase(k) == "etag", do: v + end) + end + + defp build_path(name_suffix, shape, ext) do + "/OpenFn/adaptors/#{github_ref()}/packages/#{name_suffix}/assets/#{shape}.#{ext}" + end + + defp strip_scope(@scope_prefix <> @language_prefix <> rest), do: rest + defp strip_scope(@scope_prefix <> rest), do: rest + defp strip_scope(name), do: name + + defp raw_client do + build_client([ + {Tesla.Middleware.BaseUrl, github_url()}, + Tesla.Middleware.FollowRedirects + ]) + end + + defp build_client(middleware) do + case Application.get_env(:tesla, :adapter) do + {Tesla.Adapter.Finch, _opts} -> + Tesla.client( + middleware, + {Tesla.Adapter.Finch, + name: Lightning.Finch, receive_timeout: http_timeout()} + ) + + _other -> + Tesla.client(middleware) + end + end + + defp github_url do + Config.strategy_opts(Lightning.Adaptors.NPM)[:github_url] || + @default_github_url + end + + defp github_ref do + Config.strategy_opts(Lightning.Adaptors.NPM)[:github_ref] || + @default_github_ref + end + + defp http_timeout do + Config.strategy_opts(Lightning.Adaptors.NPM)[:http_timeout] || + @default_http_timeout + end + + defp max_concurrency do + Config.strategy_opts(Lightning.Adaptors.NPM)[:icon_max_concurrency] || + @default_max_concurrency + end +end diff --git a/lib/lightning/adaptors/npm/registry.ex b/lib/lightning/adaptors/npm/registry.ex new file mode 100644 index 00000000000..a05e93dfb02 --- /dev/null +++ b/lib/lightning/adaptors/npm/registry.ex @@ -0,0 +1,195 @@ +defmodule Lightning.Adaptors.NPM.Registry do + @moduledoc """ + NPM registry HTTP client for `Lightning.Adaptors.NPM`. + + Talks to `registry.npmjs.org`. Responsible for the `search` endpoint + used by `c:Lightning.Adaptors.Strategy.list_adaptors/0` and the + `packument` endpoint used by `fetch_adaptor/1` and `fetch_icon/2`. + + Base URL via `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)[:registry_url]`, + default `https://registry.npmjs.org`. + + Search results are filtered down to `@openfn/language-*` packages, + matching the legacy `AdaptorRegistry` semantics; non-language packages + in the `@openfn/` scope (e.g. `@openfn/cli`) are rejected. + """ + + alias Lightning.Adaptors.Config + + @default_registry_url "https://registry.npmjs.org" + @default_http_timeout :timer.seconds(30) + + @search_scope "openfn" + @search_size 250 + + @language_prefix "@openfn/language-" + + @doc """ + Single `/-/v1/search` call returning `name + latest_version` for every + `@openfn/language-*` package. + """ + @spec list_adaptors() :: + {:ok, [%{name: String.t(), latest_version: String.t()}]} + | {:error, term()} + def list_adaptors do + case Tesla.get(json_client(), "/-/v1/search", + query: [text: "@" <> @search_scope, size: @search_size] + ) do + {:ok, %Tesla.Env{status: 200, body: body}} when is_map(body) -> + listing = + body + |> Map.get("objects", []) + |> Enum.map(&extract_listing_entry/1) + |> Enum.reject(&is_nil/1) + + {:ok, listing} + + {:ok, %Tesla.Env{status: status}} -> + {:error, {:http_status, status}} + + {:error, reason} -> + {:error, reason} + end + end + + @doc """ + Fetch the full packument for a package. + """ + @spec get_packument(String.t()) :: + {:ok, map()} | {:error, :not_found} | {:error, term()} + def get_packument(name) do + case Tesla.get(json_client(), "/" <> name) do + {:ok, %Tesla.Env{status: 200, body: body}} when is_map(body) -> + {:ok, body} + + {:ok, %Tesla.Env{status: 404}} -> + {:error, :not_found} + + {:ok, %Tesla.Env{status: status}} -> + {:error, {:http_status, status}} + + {:error, reason} -> + {:error, reason} + end + end + + @doc """ + Extract the `dist-tags.latest` version from a packument. + """ + @spec latest_version(map()) :: + {:ok, String.t()} | {:error, :no_latest_version} + def latest_version(packument) do + case get_in(packument, ["dist-tags", "latest"]) do + v when is_binary(v) -> {:ok, v} + _ -> {:error, :no_latest_version} + end + end + + @doc """ + Resolve the tarball URL for `version` in `packument`. + """ + @spec require_tarball_url(map(), String.t()) :: + {:ok, String.t()} | {:error, :no_tarball_url} + def require_tarball_url(packument, version) do + case get_in(packument, ["versions", version, "dist", "tarball"]) do + url when is_binary(url) -> {:ok, url} + _ -> {:error, :no_tarball_url} + end + end + + @doc """ + Build the per-version `version_record` list from a packument. + """ + @spec build_versions(map()) :: [map()] + def build_versions(packument) do + versions = Map.get(packument, "versions", %{}) + times = Map.get(packument, "time", %{}) + + Enum.map(versions, fn {version, info} -> + %{ + version: version, + integrity: get_in(info, ["dist", "integrity"]), + tarball_url: get_in(info, ["dist", "tarball"]), + size_bytes: get_in(info, ["dist", "unpackedSize"]), + dependencies: Map.get(info, "dependencies", %{}), + peer_dependencies: Map.get(info, "peerDependencies", %{}), + published_at: parse_time(Map.get(times, version)), + deprecated: deprecated_marker?(info) + } + end) + end + + @doc """ + Is the given `version` in `packument` flagged as deprecated? + """ + @spec deprecated?(map(), String.t()) :: boolean() + def deprecated?(packument, version) do + deprecated_marker?(get_in(packument, ["versions", version]) || %{}) + end + + @doc """ + Normalise the packument's `repository` field to a plain URL string. + """ + @spec repository_url(term()) :: String.t() | nil + def repository_url(%{"url" => url}) when is_binary(url), do: url + def repository_url(url) when is_binary(url), do: url + def repository_url(_), do: nil + + defp extract_listing_entry(%{ + "package" => %{"name" => name, "version" => version} + }) + when is_binary(name) and is_binary(version) do + if String.starts_with?(name, @language_prefix) do + %{name: name, latest_version: version} + end + end + + defp extract_listing_entry(_), do: nil + + defp deprecated_marker?(%{"deprecated" => v}) when is_binary(v) and v != "", + do: true + + defp deprecated_marker?(%{"deprecated" => true}), do: true + defp deprecated_marker?(_), do: false + + defp parse_time(time) when is_binary(time) do + case DateTime.from_iso8601(time) do + {:ok, dt, _offset} -> dt + _ -> nil + end + end + + defp parse_time(_), do: nil + + defp json_client do + build_client([ + {Tesla.Middleware.BaseUrl, registry_url()}, + Tesla.Middleware.JSON, + Tesla.Middleware.FollowRedirects + ]) + end + + defp build_client(middleware) do + case Application.get_env(:tesla, :adapter) do + {Tesla.Adapter.Finch, _opts} -> + Tesla.client( + middleware, + {Tesla.Adapter.Finch, + name: Lightning.Finch, receive_timeout: http_timeout()} + ) + + _other -> + Tesla.client(middleware) + end + end + + defp registry_url do + Config.strategy_opts(Lightning.Adaptors.NPM)[:registry_url] || + @default_registry_url + end + + defp http_timeout do + Config.strategy_opts(Lightning.Adaptors.NPM)[:http_timeout] || + @default_http_timeout + end +end diff --git a/lib/lightning/adaptors/npm/schema.ex b/lib/lightning/adaptors/npm/schema.ex new file mode 100644 index 00000000000..8975888e559 --- /dev/null +++ b/lib/lightning/adaptors/npm/schema.ex @@ -0,0 +1,83 @@ +defmodule Lightning.Adaptors.NPM.Schema do + @moduledoc """ + jsDelivr CDN client for adaptor configuration schemas. + + Fetches `/npm/@/configuration-schema.json` from + `cdn.jsdelivr.net`, decodes it, and returns + `{schema_data, schema_sha256}` (or `{nil, nil}` on any failure — + schema fetch is best-effort). + + Base URL via `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)[:jsdelivr_url]`, + default `https://cdn.jsdelivr.net`. + """ + + alias Lightning.Adaptors.Config + + @default_jsdelivr_url "https://cdn.jsdelivr.net" + @default_http_timeout :timer.seconds(30) + + @doc """ + Fetch the configuration schema for `name@version` from jsDelivr. + + Returns `{schema_data, schema_sha256}` on success, `{nil, nil}` on + any failure (best-effort — schema absence must not fail the adaptor + record assembly). + """ + @spec schema(String.t(), String.t()) :: + {map() | nil, String.t() | nil} + def schema(name, version) do + with {:ok, body} <- fetch_schema_bytes(name, version), + {:ok, data} <- Jason.decode(body) do + sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) + {data, sha} + else + _ -> {nil, nil} + end + end + + defp fetch_schema_bytes(name, version) do + url = "/npm/#{name}@#{version}/configuration-schema.json" + + case Tesla.get(jsdelivr_client(), url) do + {:ok, %Tesla.Env{status: 200, body: body}} when is_binary(body) -> + {:ok, body} + + {:ok, %Tesla.Env{status: status}} -> + {:error, {:http_status, status}} + + {:error, reason} -> + {:error, reason} + end + end + + defp jsdelivr_client do + build_client([ + {Tesla.Middleware.BaseUrl, jsdelivr_url()}, + Tesla.Middleware.FollowRedirects + ]) + end + + defp build_client(middleware) do + case Application.get_env(:tesla, :adapter) do + {Tesla.Adapter.Finch, _opts} -> + Tesla.client( + middleware, + {Tesla.Adapter.Finch, + name: Lightning.Finch, receive_timeout: http_timeout()} + ) + + _other -> + Tesla.client(middleware) + end + end + + defp jsdelivr_url do + Config.strategy_opts(Lightning.Adaptors.NPM)[:jsdelivr_url] || + @default_jsdelivr_url + end + + defp http_timeout do + Config.strategy_opts(Lightning.Adaptors.NPM)[:http_timeout] || + @default_http_timeout + end +end diff --git a/lib/lightning/adaptors/repo.ex b/lib/lightning/adaptors/repo.ex new file mode 100644 index 00000000000..f15f2ad690d --- /dev/null +++ b/lib/lightning/adaptors/repo.ex @@ -0,0 +1,339 @@ +defmodule Lightning.Adaptors.Repo do + @moduledoc """ + Query and write helpers over the `adaptors` and `adaptor_versions` tables. + + Despite the name, this is **not** an `Ecto.Repo` — it is a thin + data-access module that wraps `Lightning.Repo` (the real + `Ecto.Repo`). The two schemas it targets live as siblings: + `Lightning.Adaptors.Repo.Adaptor` and + `Lightning.Adaptors.Repo.AdaptorVersion`. + + Every read helper takes the desired `:source` (`:npm | :local`) + explicitly; the module itself stays source-agnostic. Callers resolve + the active source via `Lightning.Adaptors.Config.current_source/0` + (see §4.4 source-tagging invariant and §6.4 in + `.context/lightning/adaptors/REWRITE-2026-05.md`). + + `upsert_adaptor/1` is the only writer the Scheduler uses. It is + idempotent, transactional, and diff-aware: `checked_at` advances on + every call, while `updated_at` only advances when the row's + meaningful fields differ from what was already in the DB. Version + rows are replaced inside the same transaction so a partial failure + cannot leave the table half-rewritten. + """ + + import Ecto.Query + + alias Ecto.Multi + alias Lightning.Adaptors.Repo.Adaptor + alias Lightning.Adaptors.Repo.AdaptorVersion + + @type source :: :npm | :local + + @type package_meta :: %{ + name: String.t(), + latest_version: String.t(), + description: String.t() | nil, + deprecated: boolean(), + updated_at: DateTime.t(), + icon_square_ext: String.t() | nil, + icon_rectangle_ext: String.t() | nil, + icon_square_sha256: binary() | nil, + icon_rectangle_sha256: binary() | nil + } + + @version_row_fields ~w(adaptor_id version integrity tarball_url + size_bytes dependencies peer_dependencies + published_at deprecated)a + + @doc """ + Picker-facing lean projection for a source. Avoids the heavy JSONB + columns (`schema_data`, `dependencies`, `peer_dependencies`). + """ + @spec list_package_metas(source()) :: [package_meta()] + def list_package_metas(source) do + Lightning.Repo.all( + from a in Adaptor, + where: a.source == ^source, + select: %{ + name: a.name, + latest_version: a.latest_version, + description: a.description, + deprecated: a.deprecated, + updated_at: a.updated_at, + icon_square_ext: a.icon_square_ext, + icon_rectangle_ext: a.icon_rectangle_ext, + icon_square_sha256: a.icon_square_sha256, + icon_rectangle_sha256: a.icon_rectangle_sha256 + } + ) + end + + @doc """ + Full structs for a source. Rare — used by debug tools and admin + views. Picker traffic goes through `list_package_metas/1`. + """ + @spec list_adaptors(source()) :: [Adaptor.t()] + def list_adaptors(source) do + Lightning.Repo.all(from a in Adaptor, where: a.source == ^source) + end + + @doc """ + Fetch a single adaptor by `name` within a `source`. Returns `nil` + when no row matches. + """ + @spec get_adaptor(String.t(), source()) :: Adaptor.t() | nil + def get_adaptor(name, source) do + Lightning.Repo.get_by(Adaptor, name: name, source: source) + end + + @doc """ + All versions of an adaptor (`name`, `source`), in insertion order. + """ + @spec list_versions(String.t(), source()) :: [AdaptorVersion.t()] + def list_versions(name, source) do + Lightning.Repo.all( + from v in AdaptorVersion, + join: a in Adaptor, + on: v.adaptor_id == a.id, + where: a.name == ^name and a.source == ^source, + order_by: [asc: v.inserted_at] + ) + end + + @doc """ + Idempotent, transactional, diff-aware upsert of one adaptor record + plus its version rows. The `:source` is read from the record. + + Behaviour: + + * On every call, `checked_at` is advanced to "now". + * `updated_at` only advances when at least one non-`checked_at` + field of the adaptor row actually differs from the existing row. + * Version rows are replaced (delete + insert) inside the same + transaction. + * Every row is run through its schema changeset before write, so a + corrupt Strategy response cannot poison the DB. + + Raises if the underlying transaction fails (e.g. invalid input from + a misbehaving strategy) — the success type is the only contract the + Scheduler relies on. + """ + @spec upsert_adaptor(map()) :: {:ok, Adaptor.t()} + def upsert_adaptor(record) when is_map(record) do + now = DateTime.utc_now() + + {versions, adaptor_attrs} = + record + |> Map.put(:checked_at, now) + |> Map.pop(:versions, []) + + name = Map.fetch!(adaptor_attrs, :name) + source = Map.fetch!(adaptor_attrs, :source) + + multi = + Multi.new() + |> Multi.run(:existing, fn repo, _ -> + {:ok, repo.get_by(Adaptor, name: name, source: source)} + end) + |> Multi.run(:adaptor, fn repo, %{existing: existing} -> + upsert_adaptor_row(repo, existing, adaptor_attrs, now) + end) + |> Multi.run(:delete_versions, fn repo, %{adaptor: adaptor} -> + {count, _} = + repo.delete_all( + from v in AdaptorVersion, where: v.adaptor_id == ^adaptor.id + ) + + {:ok, count} + end) + |> Multi.run(:insert_versions, fn repo, %{adaptor: adaptor} -> + insert_version_rows(repo, adaptor.id, versions, now) + end) + + case Lightning.Repo.transaction(multi) do + {:ok, %{adaptor: adaptor}} -> + {:ok, adaptor} + + {:error, step, reason, _changes} -> + raise ArgumentError, + "Lightning.Adaptors.Repo.upsert_adaptor/1 failed at #{inspect(step)}: " <> + inspect(reason) + end + end + + @doc """ + Advance `checked_at` for a known `(name, source)` row without + loading it. No-op when no row matches. + + Used by the Scheduler's "polled NPM, nothing changed" path — + cheaper than a full upsert and never bumps `updated_at`. + """ + @spec touch_checked_at(String.t(), source()) :: :ok + def touch_checked_at(name, source) do + now = DateTime.utc_now() + + Lightning.Repo.update_all( + from(a in Adaptor, where: a.name == ^name and a.source == ^source), + set: [checked_at: now] + ) + + :ok + end + + @doc """ + The `limit` rows for a given `source` whose `checked_at` is oldest + first. Backs the Scheduler's per-tick work list. + """ + @spec stalest(pos_integer(), source()) :: [Adaptor.t()] + def stalest(limit, source) when is_integer(limit) and limit > 0 do + Lightning.Repo.all( + from a in Adaptor, + where: a.source == ^source, + order_by: [asc: a.checked_at], + limit: ^limit + ) + end + + @doc """ + Lean list of source-scoped adaptors that are missing at least one icon + shape. Returns only the fields the Scheduler needs to decide whether to + re-apply the bulk icon fetch result. + """ + @spec list_missing_icons(source()) :: [ + %{ + name: String.t(), + icon_square_sha256: binary() | nil, + icon_rectangle_sha256: binary() | nil + } + ] + def list_missing_icons(source) do + Lightning.Repo.all( + from a in Adaptor, + where: + a.source == ^source and + (is_nil(a.icon_square_sha256) or is_nil(a.icon_rectangle_sha256)), + select: %{ + name: a.name, + icon_square_sha256: a.icon_square_sha256, + icon_rectangle_sha256: a.icon_rectangle_sha256 + } + ) + end + + @doc """ + Update only the icon columns for a single `(name, source)` row. + + `attrs` may include any subset of `:icon_square_ext`, + `:icon_square_sha256`, `:icon_rectangle_ext`, `:icon_rectangle_sha256`, + `:icon_square_etag`, `:icon_rectangle_etag`. + `updated_at` is advanced so callers can observe the change. + + Sidesteps `upsert_adaptor/1` deliberately: that helper rewrites the + `adaptor_versions` rows in the same transaction, which is the wrong + thing to do for an icon-only fix-up. + """ + @spec update_icons(String.t(), source(), map()) :: {integer(), nil} + def update_icons(name, source, attrs) when is_map(attrs) do + allowed = + attrs + |> Map.take([ + :icon_square_ext, + :icon_square_sha256, + :icon_rectangle_ext, + :icon_rectangle_sha256, + :icon_square_etag, + :icon_rectangle_etag + ]) + |> Map.put(:updated_at, DateTime.utc_now()) + |> Enum.into([]) + + Lightning.Repo.update_all( + from(a in Adaptor, where: a.name == ^name and a.source == ^source), + set: allowed + ) + end + + @doc """ + Maximum `checked_at` seen for `source`, or `nil` when the table is + empty for that source. Backs the Scheduler's smart-init timing. + """ + @spec max_checked_at(source()) :: DateTime.t() | nil + def max_checked_at(source) do + Lightning.Repo.one( + from a in Adaptor, + where: a.source == ^source, + select: max(a.checked_at) + ) + end + + defp upsert_adaptor_row(repo, nil, attrs, _now) do + %Adaptor{} + |> Adaptor.changeset(attrs) + |> repo.insert() + end + + defp upsert_adaptor_row(repo, %Adaptor{} = existing, attrs, now) do + changeset = Adaptor.changeset(existing, attrs) + + # `Ecto.Changeset.cast/3` only records a change when the cast value + # differs from the underlying struct, so the set of "real" changes + # is `:changes` minus the `:checked_at` tick we apply on every call. + meaningful_changes? = + changeset.changes + |> Map.delete(:checked_at) + |> map_size() > 0 + + if meaningful_changes? do + repo.update(changeset) + else + {1, _} = + repo.update_all( + from(a in Adaptor, where: a.id == ^existing.id), + set: [checked_at: now] + ) + + {:ok, %{existing | checked_at: now}} + end + end + + defp insert_version_rows(_repo, _adaptor_id, [], _now), do: {:ok, 0} + + defp insert_version_rows(repo, adaptor_id, records, now) do + case build_version_rows(adaptor_id, records, now) do + {:ok, rows} -> + {count, _} = repo.insert_all(AdaptorVersion, rows) + {:ok, count} + + {:error, changeset} -> + {:error, changeset} + end + end + + defp build_version_rows(adaptor_id, records, now) do + records + |> Enum.reduce_while({:ok, []}, fn record, {:ok, acc} -> + attrs = Map.put(record, :adaptor_id, adaptor_id) + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) + + if changeset.valid? do + {:cont, {:ok, [version_row_from_changeset(changeset, now) | acc]}} + else + {:halt, {:error, changeset}} + end + end) + |> case do + {:ok, rows} -> {:ok, Enum.reverse(rows)} + err -> err + end + end + + defp version_row_from_changeset(changeset, now) do + changeset + |> Ecto.Changeset.apply_changes() + |> Map.from_struct() + |> Map.take(@version_row_fields) + |> Map.put(:id, Ecto.UUID.generate()) + |> Map.put(:inserted_at, now) + end +end diff --git a/lib/lightning/adaptors/repo_adaptor.ex b/lib/lightning/adaptors/repo_adaptor.ex new file mode 100644 index 00000000000..469620df5fe --- /dev/null +++ b/lib/lightning/adaptors/repo_adaptor.ex @@ -0,0 +1,163 @@ +defmodule Lightning.Adaptors.Repo.Adaptor do + @moduledoc """ + Ecto schema for one row of the `adaptors` table — the per-package + metadata projection used by the picker and Scheduler. + + Source-tagged via `:source` (`:npm | :local`) so the same package + name can coexist across sources; the unique index is `[:name, :source]` + (see §4.4 source-tagging invariant in + `.context/lightning/adaptors/REWRITE-2026-05.md`). + + Mirrors `Lightning.Adaptors.Strategy.adaptor_record` minus `:versions`, + which lives on `Lightning.Adaptors.Repo.AdaptorVersion`. + """ + + use Ecto.Schema + + import Ecto.Changeset + + defmodule JSONBinary do + @moduledoc """ + Ecto type backing `schema_data` with a text column while preserving + JSON field order on read. + + Storage is a JSON binary in a `text` column. Inputs may be either a + binary or a map — maps are encoded with `Jason.encode!/1` at the + dumper to keep `Lightning.Factories.adaptor/2` and other direct + struct inserts compatible without forcing every caller to encode + up-front. Loads always return a binary so credential-form rendering + can re-engage `Jason.decode!(_, objects: :ordered_objects)`. + """ + + use Ecto.Type + + @impl true + def type, do: :string + + @impl true + def cast(nil), do: {:ok, nil} + def cast(value) when is_binary(value), do: {:ok, value} + def cast(value) when is_map(value), do: {:ok, Jason.encode!(value)} + def cast(_), do: :error + + @impl true + def load(nil), do: {:ok, nil} + def load(value) when is_binary(value), do: {:ok, value} + + @impl true + def dump(nil), do: {:ok, nil} + def dump(value) when is_binary(value), do: {:ok, value} + def dump(value) when is_map(value), do: {:ok, Jason.encode!(value)} + def dump(_), do: :error + end + + @type t :: %__MODULE__{ + id: Ecto.UUID.t() | nil, + name: String.t() | nil, + source: :npm | :local | nil, + description: String.t() | nil, + homepage: String.t() | nil, + repository: String.t() | nil, + license: String.t() | nil, + latest_version: String.t() | nil, + deprecated: boolean(), + schema_data: String.t() | nil, + schema_sha256: String.t() | nil, + icon_square_ext: String.t() | nil, + icon_rectangle_ext: String.t() | nil, + icon_square_sha256: binary() | nil, + icon_rectangle_sha256: binary() | nil, + icon_square_etag: String.t() | nil, + icon_rectangle_etag: String.t() | nil, + checked_at: DateTime.t() | nil, + inserted_at: DateTime.t() | nil, + updated_at: DateTime.t() | nil + } + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + @timestamps_opts [type: :utc_datetime_usec] + + schema "adaptors" do + field :name, :string + field :source, Ecto.Enum, values: [:npm, :local] + field :description, :string + field :homepage, :string + field :repository, :string + field :license, :string + field :latest_version, :string + field :deprecated, :boolean, default: false + field :schema_data, Lightning.Adaptors.Repo.Adaptor.JSONBinary + field :schema_sha256, :string + field :icon_square_ext, :string + field :icon_rectangle_ext, :string + field :icon_square_sha256, :binary + field :icon_rectangle_sha256, :binary + field :icon_square_etag, :string + field :icon_rectangle_etag, :string + field :checked_at, :utc_datetime_usec + + timestamps() + end + + @required ~w(name source latest_version checked_at)a + @optional ~w(description homepage repository license deprecated + schema_data schema_sha256 + icon_square_ext icon_rectangle_ext + icon_square_sha256 icon_rectangle_sha256 + icon_square_etag icon_rectangle_etag)a + + @doc """ + Build a changeset for upserting a single adaptor row. + + This is the single clause used by every write path on + `Lightning.Adaptors.Repo` — there is no separate update path because + the writer always rewrites the full row. + """ + @spec changeset(t(), map()) :: Ecto.Changeset.t() + def changeset(struct, attrs) do + struct + |> cast(attrs, @required ++ @optional) + |> validate_required(@required) + |> validate_length(:name, max: 214) + |> validate_inclusion(:icon_square_ext, ~w(png svg)) + |> validate_inclusion(:icon_rectangle_ext, ~w(png svg)) + |> validate_icon_sha256_pair(:icon_square) + |> validate_icon_sha256_pair(:icon_rectangle) + |> unique_constraint([:name, :source]) + end + + # Enforces the §6.4 invariant: a non-nil `icon__ext` requires a + # non-nil `icon__sha256`, and vice versa. Either both fields + # are set or both are nil — half-populated pairs fail the changeset. + @spec validate_icon_sha256_pair( + Ecto.Changeset.t(), + :icon_square | :icon_rectangle + ) :: Ecto.Changeset.t() + defp validate_icon_sha256_pair(changeset, shape) do + ext_field = :"#{shape}_ext" + sha_field = :"#{shape}_sha256" + + case {get_field(changeset, ext_field), get_field(changeset, sha_field)} do + {nil, nil} -> + changeset + + {nil, _sha} -> + add_error( + changeset, + sha_field, + "must be nil when #{ext_field} is nil" + ) + + {_ext, nil} -> + add_error( + changeset, + sha_field, + "must not be nil when #{ext_field} is set" + ) + + {_ext, _sha} -> + changeset + end + end +end diff --git a/lib/lightning/adaptors/repo_adaptor_version.ex b/lib/lightning/adaptors/repo_adaptor_version.ex new file mode 100644 index 00000000000..ec048b81806 --- /dev/null +++ b/lib/lightning/adaptors/repo_adaptor_version.ex @@ -0,0 +1,73 @@ +defmodule Lightning.Adaptors.Repo.AdaptorVersion do + @moduledoc """ + Ecto schema for one row of the `adaptor_versions` table — per-version + metadata for an adaptor package (`integrity`, `tarball_url`, + `size_bytes`, `dependencies`, `peer_dependencies`, `published_at`, + `deprecated`). + + Belongs to `Lightning.Adaptors.Repo.Adaptor` and cascade-deletes with + its parent. Mirrors `Lightning.Adaptors.Strategy.version_record` (see + §6.1 and §6.4 in `.context/lightning/adaptors/REWRITE-2026-05.md`). + """ + + use Ecto.Schema + + import Ecto.Changeset + + alias Lightning.Adaptors.Repo.Adaptor + + @type t :: %__MODULE__{ + id: Ecto.UUID.t() | nil, + adaptor_id: Ecto.UUID.t() | nil, + adaptor: Adaptor.t() | Ecto.Association.NotLoaded.t() | nil, + version: String.t() | nil, + integrity: String.t() | nil, + tarball_url: String.t() | nil, + size_bytes: integer() | nil, + dependencies: map() | nil, + peer_dependencies: map() | nil, + published_at: DateTime.t() | nil, + deprecated: boolean(), + inserted_at: DateTime.t() | nil + } + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + @timestamps_opts [type: :utc_datetime_usec] + + schema "adaptor_versions" do + field :version, :string + field :integrity, :string + field :tarball_url, :string + field :size_bytes, :integer + field :dependencies, :map + field :peer_dependencies, :map + field :published_at, :utc_datetime_usec + field :deprecated, :boolean, default: false + + belongs_to :adaptor, Adaptor + + timestamps(updated_at: false) + end + + @required ~w(adaptor_id version)a + @optional ~w(integrity tarball_url size_bytes + dependencies peer_dependencies + published_at deprecated)a + + @doc """ + Build a changeset for inserting an `adaptor_versions` row. + + `Lightning.Adaptors.Repo.upsert_adaptor/1` replaces version rows with + a delete-then-insert inside a transaction, so there is no separate + update path. + """ + @spec changeset(t(), map()) :: Ecto.Changeset.t() + def changeset(struct, attrs) do + struct + |> cast(attrs, @required ++ @optional) + |> validate_required(@required) + |> unique_constraint([:adaptor_id, :version]) + |> assoc_constraint(:adaptor) + end +end diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex new file mode 100644 index 00000000000..6077fc48d2a --- /dev/null +++ b/lib/lightning/adaptors/scheduler.ex @@ -0,0 +1,597 @@ +defmodule Lightning.Adaptors.Scheduler do + @moduledoc """ + Cluster-singleton GenServer that periodically refreshes the active + source's ledger via the configured strategy, persists through + `Lightning.Adaptors.Repo`, and broadcasts `{:changed, name, source}`. + + Wrapped by `HighlanderPG` so only one node in the cluster runs the + Scheduler at a time. The inner GenServer registers under + `{:global, Lightning.Adaptors.Supervisor.global_scheduler_name(name)}`, + so callers on any node reach the leader transparently via Erlang + distribution. Peer nodes react to refreshes via + `Lightning.Adaptors.Invalidator` and + `Lightning.Adaptors.ChannelBroadcaster`. + + Smart-init timing: the first tick is scheduled at + `max(0, last_checked_at + interval - now)` to avoid double-refreshing + shortly after a deploy. An empty table or an overdue schedule fires + immediately (`delay = 0`). Interval `0` disables scheduling entirely. + + ## Two-pipeline refresh + + A tick runs two parallel pipelines under the per-instance + `Task.Supervisor`: + + * **Pipeline A** — `strategy.fetch_icons/1` for every adaptor. + * **Pipeline B** — `strategy.list_adaptors/0` followed by a bounded + per-adaptor fan-out (`async_stream_nolink`) calling + `strategy.fetch_adaptor/1` only for names whose `latest_version` + changed since the last tick. + + Once both pipelines complete the join step merges the icons map into + each fetched record, writes the icon bytes to disk via + `Lightning.Adaptors.IconCache.write!/5`, and upserts each adaptor in + one go. `refresh_package/2` deliberately bypasses the icon pipeline — + on-demand single-package refreshes do not refetch icons. + """ + + use GenServer + + alias Lightning.Adaptors.Config + alias Lightning.Adaptors.IconCache + alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + require Logger + + @fetch_max_concurrency 8 + @icons_task_timeout :timer.seconds(60) + + @doc """ + Start the Scheduler for the given supervisor instance. + + Required opts: `:name`, `:sup`, `:lock_key`, `:cache`, `:tasks`, + `:source_topic`. + """ + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts) do + name = Keyword.fetch!(opts, :name) + _ = Keyword.fetch!(opts, :sup) + _ = Keyword.fetch!(opts, :lock_key) + _ = Keyword.fetch!(opts, :cache) + _ = Keyword.fetch!(opts, :tasks) + _ = Keyword.fetch!(opts, :source_topic) + GenServer.start_link(__MODULE__, opts, name: name) + end + + @doc """ + Trigger an immediate refresh tick. + + Routes via `:global` to the leader-held GenServer. + """ + @spec refresh_now(GenServer.server()) :: :ok | {:error, term()} + def refresh_now(scheduler_name) do + GenServer.call(scheduler_name, :refresh_now) + end + + @doc """ + Force a single-adaptor refresh, bypassing the diff. 30-second timeout. + + Returns `{:error, :not_found}` or `{:error, term()}` from a failed + strategy fetch. + """ + @spec refresh_package(GenServer.server(), String.t()) :: + :ok | {:error, :not_found | term()} + def refresh_package(scheduler_name, name) do + GenServer.call(scheduler_name, {:refresh_package, name}, 30_000) + end + + @doc """ + Refresh icons only, against every source-scoped adaptor row. + + Runs `strategy.fetch_icons/1` and re-applies any shape whose `sha256` + differs from what is on the row. Adaptor metadata and version rows + are not touched. Returns `{:ok, %{updated: n, unchanged: m}}` on + success or `{:error, reason}` if the bulk fetch fails. + """ + @spec refresh_icons(GenServer.server()) :: + {:ok, %{updated: non_neg_integer(), unchanged: non_neg_integer()}} + | {:error, term()} + def refresh_icons(scheduler_name) do + GenServer.call(scheduler_name, :refresh_icons, 120_000) + end + + @impl true + def init(opts) do + sup = Keyword.fetch!(opts, :sup) + source_topic = Keyword.fetch!(opts, :source_topic) + cache = Keyword.fetch!(opts, :cache) + tasks = Keyword.fetch!(opts, :tasks) + + source = AdaptorsSupervisor.source(sup) + interval_ms = Config.refresh_interval() + + if interval_ms > 0 do + delay = + time_until_next_ms(AdaptorsRepo.max_checked_at(source), interval_ms) + + Process.send_after(self(), :tick, delay) + + Logger.info( + "Adaptors[#{source}]: scheduler started interval=#{interval_ms}ms next_tick_in=#{delay}ms" + ) + else + Logger.info("Adaptors[#{source}]: scheduler started interval=0 (disabled)") + end + + {:ok, + %{ + sup: sup, + source: source, + interval_ms: interval_ms, + source_topic: source_topic, + cache: cache, + tasks: tasks + }} + end + + @impl true + def handle_info(:tick, state) do + if state.interval_ms > 0 do + Process.send_after(self(), :tick, state.interval_ms) + end + + Task.Supervisor.start_child(state.tasks, fn -> do_refresh(state) end) + + {:noreply, state} + end + + @impl true + def handle_call(:refresh_now, _from, state) do + Logger.info("Adaptors[#{state.source}]: refresh_now requested") + send(self(), :tick) + {:reply, :ok, state} + end + + def handle_call({:refresh_package, name}, _from, state) do + Logger.info("Adaptors[#{state.source}]: refresh_package(#{name}) requested") + + strategy = AdaptorsSupervisor.strategy(state.sup) + result = force_refresh_one(strategy, name, state) + {:reply, result, state} + end + + def handle_call(:refresh_icons, _from, state) do + Logger.info("Adaptors[#{state.source}]: refresh_icons requested") + strategy = AdaptorsSupervisor.strategy(state.sup) + + existing = AdaptorsRepo.list_adaptors(state.source) + prior_etags = prior_etags_from_rows(existing) + + case strategy.fetch_icons(prior_etags: prior_etags) do + {:ok, icons} -> + result = reapply_icons(existing, icons, state) + + Logger.info( + "Adaptors[#{state.source}]: refresh_icons done " <> + "rows=#{length(existing)} icons=#{map_size(icons)} " <> + "updated=#{result.updated} unchanged=#{result.unchanged}" + ) + + {:reply, {:ok, result}, state} + + {:error, reason} -> + Logger.warning( + "Adaptors[#{state.source}]: refresh_icons strategy fetch failed: #{inspect(reason)}" + ) + + {:reply, {:error, reason}, state} + end + end + + defp do_refresh(state) do + started_at = System.monotonic_time(:millisecond) + strategy = AdaptorsSupervisor.strategy(state.sup) + + # Single DB round-trip serves both the icons-task input (prior etags) + # and the version diff used below to decide which adaptors to fetch. + existing_rows = AdaptorsRepo.list_adaptors(state.source) + prior_etags = prior_etags_from_rows(existing_rows) + + existing_by_name = + Map.new(existing_rows, fn a -> {a.name, a.latest_version} end) + + icons_task = + Task.Supervisor.async_nolink(state.tasks, fn -> + strategy.fetch_icons(prior_etags: prior_etags) + end) + + case strategy.list_adaptors() do + {:ok, upstream} -> + {fetched, changed, errors} = + state.tasks + |> Task.Supervisor.async_stream_nolink( + upstream, + &fetch_if_changed(strategy, &1, existing_by_name, state), + max_concurrency: @fetch_max_concurrency, + ordered: false, + on_timeout: :kill_task + ) + |> Enum.reduce({[], 0, 0}, fn + {:ok, {:fetched, record}}, {acc, c, e} -> {[record | acc], c + 1, e} + {:ok, :touched}, {acc, c, e} -> {acc, c, e} + {:ok, {:error, _reason}}, {acc, c, e} -> {acc, c, e + 1} + {:exit, _reason}, {acc, c, e} -> {acc, c, e + 1} + end) + + icons = await_icons(icons_task) + + persisted = + fetched + |> Enum.map(fn record -> persist_with_icons(record, icons, state) end) + |> Enum.count(&(&1 == :ok)) + + healed = heal_missing_icons(icons, state) + not_modified = count_not_modified(icons) + + listed = length(upstream) + touched = listed - changed - errors + duration_ms = System.monotonic_time(:millisecond) - started_at + + Logger.info( + "Adaptors[#{state.source}]: refresh tick listed=#{listed} " <> + "changed=#{changed} touched=#{touched} fetched=#{persisted} " <> + "icons=#{map_size(icons)} healed=#{healed} " <> + "not_modified=#{not_modified} " <> + "errors=#{errors} duration=#{duration_ms}ms" + ) + + {:error, reason} -> + Logger.warning("Scheduler: list_adaptors failed: #{inspect(reason)}") + _ = await_icons(icons_task) + duration_ms = System.monotonic_time(:millisecond) - started_at + + Logger.info( + "Adaptors[#{state.source}]: refresh tick listed=0 changed=0 " <> + "touched=0 fetched=0 icons=0 errors=1 duration=#{duration_ms}ms" + ) + + :ok + end + end + + defp fetch_if_changed( + strategy, + %{name: name, latest_version: version}, + existing_by_name, + state + ) do + if Map.get(existing_by_name, name) == version do + AdaptorsRepo.touch_checked_at(name, state.source) + :touched + else + case strategy.fetch_adaptor(name) do + {:ok, record} -> + Logger.debug( + "Adaptors[#{state.source}]: fetched #{name}@#{record.version}" + ) + + {:fetched, record} + + {:error, reason} -> + Logger.warning( + "Scheduler: fetch_adaptor(#{name}) failed: #{inspect(reason)}" + ) + + {:error, reason} + end + end + end + + defp await_icons(task) do + case Task.yield(task, @icons_task_timeout) || Task.shutdown(task) do + {:ok, {:ok, map}} when is_map(map) -> + map + + {:ok, {:error, reason}} -> + Logger.warning( + "Scheduler: fetch_icons failed: #{inspect(reason)} — persisting records without icons" + ) + + %{} + + {:exit, reason} -> + Logger.warning( + "Scheduler: fetch_icons crashed: #{inspect(reason)} — persisting records without icons" + ) + + %{} + + nil -> + Logger.warning( + "Scheduler: fetch_icons timed out — persisting records without icons" + ) + + %{} + end + end + + defp persist_with_icons(record, icons, state) do + name = record.name + package_icons = Map.get(icons, name, %{}) + + record_with_icons = + record + |> Map.put(:source, state.source) + |> merge_icon(:square, package_icons, state.source) + |> merge_icon(:rectangle, package_icons, state.source) + + try do + {:ok, _} = AdaptorsRepo.upsert_adaptor(record_with_icons) + + Phoenix.PubSub.broadcast( + Lightning.PubSub, + state.source_topic, + {:changed, name, state.source} + ) + + Logger.debug("Adaptors[#{state.source}]: persisted #{name}") + :ok + rescue + e -> + Logger.error( + "Scheduler: upsert_adaptor(#{name}) failed: #{Exception.message(e)}" + ) + + :error + end + end + + defp merge_icon(record, shape, package_icons, source) do + case Map.get(package_icons, shape) do + %{data: bytes, ext: ext, sha256: sha} = entry when is_binary(bytes) -> + try do + {:ok, ^sha} = IconCache.write!(source, record.name, shape, ext, bytes) + + record + |> Map.put(:"icon_#{shape}_ext", ext) + |> Map.put(:"icon_#{shape}_sha256", sha) + |> maybe_put_etag(shape, Map.get(entry, :etag)) + rescue + e -> + Logger.warning( + "Scheduler: IconCache.write!(#{record.name}, #{shape}) failed: #{Exception.message(e)}" + ) + + record + end + + :not_modified -> + # Upstream confirmed unchanged — leave row's existing icon and + # etag in place. Counted in the tick summary via + # count_not_modified/1. + record + + _ -> + record + end + end + + # Stamp the etag onto the record only when the strategy supplied one + # (NPM 200 entries always have the key; Local omits it). A nil etag is + # not stamped — we preserve whatever was already on the row. + defp maybe_put_etag(record, _shape, nil), do: record + + defp maybe_put_etag(record, shape, etag) when is_binary(etag) do + Map.put(record, :"icon_#{shape}_etag", etag) + end + + # Top up icons on rows that currently have NULL on at least one shape. + # Runs after the main upsert pass on every tick — cheap, scoped to + # rows with gaps, and self-correcting after a strategy outage or a + # past bug like the one that left every row iconless. + defp heal_missing_icons(icons, _state) when map_size(icons) == 0, do: 0 + + defp heal_missing_icons(icons, state) do + state.source + |> AdaptorsRepo.list_missing_icons() + |> Enum.reduce(0, fn row, acc -> + package_icons = Map.get(icons, row.name, %{}) + + case apply_icons_to_existing(row, package_icons, state) do + :updated -> acc + 1 + :unchanged -> acc + end + end) + end + + defp reapply_icons(existing_rows, icons, state) do + Enum.reduce(existing_rows, %{updated: 0, unchanged: 0}, fn row, acc -> + package_icons = Map.get(icons, row.name, %{}) + + case apply_icons_to_existing(row, package_icons, state) do + :updated -> %{acc | updated: acc.updated + 1} + :unchanged -> %{acc | unchanged: acc.unchanged + 1} + end + end) + end + + # `row` is either an Adaptor struct (from list_adaptors/1) or a lean + # map (from list_missing_icons/1) — both expose :name and the icon + # sha256 fields, which is all we need. + defp apply_icons_to_existing(_row, package_icons, _state) + when map_size(package_icons) == 0, + do: :unchanged + + defp apply_icons_to_existing(row, package_icons, state) do + changes = + [:square, :rectangle] + |> Enum.reduce(%{}, fn shape, acc -> + accumulate_icon_change(acc, shape, row, package_icons, state) + end) + + if map_size(changes) > 0 do + {1, _} = AdaptorsRepo.update_icons(row.name, state.source, changes) + + Phoenix.PubSub.broadcast( + Lightning.PubSub, + state.source_topic, + {:changed, row.name, state.source} + ) + + :updated + else + :unchanged + end + end + + defp accumulate_icon_change(acc, shape, row, package_icons, state) do + sha_key = :"icon_#{shape}_sha256" + ext_key = :"icon_#{shape}_ext" + etag_key = :"icon_#{shape}_etag" + + case Map.get(package_icons, shape) do + %{data: bytes, ext: ext, sha256: sha} = entry when is_binary(bytes) -> + if Map.get(row, sha_key) == sha do + # Same bytes already on disk; the etag may still need + # refreshing if the strategy gave us a new (non-nil) value + # that differs from what we have. nil never clobbers. + maybe_accumulate_etag(acc, etag_key, row, Map.get(entry, :etag)) + else + accumulate_fetched_icon(acc, shape, row, entry, ext, sha, bytes, state, + sha_key: sha_key, + ext_key: ext_key, + etag_key: etag_key + ) + end + + :not_modified -> + # 304 confirmed — nothing to write, etag already current. + acc + + _ -> + acc + end + end + + defp accumulate_fetched_icon(acc, shape, row, entry, ext, sha, bytes, state, + sha_key: sha_key, + ext_key: ext_key, + etag_key: etag_key + ) do + try do + {:ok, ^sha} = IconCache.write!(state.source, row.name, shape, ext, bytes) + + acc + |> Map.put(ext_key, ext) + |> Map.put(sha_key, sha) + |> maybe_accumulate_etag(etag_key, row, Map.get(entry, :etag)) + rescue + e -> + Logger.warning( + "Scheduler: IconCache.write!(#{row.name}, #{shape}) failed: " <> + Exception.message(e) + ) + + acc + end + end + + # nil → preserve existing etag on the row (do not clobber). + # value matching the row's current etag → no-op (avoid no-op write). + # value differing → emit the change. + defp maybe_accumulate_etag(acc, _etag_key, _row, nil), do: acc + + defp maybe_accumulate_etag(acc, etag_key, row, etag) when is_binary(etag) do + if Map.get(row, etag_key) == etag do + acc + else + Map.put(acc, etag_key, etag) + end + end + + defp force_refresh_one(strategy, name, state) do + case strategy.fetch_adaptor(name) do + {:ok, record} -> + record_with_source = Map.put(record, :source, state.source) + + try do + {:ok, _} = AdaptorsRepo.upsert_adaptor(record_with_source) + + Phoenix.PubSub.broadcast( + Lightning.PubSub, + state.source_topic, + {:changed, name, state.source} + ) + + Logger.info( + "Adaptors[#{state.source}]: refresh_package(#{name}) ok version=#{record.version}" + ) + + :ok + rescue + e -> + Logger.error( + "Scheduler: upsert_adaptor(#{name}) failed: #{Exception.message(e)}" + ) + + {:error, {:upsert_failed, Exception.message(e)}} + end + + {:error, reason} -> + Logger.warning( + "Scheduler: refresh_package(#{name}) strategy fetch failed: #{inspect(reason)}" + ) + + {:error, reason} + end + end + + # Project a list of adaptor rows to the prior-etag map shape expected + # by `Strategy.fetch_icons/1`: `%{name => %{shape => etag}}`. Rows + # whose etags are both nil are skipped entirely (no empty inner map); + # within a row, only shapes with a non-nil etag are kept. The consumer + # treats absence as "no prior etag, send no If-None-Match", so an empty + # entry would be wasteful but harmless — we drop it for clarity. + @spec prior_etags_from_rows([map()]) :: %{ + String.t() => %{optional(:square | :rectangle) => String.t()} + } + defp prior_etags_from_rows(rows) do + Enum.reduce(rows, %{}, fn row, acc -> + inner = + %{} + |> maybe_put_shape_etag(:square, Map.get(row, :icon_square_etag)) + |> maybe_put_shape_etag(:rectangle, Map.get(row, :icon_rectangle_etag)) + + if map_size(inner) == 0 do + acc + else + Map.put(acc, row.name, inner) + end + end) + end + + defp maybe_put_shape_etag(map, _shape, nil), do: map + + defp maybe_put_shape_etag(map, shape, etag) when is_binary(etag), + do: Map.put(map, shape, etag) + + # Count :not_modified sentinels across all shapes in the icons map. + # Used in the tick summary log. + defp count_not_modified(icons) do + Enum.reduce(icons, 0, fn {_name, shapes}, acc -> + Enum.reduce(shapes, acc, fn + {_shape, :not_modified}, n -> n + 1 + {_shape, _}, n -> n + end) + end) + end + + defp time_until_next_ms(nil, _interval_ms), do: 0 + + defp time_until_next_ms(%DateTime{} = last, interval_ms) do + next = DateTime.add(last, interval_ms, :millisecond) + diff = DateTime.diff(next, DateTime.utc_now(), :millisecond) + max(0, diff) + end +end diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex new file mode 100644 index 00000000000..a1d23c781d9 --- /dev/null +++ b/lib/lightning/adaptors/store.ex @@ -0,0 +1,347 @@ +defmodule Lightning.Adaptors.Store do + @moduledoc """ + Stateless read facade over `Cachex`, `Lightning.Adaptors.Repo`, and the + active `Lightning.Adaptors.Strategy`. + + Every public read helper wraps a `Cachex.fetch/4` whose fallback first + consults the local Postgres projection (`Lightning.Adaptors.Repo`) and + only invokes the Strategy as a last resort. Cachex's courier supplies + blocking semantics and per-key coalescing of concurrent first-callers + for free — there is no GenServer mailbox in front of the reads. + + ## Source tagging + + Each cache key carries the active `:source` (`:npm | :local`) read via + `Lightning.Adaptors.Supervisor.source/1`, so the same package name can + coexist across deployment modes without manual scrubbing (see §4.4 of + `.context/adaptors/REWRITE-2026-05.md`). + + ## Commit vs ignore + + Successful Strategy/Repo lookups commit their projected value to the + cache. Failures — empty `packages/1` results, unknown adaptors for + `icon_meta/2`, Strategy errors — return `:ignore`, so a subsequent + caller retries fresh rather than seeing a poisoned cache entry. + + ## Icons + + `icon/3` returns a `Path.t/0` the controller serves via `send_file/3` + — no binary on the BEAM heap. The on-disk `Lightning.Adaptors.IconCache` + is the primary cache: a `cached?/4` hit short-circuits before Cachex + is touched. On a disk miss the lazy Strategy fetch is wrapped in + `Cachex.fetch/4` on `{:icon_bytes, source, name, shape}` so that + concurrent first-callers coalesce onto a single courier; the courier + returns `{:ignore, _}` so no entry is committed, and subsequent + callers re-read the now-populated file from disk. + """ + + alias Lightning.Adaptors.Config + alias Lightning.Adaptors.IconCache + alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + # Strategy and source are scoped to the supervisor instance — every + # `Store` call resolves both from the per-instance `:persistent_term` + # entry the supervisor populated at boot. No `Application.get_env` + # reads in the hot path; no global mutable state in tests. + + @type sup :: atom() + + @type version_meta :: %{ + version: String.t(), + integrity: String.t() | nil, + size_bytes: integer() | nil, + published_at: DateTime.t() | nil, + deprecated: boolean() + } + + @type icon_meta :: %{ + icon_square_ext: String.t() | nil, + icon_rectangle_ext: String.t() | nil, + icon_square_sha256: binary() | nil, + icon_rectangle_sha256: binary() | nil + } + + @type package_meta :: AdaptorsRepo.package_meta() + + @doc """ + Read the `schema_data` JSON blob for a single adaptor. + + Cache-then-Repo-then-Strategy. On Strategy success the full adaptor + record is upserted into Postgres and the projected schema blob is + committed to the cache. + + Returns the schema as a JSON binary so that ordered-objects decoding + can re-engage at the credential-form renderer. + """ + @spec schema(sup(), String.t()) :: {:ok, String.t()} | {:error, term()} + def schema(sup, name) do + cache = AdaptorsSupervisor.cache_name(sup) + source = AdaptorsSupervisor.source(sup) + + cache + |> Cachex.fetch( + {:schema, name, source}, + fn _key -> + case AdaptorsRepo.get_adaptor(name, source) do + %{schema_data: data} when not is_nil(data) -> + {:commit, {:ok, data}} + + _ -> + fetch_and_persist(sup, name, source, :schema_data) + end + end, + timeout: Config.cache_timeout_ms() + ) + |> unwrap() + end + + @doc """ + Read the version history for a single adaptor as a list of lean + per-version maps. See `t:version_meta/0` for the projected shape. + """ + @spec versions(sup(), String.t()) :: + {:ok, [version_meta()]} | {:error, term()} + def versions(sup, name) do + cache = AdaptorsSupervisor.cache_name(sup) + source = AdaptorsSupervisor.source(sup) + + cache + |> Cachex.fetch( + {:versions, name, source}, + fn _key -> + case AdaptorsRepo.list_versions(name, source) do + [] -> fetch_and_persist(sup, name, source, :versions) + rows -> {:commit, {:ok, project_versions(rows)}} + end + end, + timeout: Config.cache_timeout_ms() + ) + |> unwrap() + end + + @doc """ + Resolve the on-disk path of one icon variant for an adaptor. + + Disk is the cache: a cache-hit on `IconCache.cached?/4` returns the + path immediately. A cache-miss is routed through `Cachex.fetch/4` on + `{:icon_bytes, source, name, shape}` so concurrent first-callers + coalesce onto one in-flight Strategy fetch — the courier returns + `{:ignore, _}` so no cache entry is committed and the next miss reads + the freshly-written file from disk. + + Returns `{:error, :not_found}` when the icon variant is absent from + the adaptor row (the row is the source of truth). + """ + @spec icon(sup(), String.t(), :square | :rectangle) :: + {:ok, Path.t()} | {:error, :not_found | term()} + def icon(sup, name, shape) when shape in [:square, :rectangle] do + cache = AdaptorsSupervisor.cache_name(sup) + source = AdaptorsSupervisor.source(sup) + strategy = AdaptorsSupervisor.strategy(sup) + + with {:ok, meta} <- icon_meta(sup, name), + {:ok, ext} <- ext_for_shape(meta, shape), + {:ok, _sha256} <- sha256_for_shape(meta, shape) do + if IconCache.cached?(source, name, shape, ext) do + {:ok, IconCache.path(source, name, shape, ext)} + else + cache + |> Cachex.fetch( + {:icon_bytes, source, name, shape}, + fn _key -> fetch_icon_bytes(strategy, source, name, shape, ext) end, + timeout: Config.cache_timeout_ms() + ) + |> unwrap() + end + end + end + + defp fetch_icon_bytes(strategy, source, name, shape, ext) do + case strategy.fetch_icon(name, shape) do + {:ok, %{data: bytes, ext: ^ext}} -> + {:ok, _sha} = IconCache.write!(source, name, shape, ext, bytes) + {:ignore, {:ok, IconCache.path(source, name, shape, ext)}} + + {:ok, %{ext: other_ext}} -> + {:ignore, {:error, {:ext_mismatch, expected: ext, got: other_ext}}} + + {:error, _} = err -> + {:ignore, err} + end + end + + @doc """ + Picker-facing lean projection: every adaptor row for the active + source, minus heavy JSONB columns (`schema_data`, `dependencies`, + `peer_dependencies`). + + An empty Repo result returns `{:ok, []}` but is **not** committed to + the cache — during cold-start the Scheduler will fill the table on + its next tick, and the next call will pick that up automatically. + """ + @spec packages(sup()) :: {:ok, [package_meta()]} | {:error, term()} + def packages(sup) do + cache = AdaptorsSupervisor.cache_name(sup) + source = AdaptorsSupervisor.source(sup) + + cache + |> Cachex.fetch( + {:packages, source}, + fn _key -> + case AdaptorsRepo.list_package_metas(source) do + [] -> {:ignore, {:ok, []}} + metas -> {:commit, {:ok, metas}} + end + end, + timeout: Config.cache_timeout_ms() + ) + |> unwrap() + end + + @doc """ + Cheap `{icon__ext, icon__sha256}` projection for the + icon controller's sha-validation path. Pure metadata — no disk I/O. + + Unknown adaptors return `{:error, :not_found}` and are **not** + cached, so a subsequent insert by the Scheduler becomes visible on + the very next call. + """ + @spec icon_meta(sup(), String.t()) :: + {:ok, icon_meta()} | {:error, :not_found} + def icon_meta(sup, name) do + cache = AdaptorsSupervisor.cache_name(sup) + source = AdaptorsSupervisor.source(sup) + + cache + |> Cachex.fetch( + {:icon_meta, name, source}, + fn _key -> + case AdaptorsRepo.get_adaptor(name, source) do + nil -> {:ignore, {:error, :not_found}} + adaptor -> {:commit, {:ok, project_icon_meta(adaptor)}} + end + end, + timeout: Config.cache_timeout_ms() + ) + |> unwrap() + end + + @doc """ + Re-warm Cachex from Postgres for the active source. + + Called by `Lightning.Adaptors.NodeMonitor` on `:nodeup` — a peer + rejoining after a partition can't know which `{:changed, name, source}` + broadcasts it missed, so it treats its entire local Cachex as + suspect and overwrites from the DB. + + Uses `Cachex.put_many/2` (never `Cachex.clear/1`-then-fill) so + concurrent callers never observe an empty cache and never trigger a + spurious cold-miss Strategy fetch during the warm. + """ + @spec warm_from_repo(sup()) :: :ok + def warm_from_repo(sup) do + cache = AdaptorsSupervisor.cache_name(sup) + source = AdaptorsSupervisor.source(sup) + + metas = AdaptorsRepo.list_package_metas(source) + + icon_metas = + Enum.map(metas, fn m -> + {{:icon_meta, m.name, source}, {:ok, project_icon_meta(m)}} + end) + + Cachex.put_many( + cache, + [{{:packages, source}, {:ok, metas}} | icon_metas] + ) + + :ok + end + + @spec fetch_and_persist(atom(), String.t(), :npm | :local, atom()) :: + {:commit, {:ok, term()}} | {:ignore, {:error, term()}} + defp fetch_and_persist(sup, name, source, field) do + case AdaptorsSupervisor.strategy(sup).fetch_adaptor(name) do + {:ok, record} -> + record = + record + |> Map.put(:source, source) + |> normalize_schema_data() + + {:ok, _} = AdaptorsRepo.upsert_adaptor(record) + {:commit, {:ok, Map.get(record, field)}} + + {:error, reason} -> + {:ignore, {:error, reason}} + end + end + + # Strategies should emit `schema_data` as a JSON binary, but legacy + # call paths (and tests) may still hand us a map. Normalize here so + # the cached value matches what subsequent DB-backed reads return. + defp normalize_schema_data(%{schema_data: data} = record) + when is_map(data) and not is_struct(data) do + %{record | schema_data: Jason.encode!(data)} + end + + defp normalize_schema_data(record), do: record + + @spec project_icon_meta(map()) :: icon_meta() + defp project_icon_meta(adaptor) do + Map.take(adaptor, [ + :icon_square_ext, + :icon_rectangle_ext, + :icon_square_sha256, + :icon_rectangle_sha256 + ]) + end + + @spec project_versions([map()]) :: [version_meta()] + defp project_versions(rows) do + Enum.map( + rows, + &Map.take(&1, [ + :version, + :integrity, + :size_bytes, + :published_at, + :deprecated + ]) + ) + end + + @spec ext_for_shape(icon_meta(), :square | :rectangle) :: + {:ok, String.t()} | {:error, :not_found} + defp ext_for_shape(meta, shape) do + case Map.get(meta, :"icon_#{shape}_ext") do + nil -> {:error, :not_found} + ext -> {:ok, ext} + end + end + + @spec sha256_for_shape(icon_meta(), :square | :rectangle) :: + {:ok, binary()} | {:error, :not_found} + defp sha256_for_shape(meta, shape) do + case Map.get(meta, :"icon_#{shape}_sha256") do + nil -> {:error, :not_found} + sha -> {:ok, sha} + end + end + + # `Cachex.fetch/4` returns one of: + # * `{:ok, value}` — cache hit (or coalesced peer of a `:commit`) + # * `{:commit, value}` — fallback ran and committed + # * `{:ignore, value}` — fallback ran and chose not to cache + # * `{:error, term}` — Cachex-side failure (fallback raised, etc.) + # + # Our fallbacks return `{:commit, {:ok, _}}` / `{:ignore, {:error, _}}`, + # so the wrapper tuple's second element is itself the public + # `{:ok, _} | {:error, _}` we want to return. Cachex-side `{:error, _}` + # passes through unchanged. + @spec unwrap(tuple()) :: {:ok, term()} | {:error, term()} + defp unwrap({:ok, inner}), do: inner + defp unwrap({:commit, inner}), do: inner + defp unwrap({:ignore, inner}), do: inner + defp unwrap({:error, _} = error), do: error +end diff --git a/lib/lightning/adaptors/strategy.ex b/lib/lightning/adaptors/strategy.ex new file mode 100644 index 00000000000..1a82a73c1f0 --- /dev/null +++ b/lib/lightning/adaptors/strategy.ex @@ -0,0 +1,148 @@ +defmodule Lightning.Adaptors.Strategy do + @moduledoc """ + Behaviour shared by every adaptor strategy (NPM, Local, and the test + mock). + + A strategy is the sole boundary between the `Lightning.Adaptors.*` + subsystem and the outside world. It defines four callbacks: + + * `c:fetch_adaptor/1` — given a package name, return a structured + `t:adaptor_record/0` covering version history, integrity hashes, + and dependency metadata. Icon fields are **not** part of this + record any more; the Scheduler stamps them on after joining the + bulk icon pipeline. + * `c:fetch_icon/2` — given a package name and an icon variant, + return the raw bytes plus extension. Used by the Store's rare + lazy-miss fallback. + * `c:fetch_icons/1` — bulk icon fetch for every adaptor known to + the strategy. The Scheduler invokes this once per tick in parallel + with its per-adaptor fan-out. Accepts a keyword list of options; + see the callback docs for `:prior_etags`. + * `c:list_adaptors/0` — the cheap change-signal: one call returning + `name + latest_version` for every `@openfn/*` package, used by + the scheduler to diff against the `adaptors` table. + + The active strategy module is resolved at runtime via + `Lightning.Adaptors.Config.strategy/0`. Implementations must surface + transient failures (5xx, timeout, nxdomain) as `{:error, term()}`; + retry policy lives at the scheduler/store layer, not here. + """ + + @typedoc """ + Per-version metadata extracted from an upstream packument or local + `package.json`. + """ + @type version_record :: %{ + version: String.t(), + integrity: String.t() | nil, + tarball_url: String.t() | nil, + size_bytes: integer() | nil, + dependencies: map(), + peer_dependencies: map(), + published_at: DateTime.t() | nil, + deprecated: boolean() + } + + @typedoc """ + The structured adaptor record returned by `c:fetch_adaptor/1`. Icon + fields are persisted separately by the Scheduler after joining + `c:fetch_icons/1` — they are not stamped onto this record. + """ + @type adaptor_record :: %{ + name: String.t(), + description: String.t() | nil, + homepage: String.t() | nil, + repository: String.t() | nil, + license: String.t() | nil, + latest_version: String.t(), + deprecated: boolean(), + schema_data: map() | nil, + schema_sha256: String.t() | nil, + versions: [version_record()] + } + + @typedoc """ + Fresh-fetch icon entry inside the `c:fetch_icons/1` result map. The + optional `:etag` field carries the upstream-provided cache validator + (verbatim from the HTTP response) and is `nil` when the upstream + didn't supply one — strategies without a transport-level validator + (e.g. `Lightning.Adaptors.Local`) omit the key entirely. + """ + @type icon_entry :: %{ + required(:data) => binary(), + required(:ext) => String.t(), + required(:sha256) => binary(), + optional(:etag) => String.t() | nil + } + + @typedoc """ + Per-shape value inside the `c:fetch_icons/1` result map. Either a + fresh `t:icon_entry/0` (200 response) or the `:not_modified` sentinel + (304 response — upstream confirmed unchanged; only ever returned when + the caller supplied a prior etag via the `:prior_etags` option). + """ + @type icon_shape_value :: icon_entry() | :not_modified + + @typedoc """ + Bulk icon map returned by `c:fetch_icons/1`. Three branches matter: + + * shape **entirely absent** — upstream had no such icon for this + package; + * shape present as `:not_modified` — upstream confirmed the icon + is unchanged since the prior etag was issued; + * shape present as a map — apply the bytes (a fresh fetch). + """ + @type icons_map :: %{ + required(String.t()) => %{ + optional(:square) => icon_shape_value(), + optional(:rectangle) => icon_shape_value() + } + } + + @doc """ + Fetch the full structured record for a single adaptor package. + """ + @callback fetch_adaptor(name :: String.t()) :: + {:ok, adaptor_record()} | {:error, term()} + + @doc """ + Fetch the raw bytes for one icon variant (`:square` or `:rectangle`) + of an adaptor package, together with the file extension. + """ + @callback fetch_icon(name :: String.t(), :square | :rectangle) :: + {:ok, %{data: binary(), ext: String.t()}} + | {:error, term()} + + @doc """ + Bulk fetch every available icon for every adaptor known to the + strategy. + + Returns `{:ok, partial_map}` where each per-shape slot is either + absent (no icon upstream), a fresh `t:icon_entry/0` (200), or the + `:not_modified` sentinel (304 — only when a prior etag was sent). + A top-level `{:error, term()}` is only returned when the whole + pipeline can't proceed (e.g. an upstream `list_adaptors/0` call + inside the bulk implementation fails). + + ## Options + + * `:prior_etags` — a map of the form + `%{name => %{optional(:square | :rectangle) => etag_string}}` + whose values are sent as `If-None-Match` per `(name, shape)`. + Defaults to `%{}`. Unknown keys in the keyword list are + ignored. Strategies without a transport-level cache validator + (e.g. `Lightning.Adaptors.Local`) ignore this option entirely + and never return `:not_modified`. + """ + @callback fetch_icons(opts :: keyword()) :: + {:ok, icons_map()} | {:error, term()} + + @doc """ + Cheap change-signal listing: `name + latest_version` for every + `@openfn/*` package known to the strategy. The scheduler diffs this + against the `adaptors` table to compute its work list. + """ + @callback list_adaptors() :: + {:ok, [%{name: String.t(), latest_version: String.t()}]} + | {:error, term()} +end diff --git a/lib/lightning/adaptors/supervisor.ex b/lib/lightning/adaptors/supervisor.ex new file mode 100644 index 00000000000..c1691630854 --- /dev/null +++ b/lib/lightning/adaptors/supervisor.ex @@ -0,0 +1,242 @@ +defmodule Lightning.Adaptors.Supervisor do + @moduledoc """ + Per-instance supervisor for the `Lightning.Adaptors.*` subsystem. + + The entire subsystem boots, crashes, and is supervised as a unit + under `:rest_for_one`. `Cachex` is the load-bearing root: if it + crashes, the supervisor restarts it and cascades to its dependents + (`Task.Supervisor`, plus the broadcaster/scheduler children added in + later phases) so they re-bind to the fresh Cachex name on the way + back up. + + No registered name, Cachex table name, PubSub topic, `Task.Supervisor` + name, or `HighlanderPG` lock key is hardcoded. Every name is derived + from a single `:name` opt — which is what lets the integration suite + spin up multiple isolated instances inside one BEAM for + `async: true` tests. Production starts exactly one instance under + `name: Lightning.Adaptors`. + + ## Cluster-singleton Scheduler + + The `Lightning.Adaptors.Scheduler` is wrapped in `HighlanderPG` + (`pg_try_advisory_lock` on `lock_key/1`) so exactly one node in a + multi-node deployment runs the refresh tick. The inner Scheduler + registers under `{:global, global_scheduler_name(name)}`; callers on + any node hit the leader transparently via Erlang distribution. + + ## Strategy injection + + The active `Lightning.Adaptors.Strategy` implementation is passed in + explicitly via the `:strategy` opt. Tests instantiate an isolated + supervisor with `strategy: Lightning.Adaptors.StrategyMock` — no + `Application.put_env` mutation, no shared mutable state. The + production caller in `lib/lightning/application.ex` passes the + default from `Lightning.Adaptors.Config.strategy/0` (resolved from + Application env at boot time). + + `strategy/1` and `source/1` expose the per-instance values back to + the stateless `Lightning.Adaptors.Store` callers. + """ + + use Supervisor + + alias Lightning.Adaptors.Config + + @doc """ + Start a supervisor instance. + + Required opts: + + * `:name` — supervisor instance name (atom). Derives every child + name via `Module.concat/2`. + + Optional opts: + + * `:strategy` — `Lightning.Adaptors.Strategy` implementation. + Defaults to `Lightning.Adaptors.Config.strategy/0`. + + * `:lock_key` — explicit `HighlanderPG` advisory-lock key. Defaults + to `lock_key(name)`. Override only in integration tests where + multiple supervisor instances must compete for the same lock. + """ + @spec start_link(keyword()) :: Supervisor.on_start() + def start_link(opts) do + name = Keyword.fetch!(opts, :name) + Supervisor.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts) do + name = Keyword.fetch!(opts, :name) + strategy = Keyword.get(opts, :strategy, Config.strategy()) + lock_key = Keyword.get(opts, :lock_key, lock_key(name)) + + :persistent_term.put(meta_key(name), %{ + strategy: strategy, + source: source_for(strategy) + }) + + cache = cache_name(name) + tasks = tasks_name(name) + source_topic = source_topic(name) + client_topic = client_topic(name) + + scheduler_child = + %{ + id: Lightning.Adaptors.Scheduler, + start: + {Lightning.Adaptors.Scheduler, :start_link, + [ + [ + name: global_scheduler_name(name), + sup: name, + lock_key: lock_key, + cache: cache, + tasks: tasks, + source_topic: source_topic + ] + ]} + } + + children = [ + {Cachex, name: cache}, + {Task.Supervisor, name: tasks}, + {Lightning.Adaptors.Invalidator, + name: invalidator_name(name), source_topic: source_topic, cache: cache}, + {Lightning.Adaptors.NodeMonitor, name: node_monitor_name(name), sup: name}, + {Lightning.Adaptors.ChannelBroadcaster, + name: channel_broadcaster_name(name), + source_topic: source_topic, + client_topic: client_topic, + sup: name}, + Supervisor.child_spec( + {HighlanderPG, + child: scheduler_child, + repo: Lightning.Repo, + name: lock_key, + sup_name: highlander_name(name)}, + id: highlander_name(name) + ) + ] + + Supervisor.init(children, strategy: :rest_for_one) + end + + @doc """ + The active strategy for the supervisor instance named `name`. + + Reads from `:persistent_term` populated at `init/1`. Raises if the + supervisor has not been started under that name. + """ + @spec strategy(atom()) :: module() + def strategy(name) do + :persistent_term.get(meta_key(name)).strategy + end + + @doc """ + The active source (`:npm | :local`) for the supervisor instance + named `name`. + """ + @spec source(atom()) :: :npm | :local + def source(name) do + :persistent_term.get(meta_key(name)).source + end + + @doc """ + Best-effort cleanup of the per-instance `:persistent_term` entry. + + Not called automatically — `:persistent_term.erase/1` triggers a + global GC and is expensive enough that we leave it to deliberate + teardown paths (e.g. release shutdown). + """ + @spec forget(atom()) :: boolean() + def forget(name) do + :persistent_term.erase(meta_key(name)) + end + + @doc "Cachex table name for the supervisor named `name`." + @spec cache_name(atom()) :: atom() + def cache_name(name), do: Module.concat(name, Cache) + + @doc "`Task.Supervisor` name for the supervisor named `name`." + @spec tasks_name(atom()) :: atom() + def tasks_name(name), do: Module.concat(name, Tasks) + + @doc "`Invalidator` GenServer name for the supervisor named `name`." + @spec invalidator_name(atom()) :: atom() + def invalidator_name(name), do: Module.concat(name, Invalidator) + + @doc "`ChannelBroadcaster` GenServer name for the supervisor named `name`." + @spec channel_broadcaster_name(atom()) :: atom() + def channel_broadcaster_name(name), + do: Module.concat(name, ChannelBroadcaster) + + @doc "`NodeMonitor` GenServer name for the supervisor named `name`." + @spec node_monitor_name(atom()) :: atom() + def node_monitor_name(name), do: Module.concat(name, NodeMonitor) + + @doc """ + Local `Scheduler` GenServer name for the supervisor named `name`. + + The inner Scheduler is actually registered globally — see + `global_scheduler_name/1`. This atom form is retained as the + child-spec `id` and for derived module names. + """ + @spec scheduler_name(atom()) :: atom() + def scheduler_name(name), do: Module.concat(name, Scheduler) + + @doc """ + `:global`-registered Scheduler name for the supervisor named `name`. + + The HighlanderPG-wrapped Scheduler registers itself under this name so + callers on any node reach the leader via Erlang distribution. Pass the + return value to `GenServer.call/3` directly. + """ + @spec global_scheduler_name(atom()) :: {:global, atom()} + def global_scheduler_name(name), do: {:global, scheduler_name(name)} + + @doc """ + `HighlanderPG` supervisor name for the supervisor named `name`. + + Used as the child-spec id and the `:sup_name` for introspection + (`HighlanderPG.which_children/1`, etc.). + """ + @spec highlander_name(atom()) :: atom() + def highlander_name(name), do: Module.concat(name, HighlanderPG) + + @doc """ + Source-side PubSub topic for the supervisor named `name`. + + Used by the `Scheduler` and `Invalidator` to broadcast and receive + `{:changed, name, source}` style events. + """ + @spec source_topic(atom()) :: String.t() + def source_topic(name), do: "adaptors:#{inspect(name)}" + + @doc """ + Client-side PubSub topic for the supervisor named `name`. + + The `ChannelBroadcaster` republishes throttled updates from + `source_topic/1` onto this topic for `WorkflowChannel` subscribers. + """ + @spec client_topic(atom()) :: String.t() + def client_topic(name), do: "adaptors:client_update:#{inspect(name)}" + + @doc """ + Postgres advisory-lock key for the supervisor named `name`. + + Derived as `:erlang.phash2({:adaptors, name})` so each supervisor + instance leases its `HighlanderPG`-wrapped `Scheduler` against a + distinct `int4` key — two concurrent test supervisors with different + names cannot collide on advisory locks. The §12.7 integration test + overrides `:lock_key` on `start_link/1` to force two supervisors to + compete for the same lock. + """ + @spec lock_key(atom()) :: non_neg_integer() + def lock_key(name), do: :erlang.phash2({:adaptors, name}) + + defp meta_key(name), do: {__MODULE__, name} + + defp source_for(Lightning.Adaptors.Local), do: :local + defp source_for(_other), do: :npm +end diff --git a/lib/lightning/application.ex b/lib/lightning/application.ex index 34c3ce853f1..4d9a19c3cbd 100644 --- a/lib/lightning/application.ex +++ b/lib/lightning/application.ex @@ -170,6 +170,7 @@ defmodule Lightning.Application do LightningWeb.WorkerPresence, adaptor_registry_childspec, adaptor_service_childspec, + {Lightning.Adaptors.Supervisor, name: Lightning.Adaptors}, {Lightning.TaskWorker, name: :cli_task_worker}, {Lightning.Runtime.RuntimeManager, worker_secret: Lightning.Config.worker_secret(), diff --git a/mix.exs b/mix.exs index cd65eea3a9e..a71344f34cc 100644 --- a/mix.exs +++ b/mix.exs @@ -129,6 +129,7 @@ defmodule Lightning.MixProject do # webtransport dep requires h2 ~> 0.10.4, so newer releases cannot resolve. {:hackney, "~> 4.6.0", override: true}, {:heroicons, "~> 0.5.3"}, + {:highlander_pg, "~> 1.0"}, {:httpoison, "~> 3.0.0", override: true}, {:jason, "~> 1.4"}, {:joken, "~> 2.6.0"}, @@ -155,7 +156,9 @@ defmodule Lightning.MixProject do {:phoenix_live_view, "~> 1.0.17"}, {:cors_plug, "~> 3.0"}, {:plug_cowboy, "~> 2.5"}, - {:postgrex, ">= 0.0.0"}, + # highlander_pg 1.0.8 caps postgrex at ~> 0.21; it only issues advisory + # locks, so override rather than hold the rest of the app back. + {:postgrex, ">= 0.0.0", override: true}, {:prom_ex, "~> 1.11.0"}, {:rambo, "~> 0.3.4"}, {:retry, "~> 0.18"}, diff --git a/mix.lock b/mix.lock index a381f8eb89c..f2140fa1964 100644 --- a/mix.lock +++ b/mix.lock @@ -57,6 +57,7 @@ "hammer": {:hex, :hammer, "6.2.1", "5ae9c33e3dceaeb42de0db46bf505bd9c35f259c8defb03390cd7556fea67ee2", [:mix], [{:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "b9476d0c13883d2dc0cc72e786bac6ac28911fba7cc2e04b70ce6a6d9c4b2bdc"}, "hammer_backend_mnesia": {:hex, :hammer_backend_mnesia, "0.6.1", "d10d94fc29cbffbf04ecb3c3127d705ce4cc1cecfb9f3d6b18a554c3cae9af2c", [:mix], [{:hammer, "~> 6.1", [hex: :hammer, repo: "hexpm", optional: false]}], "hexpm", "85ad2ef6ebe035207dd9a03a116dc6a7ee43fbd53e8154cf32a1e33b9200fb62"}, "heroicons": {:hex, :heroicons, "0.5.6", "95d730e7179c633df32d95c1fdaaecdf81b0da11010b89b737b843ac176a7eb5", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:phoenix_live_view, ">= 0.18.2", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "ca267f02a5fa695a4178a737b649fb6644a2e399639d4ba7964c18e8a58c2352"}, + "highlander_pg": {:hex, :highlander_pg, "1.0.8", "30c5c2cd23cd48991d4b5f66368997ac711d3e736217e0a9e762051d22b0329a", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.1 or ~> 0.17.0 or ~> 0.18.0 or ~> 0.19.0 or ~> 0.20.0 or ~> 0.21.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "fe03a830971e3d626d776ed1c62e0e1148e953274f1e82eba8a0618dde1c415c"}, "hpax": {:hex, :hpax, "1.0.4", "777de5d433b0fbdc7c418159c8055910faa8047ffdb3d6b31098d2a46cd7685c", [:mix], [], "hexpm", "afc7cb142ebcc2d01ce7816190b98ce5dd49e799111b24249f3443d730f377ca"}, "httpoison": {:hex, :httpoison, "3.0.0", "8566a933bb9175236d1ec335978445b67cd1f5b5d3ead6ca4b80be469d41f5d9", [:mix], [{:hackney, "~> 4.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "9130197b7658901c493d6fcfb842fb9676300fa8a6c8ed058c8889cf1a77f3c2"}, "idna": {:hex, :idna, "7.1.0", "1067a13043538129602d2f2ce6899d8713125c7d19734aa557ce2e3ea55bd4f1", [:rebar3], [], "hexpm", "6ae959a025bf36df61a8cab8508d9654891b5426a84c44d82deaffd6ddf8c71f"}, diff --git a/priv/repo/migrations/20260514150000_create_adaptors.exs b/priv/repo/migrations/20260514150000_create_adaptors.exs new file mode 100644 index 00000000000..0e6268c960d --- /dev/null +++ b/priv/repo/migrations/20260514150000_create_adaptors.exs @@ -0,0 +1,52 @@ +defmodule Lightning.Repo.Migrations.CreateAdaptors do + use Ecto.Migration + + def change do + create table(:adaptors, primary_key: false) do + add :id, :binary_id, primary_key: true + add :name, :string, null: false + add :source, :string, null: false, default: "npm" + add :description, :text + add :homepage, :string + add :repository, :string + add :license, :string + add :latest_version, :string, null: false + add :deprecated, :boolean, default: false, null: false + add :schema_data, :text + add :schema_sha256, :string + add :icon_square_ext, :string + add :icon_rectangle_ext, :string + add :icon_square_sha256, :binary + add :icon_rectangle_sha256, :binary + add :icon_square_etag, :string + add :icon_rectangle_etag, :string + add :checked_at, :utc_datetime_usec, null: false + + timestamps(type: :utc_datetime_usec, null: false) + end + + create unique_index(:adaptors, [:name, :source]) + create index(:adaptors, [:source, :checked_at]) + + create table(:adaptor_versions, primary_key: false) do + add :id, :binary_id, primary_key: true + + add :adaptor_id, + references(:adaptors, type: :binary_id, on_delete: :delete_all), + null: false + + add :version, :string, null: false + add :integrity, :string + add :tarball_url, :string + add :size_bytes, :integer + add :dependencies, :map + add :peer_dependencies, :map + add :published_at, :utc_datetime_usec + add :deprecated, :boolean, default: false, null: false + + timestamps(type: :utc_datetime_usec, updated_at: false, null: false) + end + + create unique_index(:adaptor_versions, [:adaptor_id, :version]) + end +end diff --git a/test/lightning/adaptors/channel_broadcaster_test.exs b/test/lightning/adaptors/channel_broadcaster_test.exs new file mode 100644 index 00000000000..002f0e19978 --- /dev/null +++ b/test/lightning/adaptors/channel_broadcaster_test.exs @@ -0,0 +1,238 @@ +defmodule Lightning.Adaptors.ChannelBroadcasterTest do + @moduledoc """ + Tests `:flush` via `Lightning.Adaptors.packages/1` (the 2-arity facade), + not `packages/0`, because each test spins up its own isolated supervisor + instance. The Batch 7 review should confirm that `/1` and `/0` are + behaviourally identical in production (both delegate to `Store.packages/1`). + """ + + use ExUnit.Case, async: true + + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + setup do + sup = :"cb_test_#{System.unique_integer([:positive])}" + + # The supervisor's :rest_for_one child list starts the + # ChannelBroadcaster automatically — registered under + # `channel_broadcaster_name(sup)`. + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + ) + + # Stop the auto-started Invalidator — these tests pre-populate the + # Cachex `{:packages, source}` key directly to exercise the + # ChannelBroadcaster's `:flush` path in isolation. The Invalidator + # subscribes to the same source_topic and would race the broadcaster + # by deleting the cached entry before the flush window expires. + :ok = Supervisor.terminate_child(sup, Lightning.Adaptors.Invalidator) + + source_topic = AdaptorsSupervisor.source_topic(sup) + client_topic = AdaptorsSupervisor.client_topic(sup) + cb_name = AdaptorsSupervisor.channel_broadcaster_name(sup) + cache = AdaptorsSupervisor.cache_name(sup) + source = AdaptorsSupervisor.source(sup) + + packages = [%{name: "@openfn/language-http", latest_version: "1.0.0"}] + Cachex.put!(cache, {:packages, source}, {:ok, packages}) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, client_topic) + + {:ok, + sup: sup, + cb_name: cb_name, + source_topic: source_topic, + cache: cache, + source: source, + packages: packages} + end + + describe "start_link/1" do + test "registers under the :name opt", %{cb_name: cb_name} do + assert is_pid(Process.whereis(cb_name)) + end + end + + describe "handle_info/2 - {:changed, ...}" do + test "first message in idle state arms the 250ms timer", %{ + cb_name: cb_name, + source_topic: source_topic + } do + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, "pkg", :npm} + ) + + %{timer: timer} = :sys.get_state(cb_name) + assert is_reference(timer) + end + + test "subsequent messages within the window are dropped — one broadcast per burst", + %{source_topic: source_topic, packages: packages} do + for _ <- 1..5 do + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, "pkg", :npm} + ) + end + + assert_receive %{ + event: "adaptors_updated", + payload: %{adaptors: ^packages} + }, + 500 + + refute_receive %{event: "adaptors_updated"}, 100 + end + + test "timer resets to nil after :flush fires", %{ + cb_name: cb_name, + source_topic: source_topic + } do + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, "pkg", :npm} + ) + + assert_receive %{event: "adaptors_updated"}, 500 + %{timer: timer} = :sys.get_state(cb_name) + assert timer == nil + end + end + + describe "handle_info/2 - :flush" do + test "broadcasts the envelope to client_topic with the correct shape", %{ + source_topic: source_topic, + packages: packages + } do + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, "pkg", :npm} + ) + + assert_receive %{ + event: "adaptors_updated", + payload: %{adaptors: ^packages} + }, + 500 + end + + test "broadcasts with empty adaptors list when packages returns {:ok, []}", + %{ + cache: cache, + source: source, + source_topic: source_topic + } do + Cachex.put!(cache, {:packages, source}, {:ok, []}) + + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, "pkg", :npm} + ) + + assert_receive %{event: "adaptors_updated", payload: %{adaptors: []}}, 500 + end + end + + describe "crash recovery" do + test "supervisor restarts the GenServer; next {:changed} re-arms cleanly", %{ + cb_name: cb_name, + source_topic: source_topic, + packages: packages + } do + original_pid = Process.whereis(cb_name) + assert is_pid(original_pid) + + ref = Process.monitor(original_pid) + + # Arm the timer, then kill the process mid-burst. + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, "pkg", :npm} + ) + + Process.exit(original_pid, :kill) + + # Confirm death before looking for the restarted process. + assert_receive {:DOWN, ^ref, :process, ^original_pid, :killed}, 500 + + new_pid = await_registered(cb_name) + assert is_pid(new_pid) + assert new_pid != original_pid + + # The new instance starts with timer: nil — one more {:changed} opens a + # fresh 250ms window and produces a clean broadcast. + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, "pkg", :npm} + ) + + assert_receive %{ + event: "adaptors_updated", + payload: %{adaptors: ^packages} + }, + 500 + end + end + + describe "leading-edge throttle invariant" do + test "a 10ms drip over 500ms yields multiple broadcasts, not 0 and not one-per-message", + %{source_topic: source_topic} do + task = + Task.async(fn -> + for _ <- 1..50 do + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, "pkg", :npm} + ) + + Process.sleep(10) + end + end) + + Task.await(task, 3_000) + # Allow time for the final flush window to fire. + Process.sleep(300) + + count = drain_broadcasts() + + # Leading-edge invariant: throttle produces some broadcasts (> 0) + # but far fewer than one per message (< 50). + assert count > 0 and count < 50, + "Expected leading-edge throttling (1..49), got #{count}" + end + end + + defp await_registered(name, deadline \\ nil) do + deadline = deadline || System.monotonic_time(:millisecond) + 500 + + case Process.whereis(name) do + nil -> + if System.monotonic_time(:millisecond) < deadline do + Process.sleep(10) + await_registered(name, deadline) + else + raise "#{inspect(name)} did not restart within 500ms" + end + + pid -> + pid + end + end + + defp drain_broadcasts(acc \\ 0) do + receive do + %{event: "adaptors_updated"} -> drain_broadcasts(acc + 1) + after + 0 -> acc + end + end +end diff --git a/test/lightning/adaptors/config_test.exs b/test/lightning/adaptors/config_test.exs new file mode 100644 index 00000000000..cce00b3d670 --- /dev/null +++ b/test/lightning/adaptors/config_test.exs @@ -0,0 +1,114 @@ +defmodule Lightning.Adaptors.ConfigTest do + use ExUnit.Case, async: true + + alias Lightning.Adaptors.Config + + @parent_key Lightning.Adaptors + + describe "current_source/0" do + test "returns :local when strategy is Lightning.Adaptors.Local" do + put_parent(:strategy, Lightning.Adaptors.Local) + + assert Config.current_source() == :local + end + + test "returns :npm for any other strategy module" do + put_parent(:strategy, Lightning.Adaptors.NPM) + assert Config.current_source() == :npm + + put_parent(:strategy, SomeOther.Strategy) + assert Config.current_source() == :npm + end + end + + describe "icon_path/0" do + test "resolves a {:tmp, suffix} tuple against System.tmp_dir!/0" do + put_parent(:icon_path, {:tmp, "lightning/adaptor_icons_under_test"}) + + assert Config.icon_path() == + Path.join(System.tmp_dir!(), "lightning/adaptor_icons_under_test") + end + + test "returns a plain binary path verbatim" do + put_parent(:icon_path, "/var/lib/lightning/adaptor_icons") + + assert Config.icon_path() == "/var/lib/lightning/adaptor_icons" + end + end + + describe "strategy_opts/1" do + test "reads the strategy module's own Application key" do + put_strategy_opts(SomeStrategy.Module, repo_path: "/tmp/local-adaptors") + + assert Config.strategy_opts(SomeStrategy.Module) == + [repo_path: "/tmp/local-adaptors"] + end + + test "returns [] when the strategy module's Application key is unset" do + clear_strategy_opts(UnsetStrategy.Module) + + assert Config.strategy_opts(UnsetStrategy.Module) == [] + end + end + + describe "defaults when unset" do + test "refresh_interval/0 defaults to :timer.hours(1)" do + delete_parent_key(:refresh_interval) + + assert Config.refresh_interval() == :timer.hours(1) + end + + test "cache_timeout_ms/0 defaults to 15_000" do + delete_parent_key(:cache_timeout_ms) + + assert Config.cache_timeout_ms() == 15_000 + end + end + + defp put_parent(key, value) do + original = Application.get_env(:lightning, @parent_key, []) + + Application.put_env( + :lightning, + @parent_key, + Keyword.put(original, key, value) + ) + + on_exit(fn -> + Application.put_env(:lightning, @parent_key, original) + end) + end + + defp delete_parent_key(key) do + original = Application.get_env(:lightning, @parent_key, []) + Application.put_env(:lightning, @parent_key, Keyword.delete(original, key)) + + on_exit(fn -> + Application.put_env(:lightning, @parent_key, original) + end) + end + + defp put_strategy_opts(mod, value) do + original = Application.get_env(:lightning, mod, :__unset__) + Application.put_env(:lightning, mod, value) + + on_exit(fn -> + restore_app_env(mod, original) + end) + end + + defp clear_strategy_opts(mod) do + original = Application.get_env(:lightning, mod, :__unset__) + Application.delete_env(:lightning, mod) + + on_exit(fn -> + restore_app_env(mod, original) + end) + end + + defp restore_app_env(mod, :__unset__), + do: Application.delete_env(:lightning, mod) + + defp restore_app_env(mod, value), + do: Application.put_env(:lightning, mod, value) +end diff --git a/test/lightning/adaptors/end_to_end_broadcast_test.exs b/test/lightning/adaptors/end_to_end_broadcast_test.exs new file mode 100644 index 00000000000..ed4e0d765c1 --- /dev/null +++ b/test/lightning/adaptors/end_to_end_broadcast_test.exs @@ -0,0 +1,48 @@ +defmodule Lightning.Adaptors.EndToEndBroadcastTest do + @moduledoc """ + Phase A closeout — §6.5c integration smoke. + + A `{:changed, name, source}` broadcast on the per-instance source + topic (the cache-coherence audience that the `Scheduler` and + `Invalidator` share) must traverse the wired stack and arrive on + the per-instance client topic (the display-freshness audience that + `WorkflowChannel` subscribers listen to) as a single coalesced + `adaptors_updated` envelope. + + This is the only assertion that breaks if any of the four newly-wired + Supervisor children (Invalidator, NodeMonitor, ChannelBroadcaster, + Scheduler) is misconfigured for the boot path. + """ + + use Lightning.DataCase, async: false + + alias Lightning.Adaptors.ChannelBroadcaster + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + test "PubSub source-topic broadcast reaches client topic as coalesced envelope" do + sup = :"e2e_#{System.unique_integer([:positive])}" + + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.Local} + ) + + source_topic = AdaptorsSupervisor.source_topic(sup) + client_topic = AdaptorsSupervisor.client_topic(sup) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, client_topic) + + :ok = + Phoenix.PubSub.broadcast( + Lightning.PubSub, + source_topic, + {:changed, "@openfn/language-test", :local} + ) + + # The ChannelBroadcaster fans out a map envelope (see + # `Lightning.Adaptors.ChannelBroadcaster.handle_info(:flush, _)`). + # The DB is empty in this case → `Store.packages/1` returns + # `{:ok, []}` → an empty-list envelope is broadcast. + assert_receive %{event: "adaptors_updated", payload: %{adaptors: _}}, + ChannelBroadcaster.debounce_ms() + 200 + end +end diff --git a/test/lightning/adaptors/highlander_integration_test.exs b/test/lightning/adaptors/highlander_integration_test.exs new file mode 100644 index 00000000000..bbcc3472dce --- /dev/null +++ b/test/lightning/adaptors/highlander_integration_test.exs @@ -0,0 +1,107 @@ +defmodule Lightning.Adaptors.HighlanderIntegrationTest do + @moduledoc """ + §12.7 — verifies that the HighlanderPG-wrapped `Lightning.Adaptors.Scheduler` + actually behaves as a cluster singleton when two supervisor instances + compete for the same Postgres advisory lock. + + Both supervisors share an explicit `:lock_key` so they race for the + same `pg_try_advisory_lock` bucket, but each keeps its own derived + `:global` Scheduler name. Exactly one of them — the leader — registers + a Scheduler under its `{:global, …}` name; the other's HighlanderPG + polls and waits. When the leading supervisor stops (releasing its + Postgres session and thus its advisory lock), the surviving instance + must acquire the lock within ~2× the default 300ms polling interval + and register its own Scheduler under its own `{:global, …}` name. + """ + + # async: false — real advisory locks coordinate against the test DB; + # set_mox_global so the StrategyMock is visible to the wrapped child + # processes started by HighlanderPG. + use Lightning.DataCase, async: false + + import Eventually + import Mox + + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + setup :set_mox_global + setup :verify_on_exit! + + # Both supervisors come up with refresh_interval=0 (default in config/test.exs) + # so the inert Scheduler's init does no DB work; the test only cares about + # HighlanderPG's leader election, not refresh behaviour. + + test "two supervisors sharing one lock_key: only one runs the Scheduler at a time; failover on leader shutdown" do + suffix = System.unique_integer([:positive]) + sup_a = :"hl_a_#{suffix}" + sup_b = :"hl_b_#{suffix}" + shared_lock_key = :erlang.phash2({:adaptors_highlander_test, suffix}) + + on_exit(fn -> + AdaptorsSupervisor.forget(sup_a) + AdaptorsSupervisor.forget(sup_b) + end) + + {:ok, _pid_a} = + start_supervised( + Supervisor.child_spec( + {AdaptorsSupervisor, + name: sup_a, + strategy: Lightning.Adaptors.StrategyMock, + lock_key: shared_lock_key}, + id: :sup_a + ) + ) + + {:ok, _pid_b} = + start_supervised( + Supervisor.child_spec( + {AdaptorsSupervisor, + name: sup_b, + strategy: Lightning.Adaptors.StrategyMock, + lock_key: shared_lock_key}, + id: :sup_b + ) + ) + + {:global, gname_a} = AdaptorsSupervisor.global_scheduler_name(sup_a) + {:global, gname_b} = AdaptorsSupervisor.global_scheduler_name(sup_b) + + # The advisory-lock race may go to either supervisor. Allow ~3s for + # the winner's HighlanderPG to acquire the lock and start its child. + assert_eventually( + is_pid(:global.whereis_name(gname_a)) or + is_pid(:global.whereis_name(gname_b)), + 3_000 + ) + + leader_global = + cond do + is_pid(:global.whereis_name(gname_a)) -> gname_a + is_pid(:global.whereis_name(gname_b)) -> gname_b + end + + surviving_global = + if leader_global == gname_a, do: gname_b, else: gname_a + + leader_sup = if leader_global == gname_a, do: sup_a, else: sup_b + + # Singleton invariant: only the leader's :global registration is + # populated cluster-wide. + assert :global.whereis_name(surviving_global) == :undefined, + "expected only the leader to have a globally-registered Scheduler" + + # Stop the leader supervisor. start_supervised gave us a stable id + # (:sup_a / :sup_b), so we can unambiguously target it for teardown. + leader_id = if leader_sup == sup_a, do: :sup_a, else: :sup_b + :ok = stop_supervised(leader_id) + + # The surviving HighlanderPG polls at 300ms by default; give it + # comfortably more than 2× that interval to win the lock and start + # its wrapped Scheduler under its own :global name. + assert_eventually(is_pid(:global.whereis_name(surviving_global)), 3_000) + + # Sanity: the formerly-leading :global name is gone. + assert :global.whereis_name(leader_global) == :undefined + end +end diff --git a/test/lightning/adaptors/icon_cache_test.exs b/test/lightning/adaptors/icon_cache_test.exs new file mode 100644 index 00000000000..849ee02d599 --- /dev/null +++ b/test/lightning/adaptors/icon_cache_test.exs @@ -0,0 +1,156 @@ +defmodule Lightning.Adaptors.IconCacheTest do + use ExUnit.Case, async: false + + alias Lightning.Adaptors.IconCache + + @parent_key Lightning.Adaptors + + setup do + root = + Path.join( + System.tmp_dir!(), + "lightning_icon_cache_test_#{System.unique_integer([:positive])}" + ) + + File.mkdir_p!(root) + + original = Application.get_env(:lightning, @parent_key, []) + + Application.put_env( + :lightning, + @parent_key, + Keyword.put(original, :icon_path, root) + ) + + on_exit(fn -> + Application.put_env(:lightning, @parent_key, original) + File.rm_rf!(root) + end) + + {:ok, root: root} + end + + describe "path/4" do + test "joins Config.icon_path with source/name/shape.ext", %{root: root} do + assert IconCache.path(:npm, "salesforce", :square, "png") == + Path.join([root, "npm", "salesforce", "square.png"]) + end + + test "handles names containing a slash like @openfn/language-foo", %{ + root: root + } do + assert IconCache.path(:npm, "@openfn/language-foo", :square, "png") == + Path.join([ + root, + "npm", + "@openfn", + "language-foo", + "square.png" + ]) + end + + test "source-partitions paths for the same name", %{root: root} do + npm_path = IconCache.path(:npm, "salesforce", :square, "png") + local_path = IconCache.path(:local, "salesforce", :square, "png") + + assert npm_path == Path.join([root, "npm", "salesforce", "square.png"]) + + assert local_path == + Path.join([root, "local", "salesforce", "square.png"]) + + refute npm_path == local_path + end + + test "is pure — nothing is created on disk", %{root: root} do + _ = IconCache.path(:npm, "never-written", :rectangle, "svg") + + assert File.ls!(root) == [] + end + end + + describe "cached?/4" do + test "returns false when the file does not exist" do + refute IconCache.cached?(:npm, "definitely-missing", :square, "png") + end + + test "returns true after write!/5 places the file" do + {:ok, _sha} = IconCache.write!(:npm, "cached-pkg", :square, "png", "x") + + assert IconCache.cached?(:npm, "cached-pkg", :square, "png") + end + + test "stays source-partitioned: a write to :npm doesn't satisfy :local" do + {:ok, _} = IconCache.write!(:npm, "split-pkg", :square, "png", "x") + + assert IconCache.cached?(:npm, "split-pkg", :square, "png") + refute IconCache.cached?(:local, "split-pkg", :square, "png") + end + end + + describe "write!/5" do + test "writes bytes and a round-trip read returns them" do + bytes = :crypto.strong_rand_bytes(2_048) + + {:ok, _sha} = + IconCache.write!(:npm, "round-trip", :square, "png", bytes) + + assert File.read!(IconCache.path(:npm, "round-trip", :square, "png")) == + bytes + end + + test "returns the sha256 of the supplied bytes as a 32-byte binary" do + bytes = "hello, icon" + + {:ok, sha} = IconCache.write!(:npm, "sha-test", :square, "png", bytes) + + assert sha == :crypto.hash(:sha256, bytes) + assert byte_size(sha) == 32 + end + + test "is latest-only: a subsequent write for the same key overwrites" do + {:ok, _} = IconCache.write!(:npm, "overwrite", :square, "png", "first") + {:ok, _} = IconCache.write!(:npm, "overwrite", :square, "png", "second") + + assert File.read!(IconCache.path(:npm, "overwrite", :square, "png")) == + "second" + end + + test "creates intermediate directories for scoped names" do + {:ok, _} = + IconCache.write!(:npm, "@openfn/language-http", :square, "png", "abc") + + assert File.read!( + IconCache.path(:npm, "@openfn/language-http", :square, "png") + ) == "abc" + end + + test "is atomic: concurrent writers produce no half-written file and no leftover temps", + %{root: root} do + payloads = + for i <- 0..49 do + :crypto.strong_rand_bytes(16_384) <> <> + end + + payloads + |> Enum.map(fn bytes -> + Task.async(fn -> + IconCache.write!(:npm, "concurrent", :square, "png", bytes) + end) + end) + |> Task.await_many(10_000) + + final_path = IconCache.path(:npm, "concurrent", :square, "png") + final = File.read!(final_path) + + assert final in payloads, + "final file does not match any written payload — write was not atomic" + + dir = Path.dirname(final_path) + + assert dir |> File.ls!() |> Enum.reject(&(&1 == "square.png")) == [], + "leftover temp files in #{dir}: #{inspect(File.ls!(dir))}" + + _ = root + end + end +end diff --git a/test/lightning/adaptors/invalidator_test.exs b/test/lightning/adaptors/invalidator_test.exs new file mode 100644 index 00000000000..4ac7feac8d1 --- /dev/null +++ b/test/lightning/adaptors/invalidator_test.exs @@ -0,0 +1,122 @@ +defmodule Lightning.Adaptors.InvalidatorTest do + use ExUnit.Case, async: true + + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + setup do + sup = :"inv_test_#{System.unique_integer([:positive])}" + + # The supervisor's :rest_for_one child list starts the Invalidator + # automatically — registered under `invalidator_name(sup)`. + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + ) + + cache = AdaptorsSupervisor.cache_name(sup) + inv_name = AdaptorsSupervisor.invalidator_name(sup) + + {:ok, sup: sup, cache: cache, inv_name: inv_name} + end + + describe "start_link/1" do + test "registers under the :name opt", %{inv_name: inv_name} do + assert is_pid(Process.whereis(inv_name)) + end + end + + describe "handle_info/2 - {:changed, name, source}" do + test "evicts all four matching cache keys on broadcast", %{ + sup: sup, + cache: cache, + inv_name: inv_name + } do + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + name = "@openfn/language-http" + + Cachex.put!(cache, {:schema, name, source}, {:ok, %{"type" => "object"}}) + Cachex.put!(cache, {:versions, name, source}, {:ok, [%{version: "1.0.0"}]}) + + Cachex.put!( + cache, + {:icon_meta, name, source}, + {:ok, %{icon_square_ext: "svg"}} + ) + + Cachex.put!(cache, {:packages, source}, {:ok, [%{name: name}]}) + + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, name, source} + ) + + :sys.get_state(inv_name) + + assert {:ok, nil} = Cachex.get(cache, {:schema, name, source}) + assert {:ok, nil} = Cachex.get(cache, {:versions, name, source}) + assert {:ok, nil} = Cachex.get(cache, {:icon_meta, name, source}) + assert {:ok, nil} = Cachex.get(cache, {:packages, source}) + end + + test "does not evict name-scoped keys for a different adaptor", %{ + sup: sup, + cache: cache, + inv_name: inv_name + } do + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + + target = "@openfn/language-http" + bystander = "@openfn/language-dhis2" + + Cachex.put!( + cache, + {:schema, bystander, source}, + {:ok, %{"type" => "object"}} + ) + + Cachex.put!(cache, {:versions, bystander, source}, {:ok, []}) + Cachex.put!(cache, {:icon_meta, bystander, source}, {:ok, %{}}) + + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, target, source} + ) + + :sys.get_state(inv_name) + + assert {:ok, {:ok, _}} = Cachex.get(cache, {:schema, bystander, source}) + assert {:ok, {:ok, _}} = Cachex.get(cache, {:versions, bystander, source}) + assert {:ok, {:ok, _}} = Cachex.get(cache, {:icon_meta, bystander, source}) + end + + test "{:changed, _, :local} on an npm-mode node is harmless", %{ + sup: sup, + cache: cache, + inv_name: inv_name + } do + source_topic = AdaptorsSupervisor.source_topic(sup) + name = "@openfn/language-http" + npm_source = AdaptorsSupervisor.source(sup) + + Cachex.put!( + cache, + {:schema, name, npm_source}, + {:ok, %{"type" => "object"}} + ) + + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, name, :local} + ) + + :sys.get_state(inv_name) + + assert is_pid(Process.whereis(inv_name)), "invalidator must still be alive" + assert {:ok, {:ok, _}} = Cachex.get(cache, {:schema, name, npm_source}) + end + end +end diff --git a/test/lightning/adaptors/local_test.exs b/test/lightning/adaptors/local_test.exs new file mode 100644 index 00000000000..1fc69bb4e91 --- /dev/null +++ b/test/lightning/adaptors/local_test.exs @@ -0,0 +1,360 @@ +defmodule Lightning.Adaptors.LocalTest do + use ExUnit.Case, async: false + + import ExUnit.CaptureLog + + alias Lightning.Adaptors.Local + + setup do + root = + Path.join( + System.tmp_dir!(), + "lightning_adaptors_local_test_#{System.unique_integer([:positive])}" + ) + + File.mkdir_p!(Path.join(root, "packages")) + + original = Application.get_env(:lightning, Local, :__unset__) + Application.put_env(:lightning, Local, path: root) + + on_exit(fn -> + case original do + :__unset__ -> Application.delete_env(:lightning, Local) + value -> Application.put_env(:lightning, Local, value) + end + + File.rm_rf!(root) + end) + + {:ok, root: root} + end + + describe "list_adaptors/0" do + test "returns {:ok, []} when there are no package directories" do + assert Local.list_adaptors() == {:ok, []} + end + + test "returns name + latest_version for each package", %{root: root} do + write_package!(root, "language-http", "@openfn/language-http", "2.1.0") + + write_package!( + root, + "language-salesforce", + "@openfn/language-salesforce", + "4.6.3" + ) + + {:ok, listing} = Local.list_adaptors() + + assert Enum.sort_by(listing, & &1.name) == [ + %{name: "@openfn/language-http", latest_version: "2.1.0"}, + %{name: "@openfn/language-salesforce", latest_version: "4.6.3"} + ] + end + + test "collapses multiple directories sharing a name into one record with the highest semver as latest_version", + %{root: root} do + write_package!(root, "http-1", "@openfn/language-http", "1.0.0") + write_package!(root, "http-2", "@openfn/language-http", "2.3.4") + write_package!(root, "http-3", "@openfn/language-http", "2.3.1") + + assert {:ok, [%{name: "@openfn/language-http", latest_version: "2.3.4"}]} = + Local.list_adaptors() + end + + test "skips a directory with a missing package.json and logs a warning", + %{root: root} do + write_package!(root, "good", "@openfn/language-good", "1.0.0") + File.mkdir_p!(Path.join([root, "packages", "broken"])) + + {result, log} = with_log(fn -> Local.list_adaptors() end) + + assert {:ok, [%{name: "@openfn/language-good"}]} = result + assert log =~ "skipping" + assert log =~ "broken" + end + + test "skips a directory with unparseable JSON and logs a warning", + %{root: root} do + bad_dir = Path.join([root, "packages", "junk"]) + File.mkdir_p!(bad_dir) + File.write!(Path.join(bad_dir, "package.json"), "{not json") + + {result, log} = with_log(fn -> Local.list_adaptors() end) + + assert {:ok, []} = result + assert log =~ "skipping" + assert log =~ "junk" + end + + test "skips package.json that has no name or version", %{root: root} do + dir = Path.join([root, "packages", "incomplete"]) + File.mkdir_p!(dir) + + File.write!( + Path.join(dir, "package.json"), + Jason.encode!(%{"name" => "x"}) + ) + + {result, _log} = with_log(fn -> Local.list_adaptors() end) + + assert {:ok, []} = result + end + + test "returns {:error, :no_repo_path} when :path is unset" do + Application.delete_env(:lightning, Local) + + assert capture_log(fn -> + assert Local.list_adaptors() == {:error, :no_repo_path} + end) =~ "not configured" + end + end + + describe "fetch_adaptor/1" do + test "decodes a realistic on-disk package into the full adaptor_record shape", + %{root: root} do + pkg = %{ + "name" => "@openfn/language-http", + "version" => "2.1.0", + "description" => "HTTP adaptor", + "homepage" => "https://docs.openfn.org/adaptors/http", + "repository" => %{"url" => "git+https://github.com/OpenFn/adaptors.git"}, + "license" => "LGPL-3.0", + "dependencies" => %{"axios" => "^1.5.0"}, + "peerDependencies" => %{"@openfn/language-common" => "^2.0.0"} + } + + schema = %{"type" => "object", "properties" => %{"baseUrl" => %{}}} + + dir = write_package_raw!(root, "language-http", pkg) + + File.write!( + Path.join(dir, "configuration-schema.json"), + Jason.encode!(schema) + ) + + write_icon!(dir, :square, "png", "square-bytes") + write_icon!(dir, :rectangle, "svg", "") + + {:ok, record} = Local.fetch_adaptor("@openfn/language-http") + + assert record.name == "@openfn/language-http" + assert record.description == "HTTP adaptor" + assert record.homepage == "https://docs.openfn.org/adaptors/http" + assert record.repository == "git+https://github.com/OpenFn/adaptors.git" + assert record.license == "LGPL-3.0" + assert record.latest_version == "2.1.0" + assert record.deprecated == false + assert record.schema_data == Jason.encode!(schema) + + assert record.schema_sha256 == + :crypto.hash(:sha256, Jason.encode!(schema)) + |> Base.encode16(case: :lower) + + refute Map.has_key?(record, :icon_square_ext), + "fetch_adaptor/1 no longer carries icon fields — the Scheduler joins them" + + refute Map.has_key?(record, :icon_rectangle_ext) + refute Map.has_key?(record, :icon_square_sha256) + refute Map.has_key?(record, :icon_rectangle_sha256) + + refute Map.has_key?(record, :source), + "the strategy must not stamp :source — the Store owns that field" + + assert [version] = record.versions + assert version.version == "2.1.0" + assert version.dependencies == %{"axios" => "^1.5.0"} + + assert version.peer_dependencies == %{ + "@openfn/language-common" => "^2.0.0" + } + + assert version.integrity == nil + assert version.tarball_url == nil + assert version.size_bytes == nil + assert version.published_at == nil + assert version.deprecated == false + end + + test "reads schema_data from the latest version's directory specifically", + %{root: root} do + old_dir = + write_package_raw!(root, "http-old", %{ + "name" => "@openfn/language-http", + "version" => "1.0.0" + }) + + new_dir = + write_package_raw!(root, "http-new", %{ + "name" => "@openfn/language-http", + "version" => "2.0.0" + }) + + File.write!( + Path.join(old_dir, "configuration-schema.json"), + Jason.encode!(%{"version" => "old"}) + ) + + File.write!( + Path.join(new_dir, "configuration-schema.json"), + Jason.encode!(%{"version" => "new"}) + ) + + {:ok, record} = Local.fetch_adaptor("@openfn/language-http") + + assert record.schema_data == Jason.encode!(%{"version" => "new"}) + assert record.latest_version == "2.0.0" + end + + test "returns nil-shaped schema fields when files are absent", + %{root: root} do + write_package!(root, "bare", "@openfn/language-bare", "1.0.0") + + {:ok, record} = Local.fetch_adaptor("@openfn/language-bare") + + assert record.schema_data == nil + assert record.schema_sha256 == nil + end + + test "handles a plain-string repository field", %{root: root} do + write_package_raw!(root, "p", %{ + "name" => "@openfn/language-p", + "version" => "1.0.0", + "repository" => "https://github.com/example/p" + }) + + {:ok, record} = Local.fetch_adaptor("@openfn/language-p") + assert record.repository == "https://github.com/example/p" + end + + test "lists every on-disk version in :versions", %{root: root} do + write_package!(root, "http-1", "@openfn/language-http", "1.0.0") + write_package!(root, "http-2", "@openfn/language-http", "2.3.4") + write_package!(root, "http-3", "@openfn/language-http", "2.3.1") + + {:ok, record} = Local.fetch_adaptor("@openfn/language-http") + + versions = Enum.map(record.versions, & &1.version) + assert versions == ["2.3.4", "2.3.1", "1.0.0"] + end + + test "returns {:error, :not_found} for an unknown package", %{root: root} do + write_package!(root, "p", "@openfn/language-p", "1.0.0") + + assert Local.fetch_adaptor("@openfn/language-missing") == + {:error, :not_found} + end + end + + describe "fetch_icon/2" do + test "reads an icon from the latest version's assets dir", %{root: root} do + dir = write_package!(root, "http", "@openfn/language-http", "1.0.0") + write_icon!(dir, :square, "png", "PNGDATA") + + assert {:ok, %{data: "PNGDATA", ext: "png"}} = + Local.fetch_icon("@openfn/language-http", :square) + end + + test "prefers the latest version when multiple version dirs exist", + %{root: root} do + old_dir = + write_package_raw!(root, "http-old", %{ + "name" => "@openfn/language-http", + "version" => "1.0.0" + }) + + new_dir = + write_package_raw!(root, "http-new", %{ + "name" => "@openfn/language-http", + "version" => "2.0.0" + }) + + write_icon!(old_dir, :square, "png", "OLD") + write_icon!(new_dir, :square, "png", "NEW") + + assert {:ok, %{data: "NEW", ext: "png"}} = + Local.fetch_icon("@openfn/language-http", :square) + end + + test "falls back to svg when png is absent", %{root: root} do + dir = write_package!(root, "p", "@openfn/language-p", "1.0.0") + write_icon!(dir, :rectangle, "svg", "") + + assert {:ok, %{data: "", ext: "svg"}} = + Local.fetch_icon("@openfn/language-p", :rectangle) + end + + test "returns {:error, :not_found} when no icon variant exists", + %{root: root} do + write_package!(root, "p", "@openfn/language-p", "1.0.0") + + assert Local.fetch_icon("@openfn/language-p", :square) == + {:error, :not_found} + end + + test "returns {:error, :not_found} for an unknown package", %{root: root} do + write_package!(root, "p", "@openfn/language-p", "1.0.0") + + assert Local.fetch_icon("@openfn/language-missing", :square) == + {:error, :not_found} + end + end + + describe "fetch_icons/1" do + test "returns an entry per package per shape including sha256", + %{root: root} do + http = write_package!(root, "http", "@openfn/language-http", "1.0.0") + sf = write_package!(root, "sf", "@openfn/language-salesforce", "2.0.0") + + write_icon!(http, :square, "png", "HTTP_SQ") + write_icon!(http, :rectangle, "svg", "") + write_icon!(sf, :square, "png", "SF_SQ") + + {:ok, map} = Local.fetch_icons([]) + + assert %{ + "@openfn/language-http" => %{ + square: %{data: "HTTP_SQ", ext: "png", sha256: http_sq_sha}, + rectangle: %{data: "", ext: "svg"} + }, + "@openfn/language-salesforce" => %{ + square: %{data: "SF_SQ", ext: "png"} + } + } = map + + assert http_sq_sha == :crypto.hash(:sha256, "HTTP_SQ") + refute Map.has_key?(map["@openfn/language-salesforce"], :rectangle) + end + + test "returns {:ok, %{}} when no packages have icons", %{root: root} do + write_package!(root, "bare", "@openfn/language-bare", "1.0.0") + + assert {:ok, %{}} = Local.fetch_icons([]) + end + + test "returns {:error, :no_repo_path} when :path is unset" do + Application.delete_env(:lightning, Local) + + assert capture_log(fn -> + assert Local.fetch_icons([]) == {:error, :no_repo_path} + end) =~ "not configured" + end + end + + defp write_package!(root, dir_name, name, version) do + write_package_raw!(root, dir_name, %{"name" => name, "version" => version}) + end + + defp write_package_raw!(root, dir_name, package_json) do + dir = Path.join([root, "packages", dir_name]) + File.mkdir_p!(dir) + File.write!(Path.join(dir, "package.json"), Jason.encode!(package_json)) + dir + end + + defp write_icon!(dir, shape, ext, bytes) do + assets = Path.join(dir, "assets") + File.mkdir_p!(assets) + File.write!(Path.join(assets, "#{shape}.#{ext}"), bytes) + end +end diff --git a/test/lightning/adaptors/node_monitor_test.exs b/test/lightning/adaptors/node_monitor_test.exs new file mode 100644 index 00000000000..c0b212550c0 --- /dev/null +++ b/test/lightning/adaptors/node_monitor_test.exs @@ -0,0 +1,160 @@ +defmodule Lightning.Adaptors.NodeMonitorTest do + use Lightning.DataCase, async: true + + import Mox + + alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + setup :verify_on_exit! + + setup do + sup = :"nm_test_#{System.unique_integer([:positive])}" + + # The supervisor's :rest_for_one child list starts the NodeMonitor + # automatically — registered under `node_monitor_name(sup)`. + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + ) + + cache = AdaptorsSupervisor.cache_name(sup) + nm_name = AdaptorsSupervisor.node_monitor_name(sup) + + nm_pid = Process.whereis(nm_name) + Ecto.Adapters.SQL.Sandbox.allow(Lightning.Repo, self(), nm_pid) + + # Scheduler is auto-started too (wrapped in HighlanderPG, registered + # via :global). It may not be up yet at setup time — HighlanderPG + # polls at 300ms — so this is best-effort. + {:global, global_sched_name} = AdaptorsSupervisor.global_scheduler_name(sup) + sched_pid = :global.whereis_name(global_sched_name) + + if is_pid(sched_pid), + do: Ecto.Adapters.SQL.Sandbox.allow(Lightning.Repo, self(), sched_pid) + + {:ok, sup: sup, cache: cache, nm_name: nm_name} + end + + describe "start_link/1" do + test "registers under the :name opt", %{nm_name: nm_name} do + assert is_pid(Process.whereis(nm_name)) + end + end + + describe "handle_info/2 - {:nodeup, ...}" do + test "warms {:packages, source} and {:icon_meta, name, source} from Postgres", + %{ + sup: sup, + cache: cache, + nm_name: nm_name + } do + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + source = AdaptorsSupervisor.source(sup) + + send(nm_name, {:nodeup, :node@host, %{node_type: :visible}}) + :sys.get_state(nm_name) + + assert {:ok, {:ok, [pkg]}} = Cachex.get(cache, {:packages, source}) + assert pkg.name == "@openfn/language-http" + + assert {:ok, {:ok, icon_meta}} = + Cachex.get(cache, {:icon_meta, "@openfn/language-http", source}) + + assert Map.has_key?(icon_meta, :icon_square_ext) + end + + test "uses put_many; pre-existing schema keys survive the warm", %{ + sup: sup, + cache: cache, + nm_name: nm_name + } do + source = AdaptorsSupervisor.source(sup) + + Cachex.put!( + cache, + {:schema, "pre-existing", source}, + {:ok, %{"kept" => true}} + ) + + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + + send(nm_name, {:nodeup, :node@host, %{node_type: :visible}}) + :sys.get_state(nm_name) + + assert {:ok, {:ok, %{"kept" => true}}} = + Cachex.get(cache, {:schema, "pre-existing", source}) + end + + test "warm covers only the active source; :local keys are not materialised in npm mode", + %{ + cache: cache, + nm_name: nm_name + } do + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + + send(nm_name, {:nodeup, :node@host, %{node_type: :visible}}) + :sys.get_state(nm_name) + + assert {:ok, nil} = Cachex.get(cache, {:packages, :local}) + end + end + + describe "handle_info/2 - {:nodedown, ...}" do + test "nodedown is a no-op; cache and state are unchanged", %{ + sup: sup, + cache: cache, + nm_name: nm_name + } do + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + source = AdaptorsSupervisor.source(sup) + Cachex.put!(cache, {:packages, source}, {:ok, [%{name: "sentinel"}]}) + + send(nm_name, {:nodedown, :node@host, %{node_type: :visible}}) + :sys.get_state(nm_name) + + assert {:ok, {:ok, [%{name: "sentinel"}]}} = + Cachex.get(cache, {:packages, source}) + end + end + + defp adaptor_record(overrides \\ []) do + overrides = Map.new(overrides) + + %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + description: "HTTP adaptor", + homepage: nil, + repository: nil, + license: "LGPL-3.0", + deprecated: false, + schema_data: nil, + schema_sha256: nil, + icon_square_ext: nil, + icon_rectangle_ext: nil, + icon_square_sha256: nil, + icon_rectangle_sha256: nil, + versions: [ + %{ + version: "1.0.0", + integrity: "sha512-1.0.0", + tarball_url: "https://example.com/x/-/x-1.0.0.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + ] + } + |> Map.merge(overrides) + end +end diff --git a/test/lightning/adaptors/npm/github_test.exs b/test/lightning/adaptors/npm/github_test.exs new file mode 100644 index 00000000000..3f09b0ae966 --- /dev/null +++ b/test/lightning/adaptors/npm/github_test.exs @@ -0,0 +1,390 @@ +defmodule Lightning.Adaptors.NPM.GitHubTest do + use ExUnit.Case, async: false + + alias Lightning.Adaptors.NPM.GitHub + + setup do + bypass = Bypass.open() + + Application.put_env(:lightning, Lightning.Adaptors.NPM, + github_url: "http://localhost:#{bypass.port}", + github_ref: "main", + http_timeout: 1_000 + ) + + prev_adapter = Application.get_env(:tesla, :adapter) + + Application.put_env( + :tesla, + :adapter, + {Tesla.Adapter.Finch, name: Lightning.Finch} + ) + + on_exit(fn -> + Application.delete_env(:lightning, Lightning.Adaptors.NPM) + + if prev_adapter do + Application.put_env(:tesla, :adapter, prev_adapter) + else + Application.delete_env(:tesla, :adapter) + end + end) + + %{bypass: bypass} + end + + describe "fetch_one/2" do + test "returns png bytes on a happy-path 200", %{bypass: bypass} do + Bypass.expect( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/square.png", + fn conn -> Plug.Conn.resp(conn, 200, "SQUARE_PNG_BYTES") end + ) + + assert {:ok, %{data: "SQUARE_PNG_BYTES", ext: "png"}} = + GitHub.fetch_one("@openfn/language-http", :square) + end + + test "falls back to svg when png is missing", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> + case conn.request_path do + "/OpenFn/adaptors/main/packages/http/assets/rectangle.png" -> + Plug.Conn.resp(conn, 404, "") + + "/OpenFn/adaptors/main/packages/http/assets/rectangle.svg" -> + Plug.Conn.resp(conn, 200, "") + + path -> + raise "unexpected path: #{path}" + end + end) + + assert {:ok, %{data: "", ext: "svg"}} = + GitHub.fetch_one("@openfn/language-http", :rectangle) + end + + test "returns {:error, :not_found} when both exts 404", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 404, "") end) + + assert {:error, :not_found} = + GitHub.fetch_one("@openfn/language-missing", :square) + end + + test "surfaces 5xx as {:error, {:http_status, status}}", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 503, "") end) + + assert {:error, {:http_status, 503}} = + GitHub.fetch_one("@openfn/language-http", :square) + end + + test "surfaces network failure as {:error, _}", %{bypass: bypass} do + Bypass.down(bypass) + + assert {:error, _reason} = + GitHub.fetch_one("@openfn/language-http", :square) + end + + test "surfaces :http_timeout expiry as {:error, _}", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> + Bypass.pass(bypass) + Process.sleep(1_500) + Plug.Conn.resp(conn, 200, "should not arrive") + end) + + assert {:error, _reason} = + GitHub.fetch_one("@openfn/language-http", :square) + end + + test "strips the @openfn/language- prefix from the URL path", %{ + bypass: bypass + } do + Bypass.expect( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/salesforce/assets/square.png", + fn conn -> Plug.Conn.resp(conn, 200, "OK") end + ) + + assert {:ok, %{ext: "png"}} = + GitHub.fetch_one("@openfn/language-salesforce", :square) + end + + test "honours the configured :github_ref", %{bypass: bypass} do + Application.put_env(:lightning, Lightning.Adaptors.NPM, + github_url: "http://localhost:#{bypass.port}", + github_ref: "v2", + http_timeout: 1_000 + ) + + Bypass.expect( + bypass, + "GET", + "/OpenFn/adaptors/v2/packages/http/assets/square.png", + fn conn -> Plug.Conn.resp(conn, 200, "REF_BYTES") end + ) + + assert {:ok, %{data: "REF_BYTES"}} = + GitHub.fetch_one("@openfn/language-http", :square) + end + end + + describe "fetch_all/2" do + test "returns an entry per package per shape including sha256", %{ + bypass: bypass + } do + Bypass.expect(bypass, fn conn -> + case conn.request_path do + "/OpenFn/adaptors/main/packages/http/assets/square.png" -> + Plug.Conn.resp(conn, 200, "HTTP_SQ") + + "/OpenFn/adaptors/main/packages/http/assets/rectangle.png" -> + Plug.Conn.resp(conn, 200, "HTTP_RECT") + + "/OpenFn/adaptors/main/packages/salesforce/assets/square.png" -> + Plug.Conn.resp(conn, 200, "SF_SQ") + + "/OpenFn/adaptors/main/packages/salesforce/assets/rectangle.png" -> + Plug.Conn.resp(conn, 200, "SF_RECT") + + _ -> + Plug.Conn.resp(conn, 404, "") + end + end) + + {:ok, map} = + GitHub.fetch_all( + [ + "@openfn/language-http", + "@openfn/language-salesforce" + ], + %{} + ) + + assert %{ + "@openfn/language-http" => %{ + square: %{data: "HTTP_SQ", ext: "png", sha256: http_sq_sha}, + rectangle: %{data: "HTTP_RECT", ext: "png"} + }, + "@openfn/language-salesforce" => %{ + square: %{data: "SF_SQ", ext: "png"}, + rectangle: %{data: "SF_RECT", ext: "png"} + } + } = map + + assert http_sq_sha == :crypto.hash(:sha256, "HTTP_SQ") + end + + test "absent packages and missing shapes are simply absent from the map", + %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> + case conn.request_path do + "/OpenFn/adaptors/main/packages/http/assets/square.png" -> + Plug.Conn.resp(conn, 200, "ONLY_SQ") + + _ -> + Plug.Conn.resp(conn, 404, "") + end + end) + + {:ok, map} = + GitHub.fetch_all( + [ + "@openfn/language-http", + "@openfn/language-missing" + ], + %{} + ) + + assert Map.keys(map) == ["@openfn/language-http"] + assert Map.keys(map["@openfn/language-http"]) == [:square] + end + + test "returns {:ok, %{}} when the upstream is unreachable", %{ + bypass: bypass + } do + Bypass.down(bypass) + + assert {:ok, map} = + GitHub.fetch_all( + [ + "@openfn/language-http", + "@openfn/language-salesforce" + ], + %{} + ) + + assert map == %{} + end + + test "sends If-None-Match when a prior etag is supplied", %{bypass: bypass} do + etag = ~s(W/"abc") + + Bypass.expect( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/square.png", + fn conn -> + assert Plug.Conn.get_req_header(conn, "if-none-match") == [etag] + Plug.Conn.resp(conn, 304, "") + end + ) + + Bypass.stub( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/rectangle.png", + fn conn -> + Plug.Conn.resp(conn, 404, "") + end + ) + + Bypass.stub( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/rectangle.svg", + fn conn -> + Plug.Conn.resp(conn, 404, "") + end + ) + + {:ok, map} = + GitHub.fetch_all( + ["@openfn/language-http"], + %{"@openfn/language-http" => %{square: etag}} + ) + + assert get_in(map, ["@openfn/language-http", :square]) == :not_modified + end + + test "sends no If-None-Match when no prior etag", %{bypass: bypass} do + Bypass.expect( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/square.png", + fn conn -> + assert Plug.Conn.get_req_header(conn, "if-none-match") == [] + + conn + |> Plug.Conn.put_resp_header("etag", "test-etag-abc") + |> Plug.Conn.resp(200, "SQ_BYTES") + end + ) + + Bypass.stub( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/rectangle.png", + fn conn -> + Plug.Conn.resp(conn, 404, "") + end + ) + + Bypass.stub( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/rectangle.svg", + fn conn -> + Plug.Conn.resp(conn, 404, "") + end + ) + + {:ok, map} = GitHub.fetch_all(["@openfn/language-http"], %{}) + + assert %{ + data: "SQ_BYTES", + ext: "png", + etag: "test-etag-abc" + } = get_in(map, ["@openfn/language-http", :square]) + end + + test "304 short-circuits with the :not_modified sentinel in the slot", %{ + bypass: bypass + } do + etag = ~s("xyz") + + Bypass.expect( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/square.png", + fn conn -> + assert Plug.Conn.get_req_header(conn, "if-none-match") == [etag] + Plug.Conn.resp(conn, 304, "") + end + ) + + Bypass.stub( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/rectangle.png", + fn conn -> + Plug.Conn.resp(conn, 404, "") + end + ) + + Bypass.stub( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/rectangle.svg", + fn conn -> + Plug.Conn.resp(conn, 404, "") + end + ) + + {:ok, map} = + GitHub.fetch_all( + ["@openfn/language-http"], + %{"@openfn/language-http" => %{square: etag}} + ) + + # explicit sentinel — distinct from "absent" (which would mean upstream + # had no such shape at all). + assert map["@openfn/language-http"][:square] == :not_modified + end + + test "200 with a new etag overrides the prior", %{bypass: bypass} do + prior = ~s("old") + fresh = ~s("new") + + Bypass.expect( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/square.png", + fn conn -> + assert Plug.Conn.get_req_header(conn, "if-none-match") == [prior] + + conn + |> Plug.Conn.put_resp_header("etag", fresh) + |> Plug.Conn.resp(200, "FRESH") + end + ) + + Bypass.stub( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/rectangle.png", + fn conn -> + Plug.Conn.resp(conn, 404, "") + end + ) + + Bypass.stub( + bypass, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/rectangle.svg", + fn conn -> + Plug.Conn.resp(conn, 404, "") + end + ) + + {:ok, map} = + GitHub.fetch_all( + ["@openfn/language-http"], + %{"@openfn/language-http" => %{square: prior}} + ) + + assert %{data: "FRESH", ext: "png", etag: ^fresh} = + map["@openfn/language-http"][:square] + end + end +end diff --git a/test/lightning/adaptors/npm/registry_test.exs b/test/lightning/adaptors/npm/registry_test.exs new file mode 100644 index 00000000000..760b4891264 --- /dev/null +++ b/test/lightning/adaptors/npm/registry_test.exs @@ -0,0 +1,199 @@ +defmodule Lightning.Adaptors.NPM.RegistryTest do + use ExUnit.Case, async: false + + alias Lightning.Adaptors.NPM.Registry + + setup do + bypass = Bypass.open() + + Application.put_env(:lightning, Lightning.Adaptors.NPM, + registry_url: "http://localhost:#{bypass.port}", + http_timeout: 1_000 + ) + + # Per-test Tesla adapter override — config/test.exs globally pins + # `Lightning.Tesla.Mock`, but we need the real Finch adapter so that + # Bypass actually receives requests over a socket. + prev_adapter = Application.get_env(:tesla, :adapter) + + Application.put_env( + :tesla, + :adapter, + {Tesla.Adapter.Finch, name: Lightning.Finch} + ) + + on_exit(fn -> + Application.delete_env(:lightning, Lightning.Adaptors.NPM) + + if prev_adapter do + Application.put_env(:tesla, :adapter, prev_adapter) + else + Application.delete_env(:tesla, :adapter) + end + end) + + %{bypass: bypass} + end + + describe "list_adaptors/0" do + test "returns an empty list when the search has no results", %{ + bypass: bypass + } do + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> + conn = Plug.Conn.fetch_query_params(conn) + assert conn.query_params["text"] == "@openfn" + assert conn.query_params["size"] == "250" + + json_resp(conn, 200, %{"objects" => []}) + end) + + assert {:ok, []} = Registry.list_adaptors() + end + + test "returns name + latest_version for each search hit", %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> + body = %{ + "objects" => [ + %{ + "package" => %{ + "name" => "@openfn/language-http", + "version" => "2.1.0" + } + }, + %{ + "package" => %{ + "name" => "@openfn/language-salesforce", + "version" => "4.6.3" + } + } + ] + } + + json_resp(conn, 200, body) + end) + + {:ok, listing} = Registry.list_adaptors() + + assert Enum.sort_by(listing, & &1.name) == [ + %{name: "@openfn/language-http", latest_version: "2.1.0"}, + %{name: "@openfn/language-salesforce", latest_version: "4.6.3"} + ] + end + + test "filters out @openfn/* packages that aren't language-* adaptors and other scopes", + %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> + body = %{ + "objects" => [ + %{ + "package" => %{ + "name" => "@openfn/language-http", + "version" => "1.0.0" + } + }, + %{ + "package" => %{"name" => "@openfn/cli", "version" => "1.2.3"} + }, + %{ + "package" => %{ + "name" => "@openfn/buildtools", + "version" => "0.9.0" + } + }, + %{ + "package" => %{ + "name" => "@sid-indonesia/language-http", + "version" => "2.0.0" + } + }, + %{ + "package" => %{ + "name" => "language-template", + "version" => "3.0.0" + } + } + ] + } + + json_resp(conn, 200, body) + end) + + assert {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} = + Registry.list_adaptors() + end + + test "skips malformed entries that lack name or version", %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> + body = %{ + "objects" => [ + %{ + "package" => %{ + "name" => "@openfn/language-http", + "version" => "1.0.0" + } + }, + %{"package" => %{"name" => "@openfn/language-no-version"}}, + %{"score" => %{"final" => 0.5}} + ] + } + + json_resp(conn, 200, body) + end) + + assert {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} = + Registry.list_adaptors() + end + + test "surfaces 5xx responses as {:error, _}", %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> + Plug.Conn.resp(conn, 503, "") + end) + + assert {:error, {:http_status, 503}} = Registry.list_adaptors() + end + + test "surfaces network failure as {:error, _}", %{bypass: bypass} do + Bypass.down(bypass) + assert {:error, _reason} = Registry.list_adaptors() + end + end + + describe "get_packument/1" do + test "returns the decoded map on 200", %{bypass: bypass} do + packument = %{ + "name" => "@openfn/language-http", + "dist-tags" => %{"latest" => "2.1.0"} + } + + Bypass.expect(bypass, "GET", "/@openfn/language-http", fn conn -> + json_resp(conn, 200, packument) + end) + + assert {:ok, ^packument} = Registry.get_packument("@openfn/language-http") + end + + test "returns {:error, :not_found} on 404", %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/@openfn/language-missing", fn conn -> + Plug.Conn.resp(conn, 404, "") + end) + + assert {:error, :not_found} = + Registry.get_packument("@openfn/language-missing") + end + + test "surfaces 5xx as {:error, {:http_status, status}}", %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/@openfn/language-http", fn conn -> + Plug.Conn.resp(conn, 502, "") + end) + + assert {:error, {:http_status, 502}} = + Registry.get_packument("@openfn/language-http") + end + end + + defp json_resp(conn, status, body) do + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.resp(status, Jason.encode!(body)) + end +end diff --git a/test/lightning/adaptors/npm/schema_test.exs b/test/lightning/adaptors/npm/schema_test.exs new file mode 100644 index 00000000000..958fd953a2c --- /dev/null +++ b/test/lightning/adaptors/npm/schema_test.exs @@ -0,0 +1,86 @@ +defmodule Lightning.Adaptors.NPM.SchemaTest do + use ExUnit.Case, async: false + + alias Lightning.Adaptors.NPM.Schema + + @package "@openfn/language-http" + @version "2.1.0" + @path "/npm/#{@package}@#{@version}/configuration-schema.json" + + setup do + bypass = Bypass.open() + + Application.put_env(:lightning, Lightning.Adaptors.NPM, + jsdelivr_url: "http://localhost:#{bypass.port}", + http_timeout: 1_000 + ) + + prev_adapter = Application.get_env(:tesla, :adapter) + + Application.put_env( + :tesla, + :adapter, + {Tesla.Adapter.Finch, name: Lightning.Finch} + ) + + on_exit(fn -> + Application.delete_env(:lightning, Lightning.Adaptors.NPM) + + if prev_adapter do + Application.put_env(:tesla, :adapter, prev_adapter) + else + Application.delete_env(:tesla, :adapter) + end + end) + + %{bypass: bypass} + end + + describe "schema/2" do + test "returns the decoded schema and a hex sha256 on 200", %{bypass: bypass} do + schema = %{"type" => "object", "properties" => %{"baseUrl" => %{}}} + body = Jason.encode!(schema) + + expected_sha = + :sha256 + |> :crypto.hash(body) + |> Base.encode16(case: :lower) + + Bypass.expect(bypass, "GET", @path, fn conn -> + assert conn.request_path == @path + Plug.Conn.resp(conn, 200, body) + end) + + assert {^schema, ^expected_sha} = Schema.schema(@package, @version) + end + + test "returns {nil, nil} on 404", %{bypass: bypass} do + Bypass.expect(bypass, "GET", @path, fn conn -> + Plug.Conn.resp(conn, 404, "") + end) + + assert {nil, nil} = Schema.schema(@package, @version) + end + + test "returns {nil, nil} on 5xx", %{bypass: bypass} do + Bypass.expect(bypass, "GET", @path, fn conn -> + Plug.Conn.resp(conn, 500, "") + end) + + assert {nil, nil} = Schema.schema(@package, @version) + end + + test "returns {nil, nil} on invalid JSON body", %{bypass: bypass} do + Bypass.expect(bypass, "GET", @path, fn conn -> + Plug.Conn.resp(conn, 200, "this is not json {") + end) + + assert {nil, nil} = Schema.schema(@package, @version) + end + + test "returns {nil, nil} on connection refused", %{bypass: bypass} do + Bypass.down(bypass) + assert {nil, nil} = Schema.schema(@package, @version) + end + end +end diff --git a/test/lightning/adaptors/npm_test.exs b/test/lightning/adaptors/npm_test.exs new file mode 100644 index 00000000000..ad2ea8ded67 --- /dev/null +++ b/test/lightning/adaptors/npm_test.exs @@ -0,0 +1,270 @@ +defmodule Lightning.Adaptors.NPMTest do + use ExUnit.Case, async: false + + alias Lightning.Adaptors.NPM + + @package "@openfn/language-http" + @latest_version "2.1.0" + + # Three Bypass servers: one for the npm registry, one for jsDelivr, + # one for raw.githubusercontent.com. Per-test config installs all three + # URLs onto the strategy_opts block. + setup do + registry = Bypass.open() + jsdelivr = Bypass.open() + github = Bypass.open() + + Application.put_env(:lightning, Lightning.Adaptors.NPM, + registry_url: "http://localhost:#{registry.port}", + jsdelivr_url: "http://localhost:#{jsdelivr.port}", + github_url: "http://localhost:#{github.port}", + github_ref: "main", + http_timeout: 1_000 + ) + + prev_adapter = Application.get_env(:tesla, :adapter) + + Application.put_env( + :tesla, + :adapter, + {Tesla.Adapter.Finch, name: Lightning.Finch} + ) + + on_exit(fn -> + Application.delete_env(:lightning, Lightning.Adaptors.NPM) + + if prev_adapter do + Application.put_env(:tesla, :adapter, prev_adapter) + else + Application.delete_env(:tesla, :adapter) + end + end) + + %{registry: registry, jsdelivr: jsdelivr, github: github} + end + + describe "fetch_adaptor/1" do + test "decodes a packument into the icon-free adaptor_record shape", %{ + registry: registry, + jsdelivr: jsdelivr + } do + schema = %{"type" => "object", "properties" => %{"baseUrl" => %{}}} + schema_bytes = Jason.encode!(schema) + packument = build_packument() + + Bypass.expect(registry, "GET", "/" <> @package, fn conn -> + json_resp(conn, 200, packument) + end) + + Bypass.expect( + jsdelivr, + "GET", + "/npm/#{@package}@#{@latest_version}/configuration-schema.json", + fn conn -> Plug.Conn.resp(conn, 200, schema_bytes) end + ) + + {:ok, record} = NPM.fetch_adaptor(@package) + + expected_schema_sha = + :sha256 |> :crypto.hash(schema_bytes) |> Base.encode16(case: :lower) + + assert %{ + name: @package, + description: "HTTP adaptor", + homepage: "https://docs.openfn.org/adaptors/http", + repository: "git+https://github.com/OpenFn/adaptors.git", + license: "LGPL-3.0", + latest_version: @latest_version, + deprecated: false, + schema_data: ^schema_bytes, + schema_sha256: ^expected_schema_sha + } = record + + refute Map.has_key?(record, :icon_square_ext), + "fetch_adaptor/1 no longer carries icon fields — the Scheduler joins them" + + refute Map.has_key?(record, :icon_rectangle_ext) + refute Map.has_key?(record, :icon_square_sha256) + refute Map.has_key?(record, :icon_rectangle_sha256) + + refute Map.has_key?(record, :source), + "strategy must not stamp :source — the Store owns that field" + + assert length(record.versions) == 2 + + latest = Enum.find(record.versions, &(&1.version == @latest_version)) + + assert %{ + integrity: "sha512-abc", + size_bytes: 12_345, + dependencies: %{"axios" => "^1.5.0"}, + peer_dependencies: %{"@openfn/language-common" => "^2.0.0"}, + deprecated: false + } = latest + + assert %DateTime{} = latest.published_at + assert DateTime.to_iso8601(latest.published_at) =~ "2024-06-01" + + old = Enum.find(record.versions, &(&1.version == "1.0.0")) + assert old.integrity == "sha512-old" + assert old.dependencies == %{} + assert old.deprecated == true + end + + test "degrades to nil schema when jsDelivr returns 5xx", %{ + registry: registry, + jsdelivr: jsdelivr + } do + packument = build_packument() + + Bypass.expect(registry, "GET", "/" <> @package, fn conn -> + json_resp(conn, 200, packument) + end) + + Bypass.expect(jsdelivr, fn conn -> + Plug.Conn.resp(conn, 500, "") + end) + + {:ok, record} = NPM.fetch_adaptor(@package) + + assert record.schema_data == nil + assert record.schema_sha256 == nil + assert record.name == @package + assert record.latest_version == @latest_version + end + end + + describe "fetch_icon/2" do + test "delegates to NPM.GitHub for raw icon bytes", %{github: github} do + Bypass.expect( + github, + "GET", + "/OpenFn/adaptors/main/packages/http/assets/square.png", + fn conn -> Plug.Conn.resp(conn, 200, "PNG_PAYLOAD") end + ) + + assert {:ok, %{data: "PNG_PAYLOAD", ext: "png"}} = + NPM.fetch_icon(@package, :square) + end + + test "returns {:error, :not_found} when both png and svg 404", %{ + github: github + } do + Bypass.expect(github, fn conn -> Plug.Conn.resp(conn, 404, "") end) + + assert {:error, :not_found} = + NPM.fetch_icon("@openfn/language-missing", :square) + end + + test "surfaces transport failure as {:error, _}", %{github: github} do + Bypass.down(github) + + assert {:error, _reason} = NPM.fetch_icon(@package, :square) + end + end + + describe "fetch_icons/1" do + test "lists adaptors then fans out to GitHub raw fetches", %{ + registry: registry, + github: github + } do + Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> + body = %{ + "objects" => [ + %{ + "package" => %{ + "name" => "@openfn/language-http", + "version" => "2.1.0" + } + }, + %{ + "package" => %{ + "name" => "@openfn/language-salesforce", + "version" => "4.6.3" + } + } + ] + } + + json_resp(conn, 200, body) + end) + + Bypass.expect(github, fn conn -> + case conn.request_path do + "/OpenFn/adaptors/main/packages/http/assets/square.png" -> + Plug.Conn.resp(conn, 200, "HTTP_SQ") + + "/OpenFn/adaptors/main/packages/salesforce/assets/square.png" -> + Plug.Conn.resp(conn, 200, "SF_SQ") + + _ -> + Plug.Conn.resp(conn, 404, "") + end + end) + + {:ok, icons} = NPM.fetch_icons([]) + + assert %{ + "@openfn/language-http" => %{ + square: %{data: "HTTP_SQ", ext: "png"} + }, + "@openfn/language-salesforce" => %{ + square: %{data: "SF_SQ", ext: "png"} + } + } = icons + + assert icons["@openfn/language-http"].square.sha256 == + :crypto.hash(:sha256, "HTTP_SQ") + end + + test "surfaces list_adaptors errors as {:error, _}", %{registry: registry} do + Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> + Plug.Conn.resp(conn, 503, "") + end) + + assert {:error, _} = NPM.fetch_icons([]) + end + end + + # ==================== Helpers ==================== + + defp build_packument do + %{ + "name" => @package, + "description" => "HTTP adaptor", + "homepage" => "https://docs.openfn.org/adaptors/http", + "repository" => %{"url" => "git+https://github.com/OpenFn/adaptors.git"}, + "license" => "LGPL-3.0", + "dist-tags" => %{"latest" => @latest_version}, + "time" => %{ + "1.0.0" => "2023-01-01T00:00:00.000Z", + "2.1.0" => "2024-06-01T12:00:00.000Z" + }, + "versions" => %{ + "1.0.0" => %{ + "dependencies" => %{}, + "peerDependencies" => %{}, + "deprecated" => "please upgrade", + "dist" => %{ + "integrity" => "sha512-old", + "unpackedSize" => 5_000 + } + }, + @latest_version => %{ + "dependencies" => %{"axios" => "^1.5.0"}, + "peerDependencies" => %{"@openfn/language-common" => "^2.0.0"}, + "dist" => %{ + "integrity" => "sha512-abc", + "unpackedSize" => 12_345 + } + } + } + } + end + + defp json_resp(conn, status, body) do + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.resp(status, Jason.encode!(body)) + end +end diff --git a/test/lightning/adaptors/repo_adaptor_test.exs b/test/lightning/adaptors/repo_adaptor_test.exs new file mode 100644 index 00000000000..e77868fa6cd --- /dev/null +++ b/test/lightning/adaptors/repo_adaptor_test.exs @@ -0,0 +1,315 @@ +defmodule Lightning.Adaptors.Repo.AdaptorTest do + use ExUnit.Case, async: true + + alias Lightning.Adaptors.Repo.Adaptor + + @valid_attrs %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.2.3", + checked_at: ~U[2026-05-14 00:00:00.000000Z] + } + + describe "changeset/2 — required fields" do + test "is valid with the minimum required set" do + changeset = Adaptor.changeset(%Adaptor{}, @valid_attrs) + assert changeset.valid? + end + + test "requires :name" do + changeset = + Adaptor.changeset(%Adaptor{}, Map.delete(@valid_attrs, :name)) + + refute changeset.valid? + assert "can't be blank" in errors_on(changeset, :name) + end + + test "requires :source" do + changeset = + Adaptor.changeset(%Adaptor{}, Map.delete(@valid_attrs, :source)) + + refute changeset.valid? + assert "can't be blank" in errors_on(changeset, :source) + end + + test "requires :latest_version" do + changeset = + Adaptor.changeset( + %Adaptor{}, + Map.delete(@valid_attrs, :latest_version) + ) + + refute changeset.valid? + assert "can't be blank" in errors_on(changeset, :latest_version) + end + + test "requires :checked_at" do + changeset = + Adaptor.changeset(%Adaptor{}, Map.delete(@valid_attrs, :checked_at)) + + refute changeset.valid? + assert "can't be blank" in errors_on(changeset, :checked_at) + end + end + + describe "changeset/2 — :name length cap" do + test "rejects :name longer than 214 characters (npm pkg limit)" do + too_long = String.duplicate("a", 215) + + changeset = + Adaptor.changeset(%Adaptor{}, %{@valid_attrs | name: too_long}) + + refute changeset.valid? + + assert Enum.any?( + errors_on(changeset, :name), + &(&1 =~ "should be at most") + ) + end + + test "accepts :name of exactly 214 characters" do + ok = String.duplicate("a", 214) + + assert Adaptor.changeset(%Adaptor{}, %{@valid_attrs | name: ok}).valid? + end + end + + describe "changeset/2 — :source Ecto.Enum cast" do + test "round-trips atom :npm" do + changeset = + Adaptor.changeset(%Adaptor{}, %{@valid_attrs | source: :npm}) + + assert changeset.valid? + assert Ecto.Changeset.get_change(changeset, :source) == :npm + end + + test "round-trips atom :local" do + changeset = + Adaptor.changeset(%Adaptor{}, %{@valid_attrs | source: :local}) + + assert changeset.valid? + assert Ecto.Changeset.get_change(changeset, :source) == :local + end + + test "casts the string form \"npm\" back to the :npm atom" do + changeset = + Adaptor.changeset(%Adaptor{}, %{@valid_attrs | source: "npm"}) + + assert changeset.valid? + assert Ecto.Changeset.get_change(changeset, :source) == :npm + end + + test "casts the string form \"local\" back to the :local atom" do + changeset = + Adaptor.changeset(%Adaptor{}, %{@valid_attrs | source: "local"}) + + assert changeset.valid? + assert Ecto.Changeset.get_change(changeset, :source) == :local + end + + test "rejects an unknown source atom" do + changeset = + Adaptor.changeset(%Adaptor{}, %{@valid_attrs | source: :other}) + + refute changeset.valid? + assert "is invalid" in errors_on(changeset, :source) + end + + test "rejects an unknown source string" do + changeset = + Adaptor.changeset(%Adaptor{}, %{@valid_attrs | source: "other"}) + + refute changeset.valid? + assert "is invalid" in errors_on(changeset, :source) + end + end + + describe "changeset/2 — icon ext inclusion" do + test "accepts \"png\" for :icon_square_ext" do + attrs = + Map.merge(@valid_attrs, %{ + icon_square_ext: "png", + icon_square_sha256: :crypto.strong_rand_bytes(32) + }) + + assert Adaptor.changeset(%Adaptor{}, attrs).valid? + end + + test "accepts \"svg\" for :icon_square_ext" do + attrs = + Map.merge(@valid_attrs, %{ + icon_square_ext: "svg", + icon_square_sha256: :crypto.strong_rand_bytes(32) + }) + + assert Adaptor.changeset(%Adaptor{}, attrs).valid? + end + + test "rejects an unknown :icon_square_ext" do + attrs = + Map.merge(@valid_attrs, %{ + icon_square_ext: "gif", + icon_square_sha256: :crypto.strong_rand_bytes(32) + }) + + changeset = Adaptor.changeset(%Adaptor{}, attrs) + refute changeset.valid? + assert "is invalid" in errors_on(changeset, :icon_square_ext) + end + + test "accepts \"png\" for :icon_rectangle_ext" do + attrs = + Map.merge(@valid_attrs, %{ + icon_rectangle_ext: "png", + icon_rectangle_sha256: :crypto.strong_rand_bytes(32) + }) + + assert Adaptor.changeset(%Adaptor{}, attrs).valid? + end + + test "accepts \"svg\" for :icon_rectangle_ext" do + attrs = + Map.merge(@valid_attrs, %{ + icon_rectangle_ext: "svg", + icon_rectangle_sha256: :crypto.strong_rand_bytes(32) + }) + + assert Adaptor.changeset(%Adaptor{}, attrs).valid? + end + + test "rejects an unknown :icon_rectangle_ext" do + attrs = + Map.merge(@valid_attrs, %{ + icon_rectangle_ext: "jpeg", + icon_rectangle_sha256: :crypto.strong_rand_bytes(32) + }) + + changeset = Adaptor.changeset(%Adaptor{}, attrs) + refute changeset.valid? + assert "is invalid" in errors_on(changeset, :icon_rectangle_ext) + end + end + + describe "changeset/2 — validate_icon_sha256_pair (square)" do + test "accepts both nil (no icon)" do + attrs = + Map.merge(@valid_attrs, %{ + icon_square_ext: nil, + icon_square_sha256: nil + }) + + assert Adaptor.changeset(%Adaptor{}, attrs).valid? + end + + test "accepts both set" do + attrs = + Map.merge(@valid_attrs, %{ + icon_square_ext: "png", + icon_square_sha256: :crypto.strong_rand_bytes(32) + }) + + assert Adaptor.changeset(%Adaptor{}, attrs).valid? + end + + test "rejects ext set without sha256" do + attrs = + Map.merge(@valid_attrs, %{ + icon_square_ext: "png", + icon_square_sha256: nil + }) + + changeset = Adaptor.changeset(%Adaptor{}, attrs) + refute changeset.valid? + + assert Enum.any?( + errors_on(changeset, :icon_square_sha256), + &(&1 =~ "must not be nil") + ) + end + + test "rejects sha256 set without ext" do + attrs = + Map.merge(@valid_attrs, %{ + icon_square_ext: nil, + icon_square_sha256: :crypto.strong_rand_bytes(32) + }) + + changeset = Adaptor.changeset(%Adaptor{}, attrs) + refute changeset.valid? + + assert Enum.any?( + errors_on(changeset, :icon_square_sha256), + &(&1 =~ "must be nil") + ) + end + end + + describe "changeset/2 — validate_icon_sha256_pair (rectangle)" do + test "accepts both nil (no icon)" do + attrs = + Map.merge(@valid_attrs, %{ + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil + }) + + assert Adaptor.changeset(%Adaptor{}, attrs).valid? + end + + test "accepts both set" do + attrs = + Map.merge(@valid_attrs, %{ + icon_rectangle_ext: "svg", + icon_rectangle_sha256: :crypto.strong_rand_bytes(32) + }) + + assert Adaptor.changeset(%Adaptor{}, attrs).valid? + end + + test "rejects ext set without sha256" do + attrs = + Map.merge(@valid_attrs, %{ + icon_rectangle_ext: "svg", + icon_rectangle_sha256: nil + }) + + changeset = Adaptor.changeset(%Adaptor{}, attrs) + refute changeset.valid? + + assert Enum.any?( + errors_on(changeset, :icon_rectangle_sha256), + &(&1 =~ "must not be nil") + ) + end + + test "rejects sha256 set without ext" do + attrs = + Map.merge(@valid_attrs, %{ + icon_rectangle_ext: nil, + icon_rectangle_sha256: :crypto.strong_rand_bytes(32) + }) + + changeset = Adaptor.changeset(%Adaptor{}, attrs) + refute changeset.valid? + + assert Enum.any?( + errors_on(changeset, :icon_rectangle_sha256), + &(&1 =~ "must be nil") + ) + end + end + + describe "changeset/2 — unique_constraint" do + test "registers a unique_constraint on [:name, :source]" do + changeset = Adaptor.changeset(%Adaptor{}, @valid_attrs) + + assert Enum.any?(changeset.constraints, fn c -> + c.type == :unique and + c.constraint == "adaptors_name_source_index" + end) + end + end + + defp errors_on(changeset, field) do + for {f, {msg, _opts}} <- changeset.errors, f == field, do: msg + end +end diff --git a/test/lightning/adaptors/repo_adaptor_version_test.exs b/test/lightning/adaptors/repo_adaptor_version_test.exs new file mode 100644 index 00000000000..b9e9a52167a --- /dev/null +++ b/test/lightning/adaptors/repo_adaptor_version_test.exs @@ -0,0 +1,172 @@ +defmodule Lightning.Adaptors.Repo.AdaptorVersionTest do + use ExUnit.Case, async: true + + alias Lightning.Adaptors.Repo.AdaptorVersion + + @adaptor_id Ecto.UUID.generate() + + @valid_attrs %{ + adaptor_id: @adaptor_id, + version: "1.2.3" + } + + describe "changeset/2 — required fields" do + test "is valid with the minimum required set" do + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, @valid_attrs) + assert changeset.valid? + end + + test "requires :adaptor_id" do + changeset = + AdaptorVersion.changeset( + %AdaptorVersion{}, + Map.delete(@valid_attrs, :adaptor_id) + ) + + refute changeset.valid? + assert "can't be blank" in errors_on(changeset, :adaptor_id) + end + + test "requires :version" do + changeset = + AdaptorVersion.changeset( + %AdaptorVersion{}, + Map.delete(@valid_attrs, :version) + ) + + refute changeset.valid? + assert "can't be blank" in errors_on(changeset, :version) + end + end + + describe "changeset/2 — optional fields round-trip" do + test "casts :integrity" do + attrs = Map.put(@valid_attrs, :integrity, "sha512-abcdef==") + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) + + assert changeset.valid? + + assert Ecto.Changeset.get_change(changeset, :integrity) == + "sha512-abcdef==" + end + + test "casts :tarball_url" do + attrs = + Map.put( + @valid_attrs, + :tarball_url, + "https://registry.npmjs.org/x/-/x-1.2.3.tgz" + ) + + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) + + assert changeset.valid? + + assert Ecto.Changeset.get_change(changeset, :tarball_url) == + "https://registry.npmjs.org/x/-/x-1.2.3.tgz" + end + + test "casts :size_bytes" do + attrs = Map.put(@valid_attrs, :size_bytes, 12_345) + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) + + assert changeset.valid? + assert Ecto.Changeset.get_change(changeset, :size_bytes) == 12_345 + end + + test "casts :dependencies as a map (no structural validation)" do + deps = %{"axios" => "^1.0.0", "lodash" => "4.17.21"} + attrs = Map.put(@valid_attrs, :dependencies, deps) + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) + + assert changeset.valid? + assert Ecto.Changeset.get_change(changeset, :dependencies) == deps + end + + test "casts :peer_dependencies as a map (no structural validation)" do + peers = %{"react" => "^18.0.0"} + attrs = Map.put(@valid_attrs, :peer_dependencies, peers) + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) + + assert changeset.valid? + + assert Ecto.Changeset.get_change(changeset, :peer_dependencies) == + peers + end + + test "accepts an arbitrarily-shaped :dependencies map" do + weird = %{"a" => 1, "b" => %{"nested" => true}, "c" => nil} + attrs = Map.put(@valid_attrs, :dependencies, weird) + + assert AdaptorVersion.changeset(%AdaptorVersion{}, attrs).valid? + end + + test "casts :published_at" do + ts = ~U[2026-05-14 12:00:00.000000Z] + attrs = Map.put(@valid_attrs, :published_at, ts) + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) + + assert changeset.valid? + assert Ecto.Changeset.get_change(changeset, :published_at) == ts + end + + test "casts :deprecated" do + attrs = Map.put(@valid_attrs, :deprecated, true) + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) + + assert changeset.valid? + assert Ecto.Changeset.get_change(changeset, :deprecated) == true + end + + test "defaults :deprecated to false on a fresh struct" do + assert %AdaptorVersion{}.deprecated == false + end + end + + describe "changeset/2 — unique_constraint" do + test "registers a unique_constraint on [:adaptor_id, :version]" do + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, @valid_attrs) + + assert Enum.any?(changeset.constraints, fn c -> + c.type == :unique and + c.constraint == "adaptor_versions_adaptor_id_version_index" + end) + end + end + + describe "changeset/2 — FK constraint" do + test "registers a foreign_key constraint on the :adaptor association" do + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, @valid_attrs) + + assert Enum.any?(changeset.constraints, fn c -> + c.type == :foreign_key and c.field == :adaptor + end) + end + end + + describe "schema" do + test "belongs_to :adaptor uses binary_id" do + assoc = AdaptorVersion.__schema__(:association, :adaptor) + assert assoc.related == Lightning.Adaptors.Repo.Adaptor + assert assoc.owner_key == :adaptor_id + end + + test ":adaptor_id field is binary_id" do + assert AdaptorVersion.__schema__(:type, :adaptor_id) == :binary_id + end + + test ":id field is binary_id" do + assert AdaptorVersion.__schema__(:type, :id) == :binary_id + end + + test "has :inserted_at but not :updated_at" do + fields = AdaptorVersion.__schema__(:fields) + assert :inserted_at in fields + refute :updated_at in fields + end + end + + defp errors_on(changeset, field) do + for {f, {msg, _opts}} <- changeset.errors, f == field, do: msg + end +end diff --git a/test/lightning/adaptors/repo_test.exs b/test/lightning/adaptors/repo_test.exs new file mode 100644 index 00000000000..c56c9ec8162 --- /dev/null +++ b/test/lightning/adaptors/repo_test.exs @@ -0,0 +1,485 @@ +defmodule Lightning.Adaptors.RepoTest do + use Lightning.DataCase, async: true + + alias Lightning.Adaptors.Repo, as: AdaptorRepo + alias Lightning.Adaptors.Repo.Adaptor + alias Lightning.Adaptors.Repo.AdaptorVersion + + describe "upsert_adaptor/1 — initial insert" do + test "inserts the adaptor row and its versions in one transaction" do + record = + adaptor_record( + versions: [version_record("1.0.0"), version_record("1.1.0")] + ) + + assert {:ok, %Adaptor{} = adaptor} = AdaptorRepo.upsert_adaptor(record) + + assert adaptor.name == "@openfn/language-http" + assert adaptor.source == :npm + assert adaptor.latest_version == "1.0.0" + assert %DateTime{} = adaptor.checked_at + assert %DateTime{} = adaptor.updated_at + + versions = AdaptorRepo.list_versions(adaptor.name, :npm) + + assert versions |> Enum.map(& &1.version) |> Enum.sort() == [ + "1.0.0", + "1.1.0" + ] + + assert Enum.all?(versions, &(&1.adaptor_id == adaptor.id)) + end + + test "accepts a record with no versions" do + record = adaptor_record(versions: []) + + assert {:ok, %Adaptor{} = adaptor} = AdaptorRepo.upsert_adaptor(record) + assert AdaptorRepo.list_versions(adaptor.name, :npm) == [] + end + end + + describe "upsert_adaptor/1 — idempotency (§12.2)" do + test "re-upserting the same record advances :checked_at but not :updated_at" do + {:ok, first} = AdaptorRepo.upsert_adaptor(adaptor_record()) + + Process.sleep(5) + + {:ok, second} = AdaptorRepo.upsert_adaptor(adaptor_record()) + + assert second.id == first.id + assert second.updated_at == first.updated_at + assert DateTime.compare(second.checked_at, first.checked_at) == :gt + end + end + + describe "upsert_adaptor/1 — diff-aware :updated_at" do + test "changing :latest_version bumps :updated_at" do + {:ok, first} = AdaptorRepo.upsert_adaptor(adaptor_record()) + + Process.sleep(5) + + {:ok, second} = + AdaptorRepo.upsert_adaptor(adaptor_record(latest_version: "1.1.0")) + + assert second.latest_version == "1.1.0" + assert DateTime.compare(second.updated_at, first.updated_at) == :gt + end + + test "changing :description bumps :updated_at" do + {:ok, first} = AdaptorRepo.upsert_adaptor(adaptor_record()) + + Process.sleep(5) + + {:ok, second} = + AdaptorRepo.upsert_adaptor(adaptor_record(description: "new copy")) + + assert second.description == "new copy" + assert DateTime.compare(second.updated_at, first.updated_at) == :gt + end + end + + describe "upsert_adaptor/1 — version row replacement (§12.2)" do + test "replaces version rows atomically" do + {:ok, _adaptor} = + AdaptorRepo.upsert_adaptor( + adaptor_record( + versions: [version_record("1.0.0"), version_record("1.1.0")] + ) + ) + + assert AdaptorRepo.list_versions("@openfn/language-http", :npm) + |> Enum.map(& &1.version) + |> Enum.sort() == ["1.0.0", "1.1.0"] + + {:ok, _adaptor} = + AdaptorRepo.upsert_adaptor( + adaptor_record( + versions: [ + version_record("1.1.0"), + version_record("1.2.0"), + version_record("2.0.0") + ] + ) + ) + + assert AdaptorRepo.list_versions("@openfn/language-http", :npm) + |> Enum.map(& &1.version) + |> Enum.sort() == ["1.1.0", "1.2.0", "2.0.0"] + end + + test "shrinking the version set drops the missing rows" do + {:ok, _} = + AdaptorRepo.upsert_adaptor( + adaptor_record( + versions: [version_record("1.0.0"), version_record("1.1.0")] + ) + ) + + {:ok, _} = + AdaptorRepo.upsert_adaptor( + adaptor_record(versions: [version_record("1.1.0")]) + ) + + assert [%AdaptorVersion{version: "1.1.0"}] = + AdaptorRepo.list_versions("@openfn/language-http", :npm) + end + + test "persists version-row payload fields verbatim" do + payload = %{ + version: "1.0.0", + integrity: "sha512-deadbeef==", + tarball_url: "https://registry.npmjs.org/x/-/x-1.0.0.tgz", + size_bytes: 4321, + dependencies: %{"axios" => "^1.0.0"}, + peer_dependencies: %{"react" => "^18"}, + published_at: ~U[2026-05-01 10:00:00.000000Z], + deprecated: false + } + + {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(versions: [payload])) + + assert [version] = + AdaptorRepo.list_versions("@openfn/language-http", :npm) + + assert version.integrity == payload.integrity + assert version.tarball_url == payload.tarball_url + assert version.size_bytes == payload.size_bytes + assert version.dependencies == payload.dependencies + assert version.peer_dependencies == payload.peer_dependencies + assert version.published_at == payload.published_at + assert version.deprecated == payload.deprecated + end + end + + describe "upsert_adaptor/1 — source isolation" do + test "the same name can coexist across sources" do + {:ok, npm_row} = + AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) + + {:ok, local_row} = + AdaptorRepo.upsert_adaptor(adaptor_record(source: :local)) + + assert npm_row.id != local_row.id + assert npm_row.source == :npm + assert local_row.source == :local + end + end + + describe "touch_checked_at/2 (§12.2)" do + test "advances :checked_at and leaves :updated_at alone" do + {:ok, original} = AdaptorRepo.upsert_adaptor(adaptor_record()) + + Process.sleep(5) + + assert :ok = AdaptorRepo.touch_checked_at(original.name, :npm) + + reloaded = AdaptorRepo.get_adaptor(original.name, :npm) + assert DateTime.compare(reloaded.checked_at, original.checked_at) == :gt + assert reloaded.updated_at == original.updated_at + end + + test "is a no-op for an unknown (name, source) — does not require loading the row" do + assert :ok = AdaptorRepo.touch_checked_at("@openfn/never-existed", :npm) + assert AdaptorRepo.get_adaptor("@openfn/never-existed", :npm) == nil + end + + test "is source-scoped" do + {:ok, npm_row} = + AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) + + {:ok, local_row} = + AdaptorRepo.upsert_adaptor(adaptor_record(source: :local)) + + Process.sleep(5) + + :ok = AdaptorRepo.touch_checked_at(npm_row.name, :npm) + + reloaded_npm = AdaptorRepo.get_adaptor(npm_row.name, :npm) + reloaded_local = AdaptorRepo.get_adaptor(local_row.name, :local) + + assert DateTime.compare(reloaded_npm.checked_at, npm_row.checked_at) == :gt + assert reloaded_local.checked_at == local_row.checked_at + end + end + + describe "stalest/2 (§12.2)" do + test "orders by :checked_at ascending" do + base = DateTime.utc_now() + + seed_adaptor(name: "@openfn/a", checked_at: DateTime.add(base, -300)) + seed_adaptor(name: "@openfn/b", checked_at: DateTime.add(base, -100)) + seed_adaptor(name: "@openfn/c", checked_at: DateTime.add(base, -200)) + + assert AdaptorRepo.stalest(10, :npm) |> Enum.map(& &1.name) == + ["@openfn/a", "@openfn/c", "@openfn/b"] + end + + test "honours the limit" do + base = DateTime.utc_now() + seed_adaptor(name: "@openfn/a", checked_at: DateTime.add(base, -300)) + seed_adaptor(name: "@openfn/b", checked_at: DateTime.add(base, -200)) + seed_adaptor(name: "@openfn/c", checked_at: DateTime.add(base, -100)) + + assert length(AdaptorRepo.stalest(2, :npm)) == 2 + end + + test "filters by source" do + seed_adaptor(name: "@openfn/a", source: :npm) + seed_adaptor(name: "@openfn/a", source: :local) + + assert [%Adaptor{source: :npm}] = AdaptorRepo.stalest(10, :npm) + assert [%Adaptor{source: :local}] = AdaptorRepo.stalest(10, :local) + end + end + + describe "max_checked_at/1" do + test "returns the largest :checked_at for the given source" do + base = DateTime.utc_now() + newest = DateTime.add(base, -100) + seed_adaptor(name: "@openfn/a", checked_at: DateTime.add(base, -300)) + seed_adaptor(name: "@openfn/b", checked_at: newest) + + assert AdaptorRepo.max_checked_at(:npm) == newest + end + + test "returns nil when the source has no rows" do + seed_adaptor(name: "@openfn/a", source: :npm) + assert AdaptorRepo.max_checked_at(:local) == nil + end + end + + describe "get_adaptor/2" do + test "returns the matching adaptor" do + {:ok, inserted} = AdaptorRepo.upsert_adaptor(adaptor_record()) + reloaded = AdaptorRepo.get_adaptor(inserted.name, :npm) + + assert %Adaptor{} = reloaded + assert reloaded.id == inserted.id + end + + test "returns nil when not found" do + assert AdaptorRepo.get_adaptor("@openfn/never-existed", :npm) == nil + end + + test "is source-scoped" do + {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) + assert AdaptorRepo.get_adaptor("@openfn/language-http", :local) == nil + end + end + + describe "list_package_metas/1" do + test "returns the lean projection without heavy JSONB columns" do + {:ok, _} = + AdaptorRepo.upsert_adaptor( + adaptor_record( + description: "yep", + schema_data: %{"big" => "json", "nested" => %{"more" => "stuff"}} + ) + ) + + assert [meta] = AdaptorRepo.list_package_metas(:npm) + + assert meta.name == "@openfn/language-http" + assert meta.latest_version == "1.0.0" + assert meta.description == "yep" + assert meta.deprecated == false + assert %DateTime{} = meta.updated_at + + refute Map.has_key?(meta, :schema_data) + refute Map.has_key?(meta, :homepage) + end + + test "filters by source" do + {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) + {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :local)) + + assert [%{name: "@openfn/language-http"}] = + AdaptorRepo.list_package_metas(:npm) + + assert [%{name: "@openfn/language-http"}] = + AdaptorRepo.list_package_metas(:local) + end + end + + describe "list_adaptors/1" do + test "returns full structs filtered by source" do + {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) + {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :local)) + + assert [%Adaptor{source: :npm}] = AdaptorRepo.list_adaptors(:npm) + assert [%Adaptor{source: :local}] = AdaptorRepo.list_adaptors(:local) + end + end + + describe "list_missing_icons/1" do + test "returns rows where either icon shape sha256 is nil" do + {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(name: "@openfn/a")) + + {:ok, _} = + AdaptorRepo.upsert_adaptor( + adaptor_record( + name: "@openfn/b", + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "x") + ) + ) + + {:ok, _} = + AdaptorRepo.upsert_adaptor( + adaptor_record( + name: "@openfn/c", + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "y"), + icon_rectangle_ext: "png", + icon_rectangle_sha256: :crypto.hash(:sha256, "z") + ) + ) + + names = + AdaptorRepo.list_missing_icons(:npm) + |> Enum.map(& &1.name) + |> Enum.sort() + + assert names == ["@openfn/a", "@openfn/b"] + end + + test "is source-scoped" do + {:ok, _} = + AdaptorRepo.upsert_adaptor( + adaptor_record(name: "@openfn/x", source: :local) + ) + + assert AdaptorRepo.list_missing_icons(:npm) == [] + assert [%{name: "@openfn/x"}] = AdaptorRepo.list_missing_icons(:local) + end + end + + describe "update_icons/3" do + test "writes only icon columns and bumps :updated_at" do + {:ok, before} = AdaptorRepo.upsert_adaptor(adaptor_record()) + Process.sleep(5) + sha = :crypto.hash(:sha256, "PNG") + + assert {1, nil} = + AdaptorRepo.update_icons(before.name, :npm, %{ + icon_square_ext: "png", + icon_square_sha256: sha + }) + + after_row = AdaptorRepo.get_adaptor(before.name, :npm) + + assert after_row.icon_square_ext == "png" + assert after_row.icon_square_sha256 == sha + assert after_row.latest_version == before.latest_version + assert DateTime.compare(after_row.updated_at, before.updated_at) == :gt + end + + test "ignores keys outside the icon set" do + {:ok, before} = AdaptorRepo.upsert_adaptor(adaptor_record()) + + AdaptorRepo.update_icons(before.name, :npm, %{ + latest_version: "9.9.9", + icon_square_ext: "svg", + icon_square_sha256: :crypto.hash(:sha256, "S") + }) + + after_row = AdaptorRepo.get_adaptor(before.name, :npm) + assert after_row.latest_version == before.latest_version + assert after_row.icon_square_ext == "svg" + end + + test "writes icon etag columns alongside ext/sha256" do + {:ok, before} = AdaptorRepo.upsert_adaptor(adaptor_record()) + sha = :crypto.hash(:sha256, "PNG") + + assert {1, nil} = + AdaptorRepo.update_icons(before.name, :npm, %{ + icon_square_ext: "png", + icon_square_sha256: sha, + icon_square_etag: ~s("abc123") + }) + + after_row = AdaptorRepo.get_adaptor(before.name, :npm) + + assert %{ + icon_square_ext: "png", + icon_square_sha256: ^sha, + icon_square_etag: ~s("abc123"), + icon_rectangle_etag: nil + } = after_row + end + + test "leaves version rows untouched" do + {:ok, before} = + AdaptorRepo.upsert_adaptor( + adaptor_record( + versions: [version_record("1.0.0"), version_record("2.0.0")] + ) + ) + + AdaptorRepo.update_icons(before.name, :npm, %{ + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "P") + }) + + versions = AdaptorRepo.list_versions(before.name, :npm) + assert length(versions) == 2 + end + end + + defp adaptor_record(overrides \\ []) do + overrides = Map.new(overrides) + + %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + description: "HTTP adaptor", + homepage: nil, + repository: nil, + license: "LGPL-3.0", + deprecated: false, + schema_data: nil, + schema_sha256: nil, + icon_square_ext: nil, + icon_rectangle_ext: nil, + icon_square_sha256: nil, + icon_rectangle_sha256: nil, + icon_square_etag: nil, + icon_rectangle_etag: nil, + versions: [version_record("1.0.0")] + } + |> Map.merge(overrides) + end + + defp version_record(version) do + %{ + version: version, + integrity: "sha512-#{version}", + tarball_url: "https://example.com/x/-/x-#{version}.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + end + + defp seed_adaptor(opts) do + attrs = + %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + checked_at: DateTime.utc_now() + } + |> Map.merge(Map.new(opts)) + + {:ok, adaptor} = + %Adaptor{} + |> Adaptor.changeset(attrs) + |> Lightning.Repo.insert() + + adaptor + end +end diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs new file mode 100644 index 00000000000..bbf148dbffb --- /dev/null +++ b/test/lightning/adaptors/scheduler_test.exs @@ -0,0 +1,959 @@ +defmodule Lightning.Adaptors.SchedulerTest do + # async: false because: + # 1. DataCase uses shared sandbox mode (all processes access DB without allow/3) + # 2. set_mox_global is safe only when tests run serially + use Lightning.DataCase, async: false + + import Mox + + alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Scheduler + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + # set_mox_global makes expectations visible to tasks spawned by the Scheduler, + # whose $callers chain does not include the test process (only the GenServer). + setup :set_mox_global + setup :verify_on_exit! + + # Each test owns an isolated supervisor. The supervisor starts its own + # Scheduler as part of the :rest_for_one child list, but with the + # test-env `refresh_interval: 0` it's an inert no-op. Individual tests + # call `start_scheduler/2` to replace it with a controlled-interval + # Scheduler under `start_supervised!/1` (so Mox expectations can be + # registered before init fires). + setup do + sup = :"sched_test_#{System.unique_integer([:positive])}" + + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + ) + + # Default no-op icons stub for tests that don't care about the icons + # pipeline. Individual tests override via `expect` when they need to + # assert on it. + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + {:ok, sup: sup} + end + + # Replace the supervisor's inert auto-started (HighlanderPG-wrapped) + # Scheduler with a controlled one under test ownership. Application + # env is restored immediately after start_supervised!/1 returns + # because the Scheduler captures interval_ms in init/1. + # + # The test-owned Scheduler bypasses HighlanderPG entirely: we + # register the GenServer directly under the same `{:global, …}` name + # the production wrapper would, so test code can call it via + # `AdaptorsSupervisor.global_scheduler_name/1` exactly as production + # callers do. + defp start_scheduler(sup, opts \\ []) do + interval = Keyword.get(opts, :interval, 99_999_999) + original_env = Application.get_env(:lightning, Lightning.Adaptors, []) + + Application.put_env( + :lightning, + Lightning.Adaptors, + Keyword.put(original_env, :refresh_interval, interval) + ) + + global_name = AdaptorsSupervisor.global_scheduler_name(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + + # Stop the supervisor's auto-started HighlanderPG (and its wrapped + # Scheduler) so we can start a replacement under the controlled + # interval without name collision. + :ok = + Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) + + pid = + start_supervised!({ + Scheduler, + name: global_name, + sup: sup, + lock_key: AdaptorsSupervisor.lock_key(sup), + cache: AdaptorsSupervisor.cache_name(sup), + tasks: AdaptorsSupervisor.tasks_name(sup), + source_topic: source_topic + }) + + Application.put_env(:lightning, Lightning.Adaptors, original_env) + + pid + end + + defp adaptor_record(overrides \\ []) do + overrides = Map.new(overrides) + + %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + description: "HTTP adaptor", + homepage: nil, + repository: nil, + license: "LGPL-3.0", + deprecated: false, + schema_data: nil, + schema_sha256: nil, + versions: [ + %{ + version: "1.0.0", + integrity: "sha512-abc", + tarball_url: "https://example.com/x-1.0.0.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + ] + } + |> Map.merge(overrides) + end + + describe "start_link/1" do + test "raises when :name is missing", %{sup: sup} do + assert_raise KeyError, ~r/key :name not found/, fn -> + Scheduler.start_link( + sup: sup, + lock_key: 1, + cache: :cache, + tasks: :tasks, + source_topic: "t" + ) + end + end + + test "raises when :sup is missing", %{sup: sup} do + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert_raise KeyError, ~r/key :sup not found/, fn -> + Scheduler.start_link( + name: sched_name, + lock_key: 1, + cache: :cache, + tasks: :tasks, + source_topic: "t" + ) + end + end + + test "raises when :lock_key is missing", %{sup: sup} do + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert_raise KeyError, ~r/key :lock_key not found/, fn -> + Scheduler.start_link( + name: sched_name, + sup: sup, + cache: :cache, + tasks: :tasks, + source_topic: "t" + ) + end + end + + test "registers under :global with global_scheduler_name/1", %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> {:ok, []} end) + start_scheduler(sup) + {:global, global_name} = AdaptorsSupervisor.global_scheduler_name(sup) + assert is_pid(:global.whereis_name(global_name)) + end + end + + describe "tick timing" do + test "tick fires on init when table is empty", %{sup: sup} do + test_pid = self() + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :list_adaptors_called) + {:ok, []} + end) + + # Empty table → max_checked_at returns nil → delay 0 → tick fires on init. + start_scheduler(sup) + + assert_receive :list_adaptors_called, 2000 + end + + test "tick re-arms itself", %{sup: sup} do + test_pid = self() + + # Stub allows repeated calls; each fires a message so we can count them. + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :tick_ran) + {:ok, []} + end) + + # 30ms interval → two ticks fire well within 2s. + start_scheduler(sup, interval: 30) + + assert_receive :tick_ran, 2000 + assert_receive :tick_ran, 2000 + end + end + + describe "do_refresh/1 diff logic" do + test "unchanged adaptor: touch_checked_at only, no upsert, no broadcast", %{ + sup: sup + } do + test_pid = self() + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + + {:ok, existing} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + checked_at_before = existing.checked_at + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :list_adaptors_called) + {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + # With a recently-inserted adaptor, max_checked_at is "now", so the smart- + # init delay is ~99,999 seconds. Trigger an explicit tick via refresh_now. + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + Scheduler.refresh_now(sched_name) + + assert_receive :list_adaptors_called, 2000 + + # Allow the spawned task to complete before asserting no broadcast. + refute_receive {:changed, _, _}, 200 + + row = AdaptorsRepo.get_adaptor("@openfn/language-http", source) + assert DateTime.compare(row.checked_at, checked_at_before) == :gt + assert row.latest_version == "1.0.0" + end + + test "changed adaptor: upsert and broadcast per changed name", %{sup: sup} do + test_pid = self() + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :list_adaptors_called) + {:ok, [%{name: "@openfn/language-http", latest_version: "2.0.0"}]} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> + {:ok, adaptor_record(latest_version: "2.0.0")} + end + ) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + # Trigger an explicit tick since smart-init delay is large (recent checked_at). + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + Scheduler.refresh_now(sched_name) + + assert_receive :list_adaptors_called, 2000 + assert_receive {:changed, "@openfn/language-http", ^source}, 2000 + + row = AdaptorsRepo.get_adaptor("@openfn/language-http", source) + assert row.latest_version == "2.0.0" + end + + test "new adaptor (not in DB): upsert and broadcast", %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-new", latest_version: "1.0.0"}]} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-new" -> + {:ok, adaptor_record(name: "@openfn/language-new")} + end + ) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + assert_receive {:changed, "@openfn/language-new", ^source}, 2000 + assert AdaptorsRepo.get_adaptor("@openfn/language-new", source) != nil + end + + test "list_adaptors error: no DB writes, no broadcasts", %{sup: sup} do + test_pid = self() + source_topic = AdaptorsSupervisor.source_topic(sup) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :list_adaptors_called) + {:error, :timeout} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + assert_receive :list_adaptors_called, 2000 + refute_receive {:changed, _, _}, 200 + end + + test "fetch_adaptor error: logs warning, continues to next adaptor", %{ + sup: sup + } do + source_topic = AdaptorsSupervisor.source_topic(sup) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, + [ + %{name: "@openfn/bad-adaptor", latest_version: "1.0.0"}, + %{name: "@openfn/good-adaptor", latest_version: "1.0.0"} + ]} + end) + + # Single multi-clause expectation — Mox routes by pattern within + # one slot, so Scheduler's async_stream_nolink can fan out to the + # two adaptors in either order. Two separate `expect/4` calls + # would dispatch FIFO and crash with FunctionClauseError when the + # task arrival order doesn't match the expectation insertion order. + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 2, fn + "@openfn/bad-adaptor" -> + {:error, :not_found} + + "@openfn/good-adaptor" -> + {:ok, adaptor_record(name: "@openfn/good-adaptor")} + end) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + assert_receive {:changed, "@openfn/good-adaptor", _}, 2000 + refute_receive {:changed, "@openfn/bad-adaptor", _}, 200 + end + end + + describe "refresh_now/1" do + test "triggers an immediate tick on the leader", %{sup: sup} do + test_pid = self() + + # First call: from init tick. Second call: from refresh_now. + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 2, fn -> + send(test_pid, :tick_ran) + {:ok, []} + end) + + start_scheduler(sup) + + # Wait for init tick. + assert_receive :tick_ran, 2000 + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + assert :ok = Scheduler.refresh_now(sched_name) + + assert_receive :tick_ran, 2000 + end + end + + describe "icons pipeline" do + test "writes icon bytes to disk and stamps ext+sha256 on the row", %{ + sup: sup + } do + source = AdaptorsSupervisor.source(sup) + + bytes = "ICON_BYTES" + sha = :crypto.hash(:sha256, bytes) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + {:ok, adaptor_record()} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, + %{ + "@openfn/language-http" => %{ + square: %{data: bytes, ext: "png", sha256: sha} + } + }} + end) + + source_topic = AdaptorsSupervisor.source_topic(sup) + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + assert_receive {:changed, "@openfn/language-http", ^source}, 2000 + + row = AdaptorsRepo.get_adaptor("@openfn/language-http", source) + assert row.icon_square_ext == "png" + assert row.icon_square_sha256 == sha + assert row.icon_rectangle_ext == nil + assert row.icon_rectangle_sha256 == nil + + icon_path = + Lightning.Adaptors.IconCache.path( + source, + "@openfn/language-http", + :square, + "png" + ) + + assert File.exists?(icon_path) + assert File.read!(icon_path) == bytes + File.rm!(icon_path) + end + + test "fetch_icons error: records still persist without icons", %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + {:ok, adaptor_record()} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:error, :timeout} + end) + + source_topic = AdaptorsSupervisor.source_topic(sup) + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + assert_receive {:changed, "@openfn/language-http", ^source}, 2000 + + row = AdaptorsRepo.get_adaptor("@openfn/language-http", source) + assert row != nil + assert row.icon_square_ext == nil + assert row.icon_square_sha256 == nil + end + + test "self-heals iconless rows on the periodic tick", %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + + # Pre-seed a row that already matches the listed latest_version + # (so the diff path will :touch instead of :fetch). Without + # self-heal this row would stay iconless forever. + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record(name: "@openfn/language-stale") + ) + + bytes = "STALE_ICON" + sha = :crypto.hash(:sha256, bytes) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-stale", latest_version: "1.0.0"}]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn opts -> + # Row has no etags pre-seeded, so it is omitted from the + # prior-etags map entirely (no empty inner map). + assert Keyword.get(opts, :prior_etags) == %{} + + {:ok, + %{ + "@openfn/language-stale" => %{ + square: %{data: bytes, ext: "png", sha256: sha} + } + }} + end) + + source_topic = AdaptorsSupervisor.source_topic(sup) + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + # The pre-seeded row pushes max_checked_at to "now", so init + # delay = full interval — drive the tick explicitly. + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + :ok = Scheduler.refresh_now(sched_name) + + assert_receive {:changed, "@openfn/language-stale", ^source}, 2000 + + row = AdaptorsRepo.get_adaptor("@openfn/language-stale", source) + assert row.icon_square_ext == "png" + assert row.icon_square_sha256 == sha + + icon_path = + Lightning.Adaptors.IconCache.path( + source, + "@openfn/language-stale", + :square, + "png" + ) + + File.rm(icon_path) + end + + test "fetches per-adaptor in parallel (multiple concurrent fetch_adaptor calls)", + %{sup: sup} do + test_pid = self() + barrier = :ets.new(:scheduler_test_barrier, [:public, :set]) + :ets.insert(barrier, {:in_flight, 0}) + :ets.insert(barrier, {:max_in_flight, 0}) + + names = + for i <- 1..6, + do: %{name: "@openfn/language-pkg#{i}", latest_version: "1.0.0"} + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, names} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn name -> + in_flight = :ets.update_counter(barrier, :in_flight, 1) + + :ets.update_element( + barrier, + :max_in_flight, + {2, max_seen(barrier, in_flight)} + ) + + # Hold long enough that the fan-out has time to overlap. + Process.sleep(80) + :ets.update_counter(barrier, :in_flight, -1) + send(test_pid, {:fetched, name}) + {:ok, adaptor_record(name: name)} + end) + + start_scheduler(sup) + + for _ <- 1..6 do + assert_receive {:fetched, _name}, 5_000 + end + + [{:max_in_flight, max_in_flight}] = :ets.lookup(barrier, :max_in_flight) + :ets.delete(barrier) + + assert max_in_flight > 1, + "expected concurrent fetch_adaptor calls, saw at most 1 in-flight" + end + end + + defp max_seen(barrier, current) do + [{:max_in_flight, prev}] = :ets.lookup(barrier, :max_in_flight) + max(prev, current) + end + + describe "refresh_package/2" do + test "fetches and upserts a single adaptor, bypassing diff", %{sup: sup} do + test_pid = self() + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :init_tick_done) + {:ok, []} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> + {:ok, adaptor_record()} + end + ) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + # Drain the init tick (table is empty → delay 0 → fires immediately). + assert_receive :init_tick_done, 2000 + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert :ok = Scheduler.refresh_package(sched_name, "@openfn/language-http") + assert_receive {:changed, "@openfn/language-http", ^source}, 2000 + + assert AdaptorsRepo.get_adaptor("@openfn/language-http", source) != nil + end + + test "returns error tuple when fetch_adaptor fails", %{sup: sup} do + test_pid = self() + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :init_tick_done) + {:ok, []} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn _ -> + {:error, :not_found} + end) + + start_scheduler(sup) + + # Drain init tick before calling refresh_package. + assert_receive :init_tick_done, 2000 + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:error, :not_found} = + Scheduler.refresh_package(sched_name, "@openfn/language-http") + end + + test "does not call fetch_icons (icons only refresh on the periodic tick)", + %{sup: sup} do + test_pid = self() + source_topic = AdaptorsSupervisor.source_topic(sup) + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :init_tick_done) + {:ok, []} + end) + + # Exactly one fetch_icons call — the init tick. If refresh_package + # also fetched icons the count would be 2 and Mox would fail. + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, 1, fn _opts -> + send(test_pid, :icons_called) + {:ok, %{}} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> {:ok, adaptor_record()} end + ) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + assert_receive :init_tick_done, 2000 + assert_receive :icons_called, 2000 + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + assert :ok = Scheduler.refresh_package(sched_name, "@openfn/language-http") + assert_receive {:changed, "@openfn/language-http", _}, 2000 + + # Give any (mistaken) extra fetch_icons call time to happen. + Process.sleep(100) + end + end + + describe "refresh_icons/1" do + test "updates rows whose shape sha256 differs from the fetched icon", %{ + sup: sup + } do + source = AdaptorsSupervisor.source(sup) + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record(name: "@openfn/language-empty") + ) + + old_sha = :crypto.hash(:sha256, "OLD") + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: "@openfn/language-current", + icon_square_ext: "png", + icon_square_sha256: old_sha + ) + ) + + new_bytes = "NEW_BYTES" + new_sha = :crypto.hash(:sha256, new_bytes) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, + %{ + "@openfn/language-empty" => %{ + square: %{data: new_bytes, ext: "png", sha256: new_sha} + }, + "@openfn/language-current" => %{ + square: %{data: new_bytes, ext: "png", sha256: new_sha} + } + }} + end) + + # interval: 0 disables the init tick so refresh_icons is the only + # path that calls fetch_icons. + start_scheduler(sup, interval: 0) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:ok, %{updated: 2, unchanged: 0}} = + Scheduler.refresh_icons(sched_name) + + empty = AdaptorsRepo.get_adaptor("@openfn/language-empty", source) + assert empty.icon_square_ext == "png" + assert empty.icon_square_sha256 == new_sha + + current = AdaptorsRepo.get_adaptor("@openfn/language-current", source) + assert current.icon_square_sha256 == new_sha + + for name <- ["@openfn/language-empty", "@openfn/language-current"] do + Lightning.Adaptors.IconCache.path(source, name, :square, "png") + |> File.rm() + end + end + + test "leaves rows whose shape sha256 already matches unchanged, passing prior etag", + %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + sha = :crypto.hash(:sha256, "SAME") + etag = ~s("prior-etag-1") + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: "@openfn/language-same", + icon_square_ext: "png", + icon_square_sha256: sha, + icon_square_etag: etag + ) + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn opts -> + # Strategy receives the prior etag for this row's shape. + assert Keyword.get(opts, :prior_etags) == %{ + "@openfn/language-same" => %{square: etag} + } + + {:ok, %{"@openfn/language-same" => %{square: :not_modified}}} + end) + + start_scheduler(sup, interval: 0) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:ok, %{updated: 0, unchanged: 1}} = + Scheduler.refresh_icons(sched_name) + + row = AdaptorsRepo.get_adaptor("@openfn/language-same", source) + assert row.icon_square_sha256 == sha + assert row.icon_square_etag == etag + end + + test "applies new etag when shape sha256 changes", %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + old_sha = :crypto.hash(:sha256, "OLD") + new_bytes = "NEW" + new_sha = :crypto.hash(:sha256, new_bytes) + old_etag = ~s("etag-A") + new_etag = ~s("etag-B") + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: "@openfn/language-rotated", + icon_square_ext: "png", + icon_square_sha256: old_sha, + icon_square_etag: old_etag + ) + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, + %{ + "@openfn/language-rotated" => %{ + square: %{ + data: new_bytes, + ext: "png", + sha256: new_sha, + etag: new_etag + } + } + }} + end) + + start_scheduler(sup, interval: 0) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:ok, %{updated: 1, unchanged: 0}} = + Scheduler.refresh_icons(sched_name) + + row = AdaptorsRepo.get_adaptor("@openfn/language-rotated", source) + assert row.icon_square_sha256 == new_sha + assert row.icon_square_etag == new_etag + + Lightning.Adaptors.IconCache.path( + source, + "@openfn/language-rotated", + :square, + "png" + ) + |> File.rm() + end + + test "preserves existing etag when fetched entry's etag is nil or missing", + %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + old_sha = :crypto.hash(:sha256, "OLD") + new_bytes_a = "NEW_A" + new_sha_a = :crypto.hash(:sha256, new_bytes_a) + new_bytes_b = "NEW_B" + new_sha_b = :crypto.hash(:sha256, new_bytes_b) + prior_etag = ~s("etag-A") + + # Two rows: one returns 200 with etag: nil (NPM-style), the other + # returns 200 with the :etag key entirely absent (Local-style). + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: "@openfn/language-nil-etag", + icon_square_ext: "png", + icon_square_sha256: old_sha, + icon_square_etag: prior_etag + ) + ) + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: "@openfn/language-no-etag-key", + icon_square_ext: "png", + icon_square_sha256: old_sha, + icon_square_etag: prior_etag + ) + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, + %{ + "@openfn/language-nil-etag" => %{ + square: %{ + data: new_bytes_a, + ext: "png", + sha256: new_sha_a, + etag: nil + } + }, + "@openfn/language-no-etag-key" => %{ + square: %{data: new_bytes_b, ext: "png", sha256: new_sha_b} + } + }} + end) + + start_scheduler(sup, interval: 0) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:ok, %{updated: 2, unchanged: 0}} = + Scheduler.refresh_icons(sched_name) + + row_a = AdaptorsRepo.get_adaptor("@openfn/language-nil-etag", source) + assert row_a.icon_square_sha256 == new_sha_a + assert row_a.icon_square_etag == prior_etag + + row_b = AdaptorsRepo.get_adaptor("@openfn/language-no-etag-key", source) + assert row_b.icon_square_sha256 == new_sha_b + assert row_b.icon_square_etag == prior_etag + + for name <- ["@openfn/language-nil-etag", "@openfn/language-no-etag-key"] do + Lightning.Adaptors.IconCache.path(source, name, :square, "png") + |> File.rm() + end + end + + test "mixed 304 and 200: unchanged row preserves its etag verbatim", + %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + stale_old_sha = :crypto.hash(:sha256, "STALE_OLD") + stale_new_bytes = "STALE_NEW" + stale_new_sha = :crypto.hash(:sha256, stale_new_bytes) + stale_old_etag = ~s("etag-stale-old") + stale_new_etag = ~s("etag-stale-new") + + current_sha = :crypto.hash(:sha256, "CURRENT_BYTES") + current_etag = ~s("etag-current") + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: "@openfn/language-stale-etag", + icon_square_ext: "png", + icon_square_sha256: stale_old_sha, + icon_square_etag: stale_old_etag + ) + ) + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: "@openfn/language-current-etag", + icon_square_ext: "png", + icon_square_sha256: current_sha, + icon_square_etag: current_etag + ) + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn opts -> + # Both rows contribute prior etags. + assert Keyword.get(opts, :prior_etags) == %{ + "@openfn/language-stale-etag" => %{square: stale_old_etag}, + "@openfn/language-current-etag" => %{square: current_etag} + } + + {:ok, + %{ + "@openfn/language-stale-etag" => %{ + square: %{ + data: stale_new_bytes, + ext: "png", + sha256: stale_new_sha, + etag: stale_new_etag + } + }, + "@openfn/language-current-etag" => %{square: :not_modified} + }} + end) + + start_scheduler(sup, interval: 0) + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:ok, %{updated: 1, unchanged: 1}} = + Scheduler.refresh_icons(sched_name) + + stale_row = AdaptorsRepo.get_adaptor("@openfn/language-stale-etag", source) + assert stale_row.icon_square_sha256 == stale_new_sha + assert stale_row.icon_square_etag == stale_new_etag + + current_row = + AdaptorsRepo.get_adaptor("@openfn/language-current-etag", source) + + assert current_row.icon_square_sha256 == current_sha + assert current_row.icon_square_etag == current_etag + + Lightning.Adaptors.IconCache.path( + source, + "@openfn/language-stale-etag", + :square, + "png" + ) + |> File.rm() + end + + test "surfaces a strategy fetch error as {:error, reason}", %{sup: sup} do + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:error, :upstream_down} + end) + + start_scheduler(sup, interval: 0) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:error, :upstream_down} = Scheduler.refresh_icons(sched_name) + end + end +end diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs new file mode 100644 index 00000000000..02f565aa867 --- /dev/null +++ b/test/lightning/adaptors/store_test.exs @@ -0,0 +1,552 @@ +defmodule Lightning.Adaptors.StoreTest do + use Lightning.DataCase, async: true + + import Mox + + alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Store + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + setup :verify_on_exit! + + setup do + # Each test owns an isolated `Lightning.Adaptors.Supervisor` instance, + # parameterised on a unique `name:` so cache table / persistent_term + # entries don't collide across the async suite. The `:strategy` opt + # is threaded explicitly — no `Application.put_env` mutation. + sup = :"store_test_#{System.unique_integer([:positive])}" + + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + ) + + cache = AdaptorsSupervisor.cache_name(sup) + + {:ok, sup: sup, cache: cache} + end + + describe "schema/2" do + test "cache hit returns cached value without touching Strategy or DB", %{ + sup: sup, + cache: cache + } do + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + source = AdaptorsSupervisor.source(sup) + + Cachex.put!( + cache, + {:schema, "@openfn/language-http", source}, + {:ok, ~s({"type":"object"})} + ) + + assert {:ok, ~s({"type":"object"})} = + Store.schema(sup, "@openfn/language-http") + + assert AdaptorsRepo.get_adaptor("@openfn/language-http", source) == nil + end + + test "cache miss + DB hit returns DB value without calling Strategy", %{ + sup: sup + } do + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record(schema_data: ~s({"type":"object"})) + ) + + assert {:ok, ~s({"type":"object"})} = + Store.schema(sup, "@openfn/language-http") + end + + test "cache miss + DB miss calls Strategy once, upserts to DB, caches result", + %{ + sup: sup, + cache: cache + } do + source = AdaptorsSupervisor.source(sup) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> + {:ok, adaptor_record(schema_data: ~s({"type":"object"}))} + end + ) + + assert {:ok, ~s({"type":"object"})} = + Store.schema(sup, "@openfn/language-http") + + assert %{schema_data: ~s({"type":"object"})} = + AdaptorsRepo.get_adaptor("@openfn/language-http", source) + + assert {:ok, {:ok, ~s({"type":"object"})}} = + Cachex.get(cache, {:schema, "@openfn/language-http", source}) + end + + test "three concurrent calls coalesce to one Strategy call", %{sup: sup} do + name = "@openfn/language-http" + test_pid = self() + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> + # Brief sleep so the other two tasks queue up in Cachex's courier. + Process.sleep(30) + {:ok, adaptor_record(schema_data: ~s({"type":"object"}))} + end) + + tasks = + Enum.map(1..3, fn _ -> + Task.async(fn -> + receive do + :go -> Store.schema(sup, name) + end + end) + end) + + # Allow all tasks to use the test process's Mox expectations before releasing them. + Enum.each( + tasks, + &Mox.allow(Lightning.Adaptors.StrategyMock, test_pid, &1.pid) + ) + + Enum.each(tasks, &send(&1.pid, :go)) + + results = Task.await_many(tasks, 5_000) + assert Enum.all?(results, &match?({:ok, ~s({"type":"object"})}, &1)) + end + + test "Strategy error returns {:error, _} and is not cached — next call retries", + %{ + sup: sup, + cache: cache + } do + source = AdaptorsSupervisor.source(sup) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn _ -> + {:error, :upstream_error} + end) + + assert {:error, :upstream_error} = + Store.schema(sup, "@openfn/language-http") + + assert {:ok, nil} = + Cachex.get(cache, {:schema, "@openfn/language-http", source}) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> + {:ok, adaptor_record(schema_data: ~s({"type":"object"}))} + end + ) + + assert {:ok, ~s({"type":"object"})} = + Store.schema(sup, "@openfn/language-http") + end + + test "preserves JSON property order through the persistence round-trip", + %{sup: sup} do + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + ordered_body = ~s({"a":1,"z":2,"m":3}) + + {:ok, _} = + AdaptorsRepo.upsert_adaptor(adaptor_record(schema_data: ordered_body)) + + assert {:ok, ^ordered_body} = Store.schema(sup, "@openfn/language-http") + end + end + + describe "versions/2" do + test "cache miss + DB hit returns projected versions without calling Strategy", + %{sup: sup} do + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + versions: [version_record("1.0.0"), version_record("1.1.0")] + ) + ) + + assert {:ok, versions} = Store.versions(sup, "@openfn/language-http") + assert length(versions) == 2 + assert Enum.all?(versions, &Map.has_key?(&1, :version)) + assert Enum.all?(versions, &Map.has_key?(&1, :deprecated)) + end + + test "cache miss + DB miss calls Strategy and caches projected versions", %{ + sup: sup, + cache: cache + } do + source = AdaptorsSupervisor.source(sup) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> + {:ok, + adaptor_record( + versions: [version_record("1.0.0"), version_record("2.0.0")] + )} + end + ) + + assert {:ok, versions} = Store.versions(sup, "@openfn/language-http") + assert length(versions) == 2 + + assert {:ok, {:ok, cached_versions}} = + Cachex.get(cache, {:versions, "@openfn/language-http", source}) + + assert length(cached_versions) == 2 + end + end + + describe "packages/1" do + test "empty DB returns {:ok, []} but does NOT cache the empty result", %{ + sup: sup, + cache: cache + } do + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + assert {:ok, []} = Store.packages(sup) + + source = AdaptorsSupervisor.source(sup) + assert {:ok, nil} = Cachex.get(cache, {:packages, source}) + end + + test "DB with rows returns and caches package metas", %{ + sup: sup, + cache: cache + } do + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + + assert {:ok, [pkg]} = Store.packages(sup) + assert pkg.name == "@openfn/language-http" + + source = AdaptorsSupervisor.source(sup) + assert {:ok, {:ok, [_]}} = Cachex.get(cache, {:packages, source}) + end + end + + describe "icon/3" do + # Each test uses a unique adaptor name so the on-disk cache (shared + # default {:tmp, "lightning/adaptor_icons"} path) does not collide + # across this `async: true` suite. Directories created here are not + # cleaned up — they live under System.tmp_dir! and are namespaced + # per-name so they cannot collide. + defp unique_name(prefix) do + "@openfn/language-#{prefix}-#{System.unique_integer([:positive])}" + end + + test "disk hit returns path without calling Strategy", %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + name = unique_name("disk-hit") + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: name, + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "PRE_WARMED") + ) + ) + + {:ok, _} = + Lightning.Adaptors.IconCache.write!( + source, + name, + :square, + "png", + "PRE_WARMED" + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 0, fn _, _ -> + :unreachable + end) + + assert {:ok, path} = Store.icon(sup, name, :square) + assert File.read!(path) == "PRE_WARMED" + end + + test "disk miss + Strategy success writes to disk and returns path", %{ + sup: sup, + cache: cache + } do + source = AdaptorsSupervisor.source(sup) + name = unique_name("disk-miss") + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: name, + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "LAZY_BYTES") + ) + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 1, fn ^name, + :square -> + {:ok, %{data: "LAZY_BYTES", ext: "png"}} + end) + + assert {:ok, path} = Store.icon(sup, name, :square) + assert File.read!(path) == "LAZY_BYTES" + + # Courier returned {:ignore, _} → no committed entry on the bytes key. + assert {:ok, nil} = + Cachex.get(cache, {:icon_bytes, source, name, :square}) + end + + test "Strategy error returns {:error, _} and does not commit", %{ + sup: sup, + cache: cache + } do + source = AdaptorsSupervisor.source(sup) + name = unique_name("err") + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: name, + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "UNUSED") + ) + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 1, fn _, _ -> + {:error, :upstream_5xx} + end) + + assert {:error, :upstream_5xx} = Store.icon(sup, name, :square) + + assert {:ok, nil} = + Cachex.get(cache, {:icon_bytes, source, name, :square}) + end + + test "concurrent first-callers coalesce onto one Strategy fetch", %{ + sup: sup + } do + test_pid = self() + name = unique_name("coalesce") + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: name, + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "COALESCED") + ) + ) + + # Single Mox expectation → if both callers reach the strategy + # the second hits "no expectation" and Mox raises. + expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 1, fn ^name, + :square -> + send(test_pid, :fetch_started) + # Block long enough for the second caller to also reach the + # Cachex courier and coalesce onto this call. + Process.sleep(150) + {:ok, %{data: "COALESCED", ext: "png"}} + end) + + t1 = Task.async(fn -> Store.icon(sup, name, :square) end) + assert_receive :fetch_started, 1000 + t2 = Task.async(fn -> Store.icon(sup, name, :square) end) + + assert {:ok, p1} = Task.await(t1, 5000) + assert {:ok, p2} = Task.await(t2, 5000) + assert p1 == p2 + assert File.read!(p1) == "COALESCED" + end + + test "different (name, shape) misses fetch in parallel without false coalescing", + %{sup: sup} do + test_pid = self() + name_a = unique_name("parA") + name_b = unique_name("parB") + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: name_a, + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "A_BYTES") + ) + ) + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + name: name_b, + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "B_BYTES") + ) + ) + + # Single multi-clause expectation with count: 2 — Mox routes by + # pattern within one slot, so the two parallel courier calls can + # arrive in either order. Two separate `expect/3` calls would + # queue FIFO and crash with FunctionClauseError when the task + # arrival order doesn't match the expectation insertion order. + expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 2, fn + ^name_a, :square -> {:ok, %{data: "A_BYTES", ext: "png"}} + ^name_b, :square -> {:ok, %{data: "B_BYTES", ext: "png"}} + end) + + t_a = + Task.async(fn -> + receive do + :go -> Store.icon(sup, name_a, :square) + end + end) + + t_b = + Task.async(fn -> + receive do + :go -> Store.icon(sup, name_b, :square) + end + end) + + Mox.allow(Lightning.Adaptors.StrategyMock, test_pid, t_a.pid) + Mox.allow(Lightning.Adaptors.StrategyMock, test_pid, t_b.pid) + + send(t_a.pid, :go) + send(t_b.pid, :go) + + assert {:ok, p1} = Task.await(t_a, 5000) + assert {:ok, p2} = Task.await(t_b, 5000) + + assert File.read!(p1) == "A_BYTES" + assert File.read!(p2) == "B_BYTES" + end + end + + describe "icon_meta/2" do + test "unknown adaptor returns {:error, :not_found} and is not cached", %{ + sup: sup, + cache: cache + } do + assert {:error, :not_found} = Store.icon_meta(sup, "@openfn/never-existed") + + source = AdaptorsSupervisor.source(sup) + + assert {:ok, nil} = + Cachex.get(cache, {:icon_meta, "@openfn/never-existed", source}) + end + + test "known adaptor returns icon metadata and caches it", %{ + sup: sup, + cache: cache + } do + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + icon_square_ext: "svg", + icon_square_sha256: :crypto.hash(:sha256, "fake-svg-bytes") + ) + ) + + assert {:ok, meta} = Store.icon_meta(sup, "@openfn/language-http") + assert meta.icon_square_ext == "svg" + + source = AdaptorsSupervisor.source(sup) + + assert {:ok, {:ok, cached}} = + Cachex.get(cache, {:icon_meta, "@openfn/language-http", source}) + + assert cached.icon_square_ext == "svg" + end + end + + describe "warm_from_repo/1" do + test "populates {:packages, source} and {:icon_meta, name, source} keys", %{ + sup: sup, + cache: cache + } do + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + + assert :ok = Store.warm_from_repo(sup) + + source = AdaptorsSupervisor.source(sup) + + assert {:ok, {:ok, [pkg]}} = Cachex.get(cache, {:packages, source}) + assert pkg.name == "@openfn/language-http" + + assert {:ok, {:ok, icon_meta}} = + Cachex.get(cache, {:icon_meta, "@openfn/language-http", source}) + + assert Map.has_key?(icon_meta, :icon_square_ext) + assert Map.has_key?(icon_meta, :icon_rectangle_ext) + end + + test "overwrites existing keys without clearing unrelated ones", %{ + sup: sup, + cache: cache + } do + source = AdaptorsSupervisor.source(sup) + + Cachex.put!( + cache, + {:schema, "pre-existing", source}, + {:ok, %{"kept" => true}} + ) + + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + assert :ok = Store.warm_from_repo(sup) + + assert {:ok, {:ok, %{"kept" => true}}} = + Cachex.get(cache, {:schema, "pre-existing", source}) + end + end + + defp adaptor_record(overrides \\ []) do + overrides = Map.new(overrides) + + %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + description: "HTTP adaptor", + homepage: nil, + repository: nil, + license: "LGPL-3.0", + deprecated: false, + schema_data: nil, + schema_sha256: nil, + icon_square_ext: nil, + icon_rectangle_ext: nil, + icon_square_sha256: nil, + icon_rectangle_sha256: nil, + versions: [version_record("1.0.0")] + } + |> Map.merge(overrides) + end + + defp version_record(version) do + %{ + version: version, + integrity: "sha512-#{version}", + tarball_url: "https://example.com/x/-/x-#{version}.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + end +end diff --git a/test/lightning/adaptors/supervisor_integration_test.exs b/test/lightning/adaptors/supervisor_integration_test.exs new file mode 100644 index 00000000000..608466cb860 --- /dev/null +++ b/test/lightning/adaptors/supervisor_integration_test.exs @@ -0,0 +1,191 @@ +defmodule Lightning.Adaptors.SupervisorIntegrationTest do + @moduledoc """ + Integration-level tests for `Lightning.Adaptors.Supervisor`: prove all + Phase A children boot under a single `start_supervised!` call and that + the `:rest_for_one` cascade pins §6.5a (Invalidator subscribes at init; + if Cachex restarts without Invalidator restarting, the cache goes + stale). + """ + + use Lightning.DataCase, async: false + + import Eventually + + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + # Children with their *registered* names. We look up live PIDs by name + # (Process.whereis/1) rather than by child id from which_children/1, + # because module-based child specs share child ids like `Cachex` or + # `Lightning.Adaptors.Invalidator` — those don't carry the per-instance + # name we derive in the Supervisor. The Scheduler is registered via + # `:global` (HighlanderPG-wrapped) so it needs a `:global.whereis_name/1` + # lookup instead. + defp local_named_children(sup) do + %{ + cache: AdaptorsSupervisor.cache_name(sup), + tasks: AdaptorsSupervisor.tasks_name(sup), + invalidator: AdaptorsSupervisor.invalidator_name(sup), + node_monitor: AdaptorsSupervisor.node_monitor_name(sup), + broadcaster: AdaptorsSupervisor.channel_broadcaster_name(sup) + } + end + + defp scheduler_pid(sup) do + {:global, global_name} = AdaptorsSupervisor.global_scheduler_name(sup) + + case :global.whereis_name(global_name) do + :undefined -> nil + pid -> pid + end + end + + defp pids_by_role(sup) do + locals = + sup + |> local_named_children() + |> Enum.map(fn {role, registered_name} -> + {role, Process.whereis(registered_name)} + end) + |> Map.new() + + Map.put(locals, :scheduler, scheduler_pid(sup)) + end + + # HighlanderPG polls every 300ms by default; allow ~3s for the + # wrapped child to acquire the advisory lock and register globally. + @scheduler_wait_ms 3_000 + + setup do + sup = :"test_full_boot_#{System.unique_integer([:positive])}" + on_exit(fn -> AdaptorsSupervisor.forget(sup) end) + {:ok, sup: sup} + end + + describe "child-list boot" do + test "boots the full child list under one start_supervised! call", + %{sup: sup} do + pid = + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.Local} + ) + + children = Supervisor.which_children(pid) + + # Cachex + Task.Supervisor + Invalidator + NodeMonitor + + # ChannelBroadcaster + HighlanderPG(Scheduler) = 6. + assert length(children) == 6 + + ids = Enum.map(children, fn {id, _pid, _type, _mods} -> id end) + assert AdaptorsSupervisor.highlander_name(sup) in ids + + Enum.each(children, fn {_id, child_pid, _type, _mods} -> + assert is_pid(child_pid), + "unexpected child pid shape: #{inspect(child_pid)}" + + assert Process.alive?(child_pid), + "child pid #{inspect(child_pid)} is not alive" + end) + + # Locally-registered children are up under their derived names. + Enum.each(local_named_children(sup), fn {role, registered_name} -> + pid = Process.whereis(registered_name) + assert is_pid(pid), "expected #{role} to be registered and alive" + assert Process.alive?(pid) + end) + + # The HighlanderPG-wrapped Scheduler registers globally once it + # acquires the advisory lock — give it up to ~3s to do so. + assert_eventually(is_pid(scheduler_pid(sup)), @scheduler_wait_ms) + assert Process.alive?(scheduler_pid(sup)) + end + + test "exposes the per-instance strategy and source via :persistent_term", + %{sup: sup} do + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.Local} + ) + + assert AdaptorsSupervisor.strategy(sup) == Lightning.Adaptors.Local + assert AdaptorsSupervisor.source(sup) == :local + end + end + + describe ":rest_for_one strategy" do + test "Cachex crash cascades to Invalidator / ChannelBroadcaster / Scheduler", + %{sup: sup} do + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.Local} + ) + + # Block until the HighlanderPG-wrapped Scheduler has registered + # globally so we have a baseline pid to compare against. + assert_eventually(is_pid(scheduler_pid(sup)), @scheduler_wait_ms) + + before = pids_by_role(sup) + + cachex_pid = Map.fetch!(before, :cache) + assert is_pid(cachex_pid) + + ref = Process.monitor(cachex_pid) + Process.exit(cachex_pid, :kill) + assert_receive {:DOWN, ^ref, :process, ^cachex_pid, _}, 1_000 + + after_pids = wait_for_restart(sup, before) + + # Cachex itself comes back under a fresh pid. + assert Map.fetch!(after_pids, :cache) != cachex_pid + + # §6.5a: under :rest_for_one, all children that depend on Cachex + # (Invalidator, Broadcaster, Scheduler) must restart too so they + # re-bind to the fresh cache. + for role <- [:invalidator, :broadcaster, :scheduler] do + old = Map.fetch!(before, role) + new = Map.fetch!(after_pids, role) + assert is_pid(old) + assert is_pid(new) + + assert new != old, + "expected #{role} to restart after Cachex crash " <> + "(before=#{inspect(old)}, after=#{inspect(new)})" + end + end + end + + # Polls `pids_by_role/1` until the children we expect to be restarted + # show new PIDs, or we hit the deadline. Returns the post-restart map. + # The Scheduler restart goes through HighlanderPG (lock + poll cycle), + # so allow a slightly longer deadline than for the locally-registered + # children alone. + defp wait_for_restart(sup, before, deadline_ms \\ 3_000) do + start = System.monotonic_time(:millisecond) + roles_expected = [:invalidator, :broadcaster, :scheduler] + do_wait_for_restart(sup, before, roles_expected, start, deadline_ms) + end + + defp do_wait_for_restart(sup, before, roles, start, deadline_ms) do + current = pids_by_role(sup) + + changed? = + Enum.all?(roles, fn role -> + case {Map.get(before, role), Map.get(current, role)} do + {old, new} when is_pid(old) and is_pid(new) -> old != new + _ -> false + end + end) + + cond do + changed? -> + current + + System.monotonic_time(:millisecond) - start > deadline_ms -> + flunk( + "supervisor children did not restart within #{deadline_ms}ms; " <> + "before=#{inspect(before)} after=#{inspect(current)}" + ) + + true -> + Process.sleep(20) + do_wait_for_restart(sup, before, roles, start, deadline_ms) + end + end +end diff --git a/test/lightning/adaptors/supervisor_test.exs b/test/lightning/adaptors/supervisor_test.exs new file mode 100644 index 00000000000..a05304c8ea2 --- /dev/null +++ b/test/lightning/adaptors/supervisor_test.exs @@ -0,0 +1,178 @@ +defmodule Lightning.Adaptors.SupervisorTest do + use ExUnit.Case, async: true + + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + describe "start_link/1" do + test "raises KeyError when :name is missing" do + assert_raise KeyError, ~r/key :name not found/, fn -> + AdaptorsSupervisor.start_link([]) + end + end + + test "raises KeyError when opts has no :name key" do + assert_raise KeyError, fn -> + AdaptorsSupervisor.start_link(strategy: :ignored) + end + end + end + + describe "derived-name helpers" do + test "cache_name/1 concatenates `Cache` onto the supervisor name" do + assert AdaptorsSupervisor.cache_name(Lightning.Adaptors) == + Lightning.Adaptors.Cache + + assert AdaptorsSupervisor.cache_name(:MyAdaptors) == + Module.concat(:MyAdaptors, Cache) + end + + test "tasks_name/1 concatenates `Tasks` onto the supervisor name" do + assert AdaptorsSupervisor.tasks_name(Lightning.Adaptors) == + Lightning.Adaptors.Tasks + end + + test "invalidator_name/1 concatenates `Invalidator` onto the supervisor name" do + assert AdaptorsSupervisor.invalidator_name(Lightning.Adaptors) == + Lightning.Adaptors.Invalidator + end + + test "channel_broadcaster_name/1 concatenates `ChannelBroadcaster`" do + assert AdaptorsSupervisor.channel_broadcaster_name(Lightning.Adaptors) == + Lightning.Adaptors.ChannelBroadcaster + end + + test "node_monitor_name/1 concatenates `NodeMonitor`" do + assert AdaptorsSupervisor.node_monitor_name(Lightning.Adaptors) == + Lightning.Adaptors.NodeMonitor + end + + test "scheduler_name/1 concatenates `Scheduler`" do + assert AdaptorsSupervisor.scheduler_name(Lightning.Adaptors) == + Lightning.Adaptors.Scheduler + end + + test "source_topic/1 returns an `adaptors:` string" do + assert AdaptorsSupervisor.source_topic(Lightning.Adaptors) == + "adaptors:Lightning.Adaptors" + end + + test "client_topic/1 returns an `adaptors:client_update:` string" do + assert AdaptorsSupervisor.client_topic(Lightning.Adaptors) == + "adaptors:client_update:Lightning.Adaptors" + end + + test "source_topic/1 and client_topic/1 produce distinct strings" do + name = Lightning.Adaptors + + refute AdaptorsSupervisor.source_topic(name) == + AdaptorsSupervisor.client_topic(name) + end + + test "lock_key/1 derives an int via :erlang.phash2({:adaptors, name})" do + name = Lightning.Adaptors + + assert AdaptorsSupervisor.lock_key(name) == + :erlang.phash2({:adaptors, name}) + end + + test "lock_key/1 of a name differs from phash2 of just the name" do + name = :"Adaptors_#{System.unique_integer([:positive])}" + + assert AdaptorsSupervisor.lock_key(name) != :erlang.phash2(name) + end + end + + describe "two concurrent supervisors do not collide" do + test "derived Cachex / Task.Supervisor / GenServer names differ between instances" do + a = :"AdaptorsA_#{System.unique_integer([:positive])}" + b = :"AdaptorsB_#{System.unique_integer([:positive])}" + + assert AdaptorsSupervisor.cache_name(a) != + AdaptorsSupervisor.cache_name(b) + + assert AdaptorsSupervisor.tasks_name(a) != + AdaptorsSupervisor.tasks_name(b) + + assert AdaptorsSupervisor.invalidator_name(a) != + AdaptorsSupervisor.invalidator_name(b) + + assert AdaptorsSupervisor.channel_broadcaster_name(a) != + AdaptorsSupervisor.channel_broadcaster_name(b) + + assert AdaptorsSupervisor.node_monitor_name(a) != + AdaptorsSupervisor.node_monitor_name(b) + + assert AdaptorsSupervisor.scheduler_name(a) != + AdaptorsSupervisor.scheduler_name(b) + end + + test "PubSub topics differ between instances" do + a = :"AdaptorsA_#{System.unique_integer([:positive])}" + b = :"AdaptorsB_#{System.unique_integer([:positive])}" + + assert AdaptorsSupervisor.source_topic(a) != + AdaptorsSupervisor.source_topic(b) + + assert AdaptorsSupervisor.client_topic(a) != + AdaptorsSupervisor.client_topic(b) + end + + test "HighlanderPG lock keys differ between instances" do + a = :"AdaptorsA_#{System.unique_integer([:positive])}" + b = :"AdaptorsB_#{System.unique_integer([:positive])}" + + assert AdaptorsSupervisor.lock_key(a) != + AdaptorsSupervisor.lock_key(b) + end + + test "production-equivalent lock key is stable across calls" do + first = AdaptorsSupervisor.lock_key(Lightning.Adaptors) + second = AdaptorsSupervisor.lock_key(Lightning.Adaptors) + + assert first == second + assert is_integer(first) + end + end + + describe "init/1 (child spec list)" do + # End-to-end boot (Cachex / Task.Supervisor / Invalidator / + # NodeMonitor / ChannelBroadcaster / HighlanderPG-wrapped Scheduler) + # is covered by `Lightning.Adaptors.SupervisorIntegrationTest`, + # which needs `Lightning.DataCase` and a real Postgres connection + # (HighlanderPG opens its own dedicated connection on `Lightning.Repo`). + + test "init/1 requires the :name opt" do + assert_raise KeyError, ~r/key :name not found/, fn -> + AdaptorsSupervisor.init([]) + end + end + end + + describe "derived names for HighlanderPG-wrapped Scheduler" do + test "global_scheduler_name/1 returns the {:global, atom()} pair" do + assert AdaptorsSupervisor.global_scheduler_name(Lightning.Adaptors) == + {:global, Lightning.Adaptors.Scheduler} + end + + test "global_scheduler_name/1 differs between instances" do + a = :"AdaptorsA_#{System.unique_integer([:positive])}" + b = :"AdaptorsB_#{System.unique_integer([:positive])}" + + assert AdaptorsSupervisor.global_scheduler_name(a) != + AdaptorsSupervisor.global_scheduler_name(b) + end + + test "highlander_name/1 concatenates `HighlanderPG`" do + assert AdaptorsSupervisor.highlander_name(Lightning.Adaptors) == + Lightning.Adaptors.HighlanderPG + end + + test "highlander_name/1 differs between instances" do + a = :"AdaptorsA_#{System.unique_integer([:positive])}" + b = :"AdaptorsB_#{System.unique_integer([:positive])}" + + assert AdaptorsSupervisor.highlander_name(a) != + AdaptorsSupervisor.highlander_name(b) + end + end +end diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs new file mode 100644 index 00000000000..7d52228b327 --- /dev/null +++ b/test/lightning/adaptors_test.exs @@ -0,0 +1,279 @@ +defmodule Lightning.AdaptorsTest do + use Lightning.DataCase, async: false + + import Mox + + alias Lightning.Adaptors + alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Scheduler + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + setup :set_mox_global + setup :verify_on_exit! + + setup do + sup = :"adaptors_test_#{System.unique_integer([:positive])}" + + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + ) + + {:ok, sup: sup} + end + + defp adaptor_record(overrides \\ []) do + overrides = Map.new(overrides) + + %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + description: "HTTP adaptor", + homepage: nil, + repository: nil, + license: "LGPL-3.0", + deprecated: false, + schema_data: nil, + schema_sha256: nil, + icon_square_ext: nil, + icon_rectangle_ext: nil, + icon_square_sha256: nil, + icon_rectangle_sha256: nil, + versions: [version_record("1.0.0")] + } + |> Map.merge(overrides) + end + + defp version_record(version) do + %{ + version: version, + integrity: "sha512-#{version}", + tarball_url: "https://example.com/x/-/x-#{version}.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + end + + defp start_scheduler(sup) do + original_env = Application.get_env(:lightning, Lightning.Adaptors, []) + + Application.put_env( + :lightning, + Lightning.Adaptors, + Keyword.put(original_env, :refresh_interval, 99_999_999) + ) + + # Stop the supervisor's auto-started HighlanderPG (and its wrapped + # Scheduler) so we can start a replacement under the controlled + # interval without name collision. The test-owned Scheduler registers + # directly under the same `{:global, …}` name production callers use. + :ok = + Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) + + pid = + start_supervised!({ + Scheduler, + name: AdaptorsSupervisor.global_scheduler_name(sup), + sup: sup, + lock_key: AdaptorsSupervisor.lock_key(sup), + cache: AdaptorsSupervisor.cache_name(sup), + tasks: AdaptorsSupervisor.tasks_name(sup), + source_topic: AdaptorsSupervisor.source_topic(sup) + }) + + Application.put_env(:lightning, Lightning.Adaptors, original_env) + + pid + end + + describe "packages/1" do + test "returns packages from DB", %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + {:error, :unreachable} + end) + + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + + assert {:ok, [pkg]} = Adaptors.packages(sup) + assert pkg.name == "@openfn/language-http" + end + + test "returns {:ok, []} when DB is empty", %{sup: sup} do + assert {:ok, []} = Adaptors.packages(sup) + end + end + + describe "packages/0 delegates to packages(Lightning.Adaptors)" do + test "packages/0 and packages(Lightning.Adaptors) return identical results" do + # The production `Lightning.Adaptors.Supervisor` is started under the + # name `Lightning.Adaptors` in `application.ex`; in test it uses + # `Lightning.Adaptors.StrategyMock` per `config/test.exs`. Both forms + # resolve to `Store.packages(Lightning.Adaptors)`; equality is always + # guaranteed regardless of cache state. + assert Adaptors.packages() == Adaptors.packages(Lightning.Adaptors) + end + end + + describe "versions/2" do + test "delegates to Store.versions/2 and returns version list", %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + {:error, :unreachable} + end) + + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + + assert {:ok, [v]} = Adaptors.versions(sup, "@openfn/language-http") + assert v.version == "1.0.0" + end + + test "returns {:error, _} for unknown adaptor when strategy unavailable", %{ + sup: sup + } do + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + {:error, :not_found} + end) + + assert {:error, _} = Adaptors.versions(sup, "@openfn/does-not-exist") + end + end + + describe "schema/2" do + test "delegates to Store.schema/2 and returns schema", %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + {:error, :unreachable} + end) + + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record(schema_data: ~s({"type":"object"})) + ) + + assert {:ok, ~s({"type":"object"})} = + Adaptors.schema(sup, "@openfn/language-http") + end + + test "preserves JSON property order across the DB round-trip", %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + {:error, :unreachable} + end) + + ordered_body = ~s({"a":1,"z":2,"m":3}) + + {:ok, _} = + AdaptorsRepo.upsert_adaptor(adaptor_record(schema_data: ordered_body)) + + assert {:ok, ^ordered_body} = + Adaptors.schema(sup, "@openfn/language-http") + end + end + + describe "resolve_version/2" do + test "\"latest\" resolves from DB and returns latest_version" do + {:ok, _} = + AdaptorsRepo.upsert_adaptor(adaptor_record(latest_version: "2.3.4")) + + assert {:ok, "2.3.4"} = + Adaptors.resolve_version("@openfn/language-http", "latest") + end + + test "\"local\" resolves from DB and returns latest_version" do + {:ok, _} = + AdaptorsRepo.upsert_adaptor(adaptor_record(latest_version: "1.5.0")) + + assert {:ok, "1.5.0"} = + Adaptors.resolve_version("@openfn/language-http", "local") + end + + test "\"latest\" returns {:error, :not_found} when adaptor absent from DB" do + assert {:error, :not_found} = + Adaptors.resolve_version("@openfn/does-not-exist", "latest") + end + + test "concrete semver passes through without any DB lookup" do + # No adaptor in DB: if a lookup occurred the result would be :not_found. + # Pass-through means we get {:ok, version} regardless. + assert {:ok, "3.0.0"} = + Adaptors.resolve_version("@openfn/language-http", "3.0.0") + end + end + + describe "refresh_now/1" do + test "delegates to Scheduler.refresh_now via global_scheduler_name/1", %{ + sup: sup + } do + test_pid = self() + + # list_adaptors is called by the background Task that :tick spawns. + # With an empty DB the scheduler fires an init-tick immediately, so + # we must stub before start_scheduler and drain that first tick before + # calling refresh_now (which triggers a second tick). + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :tick_ran) + {:ok, []} + end) + + start_scheduler(sup) + assert_receive :tick_ran, 2000 + + assert :ok = Adaptors.refresh_now(sup) + assert_receive :tick_ran, 2000 + end + end + + describe "refresh_package/2" do + test "delegates to Scheduler.refresh_package via global_scheduler_name/1", %{ + sup: sup + } do + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _name -> + {:ok, adaptor_record(latest_version: "2.0.0")} + end) + + start_scheduler(sup) + + assert :ok = Adaptors.refresh_package(sup, "@openfn/language-http") + end + end + + describe "icon_meta/1,2" do + test "icon_meta is @doc false for all arities" do + {:docs_v1, _, :elixir, _, _, _, docs} = Code.fetch_docs(Lightning.Adaptors) + + icon_meta_docs = + Enum.filter(docs, fn + {{:function, :icon_meta, _}, _, _, _, _} -> true + _ -> false + end) + + refute Enum.empty?(icon_meta_docs) + + Enum.each(icon_meta_docs, fn doc -> + assert {{:function, :icon_meta, _}, _, _, :hidden, _} = doc + end) + end + + test "icon_meta/2 delegates to Store.icon_meta/2 for known adaptor", %{ + sup: sup + } do + {:ok, _} = + AdaptorsRepo.upsert_adaptor( + adaptor_record( + icon_square_ext: "svg", + icon_square_sha256: :crypto.hash(:sha256, "fake-svg-bytes") + ) + ) + + assert {:ok, meta} = Adaptors.icon_meta(sup, "@openfn/language-http") + assert meta.icon_square_ext == "svg" + end + + test "icon_meta/2 returns {:error, :not_found} for unknown adaptor", %{ + sup: sup + } do + assert {:error, :not_found} = + Adaptors.icon_meta(sup, "@openfn/never-existed") + end + end +end diff --git a/test/support/factories.ex b/test/support/factories.ex index c2ff7cf23c1..73946274845 100644 --- a/test/support/factories.ex +++ b/test/support/factories.ex @@ -4,6 +4,16 @@ defmodule Lightning.Factories do alias Lightning.Workflows.Snapshot + def adaptor_factory do + %Lightning.Adaptors.Repo.Adaptor{ + name: sequence(:adaptor_name, &"@openfn/language-test-#{&1}"), + source: :npm, + latest_version: "1.0.0", + checked_at: DateTime.utc_now(), + schema_data: nil + } + end + def webhook_auth_method_factory do %Lightning.Workflows.WebhookAuthMethod{ project: build(:project), diff --git a/test/test_helper.exs b/test/test_helper.exs index e9d51891b3f..c482a2e4b9a 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -5,6 +5,8 @@ Mox.defmock(Lightning.AuthProviders.OauthHTTPClient.Mock, for: Tesla.Adapter) Mox.defmock(Lightning.MockSentry, for: Lightning.SentryBehaviour) Mox.defmock(Lightning.Tesla.Mock, for: Tesla.Adapter) +Mox.defmock(Lightning.Adaptors.StrategyMock, for: Lightning.Adaptors.Strategy) + :ok = Application.ensure_started(:ex_machina) Mimic.copy(:hackney) @@ -70,5 +72,31 @@ Application.put_env(:lightning, Lightning.Extensions, external_metrics: Lightning.Extensions.ExternalMetrics ) +# Pin the `Lightning.Adaptors.IconCache` on-disk path to a per-OS-PID +# directory and wipe it at startup so: +# 1. Each `mix test` invocation begins with an empty icon cache — +# `System.unique_integer/1` resets per-VM and recycles, so without +# this, leftover files from a prior run can mask a Mox expectation +# by short-circuiting `IconCache.cached?/4`. +# 2. Concurrent `mix test` invocations (different tmux panes, parallel +# CI shards) use distinct directories and never collide — each BEAM +# has its own OS PID. +icon_dir = + Path.join([ + System.tmp_dir!(), + "lightning_test_icons", + System.pid() + ]) + +File.rm_rf!(icon_dir) +File.mkdir_p!(icon_dir) + +Application.put_env( + :lightning, + Lightning.Adaptors, + Application.get_env(:lightning, Lightning.Adaptors, []) + |> Keyword.put(:icon_path, icon_dir) +) + ExUnit.start() Ecto.Adapters.SQL.Sandbox.mode(Lightning.Repo, :manual) From 998e86d22ace00ed0eec24197556526d13e15d39 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Tue, 25 Aug 2026 09:19:40 +0200 Subject: [PATCH 02/37] Serve adaptor icons and the catalogue over HTTP, wire into the editor - Serve adaptor icons over HTTP, add a cacheable catalogue endpoint - Superuser maintenance page for refreshing the registry - Migrate AdaptorRegistry callers onto the Lightning.Adaptors facade - Send adaptors and icon URLs to the editor over the workflow channel, re-fetching the catalogue and broadcasting changed names on every adaptors_updated push - Configurable NPM upstream URLs; scheduler log/test trims and comment cleanup --- .env.example | 13 + RUNNINGLOCAL.md | 28 + .../js/collaborative-editor/api/adaptors.ts | 23 + .../components/AdaptorIcon.tsx | 39 +- .../components/AdaptorSelectionModal.tsx | 177 +++--- .../components/AdaptorSelector.tsx | 8 +- .../components/ConfigureAdaptorModal.tsx | 8 +- .../components/diagram/WorkflowDiagram.tsx | 7 +- .../components/ide/FullScreenIDE.tsx | 17 +- .../components/inspector/JobForm.tsx | 6 +- .../contexts/StoreProvider.tsx | 6 + .../collaborative-editor/hooks/useAdaptors.ts | 121 ++-- .../stores/createAdaptorStore.ts | 207 +++---- .../js/collaborative-editor/types/adaptor.ts | 27 +- assets/js/collaborative-editor/types/index.ts | 1 - .../components/MiniMapNode.tsx | 15 +- assets/js/workflow-diagram/nodes/Job.tsx | 25 +- .../collaborative-editor/api/adaptors.test.ts | 43 ++ .../components/AdaptorIcon.test.tsx | 87 +++ .../components/AdaptorSelectionModal.test.tsx | 122 ++-- .../components/ConfigureAdaptorModal.test.tsx | 39 +- .../ide/FullScreenIDE.docs-panel.test.tsx | 5 +- .../ide/FullScreenIDE.keyboard.test.tsx | 5 +- .../components/ide/FullScreenIDE.test.tsx | 11 +- .../contexts/StoreProvider.test.tsx | 23 + .../createAdaptorStore.test.ts | 569 +++++++++--------- .../fixtures/adaptorData.ts | 84 ++- .../collaborative-editor/useAdaptors.test.tsx | 150 ++--- .../components/MiniMapNode.test.tsx | 120 ++++ .../test/workflow-diagram/nodes/Job.test.tsx | 95 +++ bin/adaptor_cache | 233 +++++++ config/dev.exs | 12 - config/test.exs | 2 +- lib/lightning/adaptors.ex | 21 +- lib/lightning/adaptors/channel_broadcaster.ex | 57 +- lib/lightning/adaptors/npm.ex | 4 +- lib/lightning/adaptors/package_name.ex | 64 ++ lib/lightning/adaptors/repo.ex | 78 ++- lib/lightning/adaptors/repo_adaptor.ex | 5 +- .../adaptors/repo_adaptor_version.ex | 3 +- lib/lightning/adaptors/scheduler.ex | 89 ++- lib/lightning/adaptors/store.ex | 3 +- lib/lightning/adaptors/supervisor.ex | 13 +- lib/lightning/ai_assistant/ai_assistant.ex | 2 +- lib/lightning/config/bootstrap.ex | 22 + lib/lightning/credentials.ex | 11 +- .../channels/run_with_options.ex | 4 +- .../channels/workflow_channel.ex | 74 +-- .../components/layouts/settings.html.heex | 7 + .../controllers/adaptor_controller.ex | 48 ++ .../controllers/adaptor_icon_controller.ex | 149 +++++ .../live/maintenance_live/index.ex | 85 +++ .../live/maintenance_live/index.html.heex | 53 ++ lib/lightning_web/router.ex | 13 + lib/mix/tasks/lightning.refresh_adaptors.ex | 59 ++ ...27084128_add_adaptor_catalogue_indexes.exs | 8 + test/integration/web_and_worker_test.exs | 7 + .../adaptors/channel_broadcaster_test.exs | 165 ++--- .../adaptors/end_to_end_broadcast_test.exs | 28 +- .../adaptors/highlander_integration_test.exs | 9 +- test/lightning/adaptors/invalidator_test.exs | 4 +- test/lightning/adaptors/node_monitor_test.exs | 4 +- test/lightning/adaptors/npm/github_test.exs | 3 +- test/lightning/adaptors/npm/registry_test.exs | 5 +- test/lightning/adaptors/npm_test.exs | 7 +- test/lightning/adaptors/package_name_test.exs | 73 +++ .../adaptors/repo_catalogue_test.exs | 145 +++++ test/lightning/adaptors/scheduler_test.exs | 66 +- .../adaptors/supervisor_integration_test.exs | 10 +- .../ai_assistant/ai_assistant_test.exs | 8 +- test/lightning/config/bootstrap_test.exs | 34 ++ test/lightning/credentials/schema_test.exs | 51 ++ test/lightning/credentials_test.exs | 11 + .../channels/run_channel_test.exs | 12 + .../channels/run_with_options_test.exs | 42 +- .../channels/workflow_channel_test.exs | 182 ++++-- .../controllers/adaptor_controller_test.exs | 136 +++++ .../adaptor_icon_controller_test.exs | 564 +++++++++++++++++ .../live/credential_live_test.exs | 7 + .../live/maintenance_live/index_test.exs | 79 +++ test/lightning_web/live/project_live_test.exs | 8 + .../live/workflow_live/collaborate_test.exs | 8 + .../tasks/lightning.refresh_adaptors_test.exs | 85 +++ test/support/adaptor_test_helpers.ex | 253 ++++++++ test/test_helper.exs | 13 +- tooling/adaptor_cache/README.md | 136 +++++ tooling/adaptor_cache/docker-compose.yml | 25 + tooling/adaptor_cache/nginx.conf | 151 +++++ 88 files changed, 4236 insertions(+), 1295 deletions(-) create mode 100644 assets/js/collaborative-editor/api/adaptors.ts create mode 100644 assets/test/collaborative-editor/api/adaptors.test.ts create mode 100644 assets/test/collaborative-editor/components/AdaptorIcon.test.tsx create mode 100644 assets/test/workflow-diagram/components/MiniMapNode.test.tsx create mode 100644 assets/test/workflow-diagram/nodes/Job.test.tsx create mode 100755 bin/adaptor_cache create mode 100644 lib/lightning/adaptors/package_name.ex create mode 100644 lib/lightning_web/controllers/adaptor_controller.ex create mode 100644 lib/lightning_web/controllers/adaptor_icon_controller.ex create mode 100644 lib/lightning_web/live/maintenance_live/index.ex create mode 100644 lib/lightning_web/live/maintenance_live/index.html.heex create mode 100644 lib/mix/tasks/lightning.refresh_adaptors.ex create mode 100644 priv/repo/migrations/20260827084128_add_adaptor_catalogue_indexes.exs create mode 100644 test/lightning/adaptors/package_name_test.exs create mode 100644 test/lightning/adaptors/repo_catalogue_test.exs create mode 100644 test/lightning_web/controllers/adaptor_controller_test.exs create mode 100644 test/lightning_web/controllers/adaptor_icon_controller_test.exs create mode 100644 test/lightning_web/live/maintenance_live/index_test.exs create mode 100644 test/mix/tasks/lightning.refresh_adaptors_test.exs create mode 100644 test/support/adaptor_test_helpers.ex create mode 100644 tooling/adaptor_cache/README.md create mode 100644 tooling/adaptor_cache/docker-compose.yml create mode 100644 tooling/adaptor_cache/nginx.conf diff --git a/.env.example b/.env.example index e38d2b0c751..32883a4d40b 100644 --- a/.env.example +++ b/.env.example @@ -253,6 +253,19 @@ # LOCAL_ADAPTORS=true # OPENFN_ADAPTORS_REPO=/path/to/repo/ # OPENFN_ADAPTORS_REPO=/path/to/private,/path/to/canonical +# +# Lightning.Adaptors.NPM upstream URLs. Leave these unset in production: the +# defaults in lib/lightning/config/bootstrap.ex are the real npm, jsDelivr and +# raw.githubusercontent endpoints. Point them at the local caching reverse +# proxy while iterating on the adaptors subsystem, so refresh ticks are served +# from disk and work offline. Start it with `bin/adaptor_cache up`; see +# tooling/adaptor_cache/README.md. +# ADAPTOR_REGISTRY_URL=http://localhost:4874/npm +# ADAPTOR_JSDELIVR_URL=http://localhost:4874/jsdelivr +# ADAPTOR_GITHUB_URL=http://localhost:4874/github +# +# The git ref under OpenFn/adaptors that icons are read from. +# ADAPTOR_GITHUB_REF=main # ============================================================================== # <><><> WEBHOOK RETRY SETTINGS <><><> diff --git a/RUNNINGLOCAL.md b/RUNNINGLOCAL.md index fa1ba4cc220..2b86c1aabb0 100644 --- a/RUNNINGLOCAL.md +++ b/RUNNINGLOCAL.md @@ -209,6 +209,34 @@ Remember to re-generate the production schemas when you've finished, or else your local app will use the local schema versions until `install_schemas` is next run! +### Caching the adaptor upstreams + +Every adaptor registry refresh (background scheduler tick, or a manual +`mix lightning.refresh_adaptors`) makes a handful of npm, jsDelivr and GitHub +requests per changed package, which gets chatty fast if you're iterating on the +subsystem or just running `refresh_adaptors` repeatedly by hand. A local caching +reverse proxy under `tooling/adaptor_cache/` makes the second and every later +run local, with no network needed at all. + +```sh +bin/adaptor_cache up # start the proxy (first time must be online) +bin/adaptor_cache check # prove all three upstreams cache correctly +``` + +Then point Lightning at it: + +```sh +export ADAPTOR_REGISTRY_URL=http://localhost:4874/npm +export ADAPTOR_JSDELIVR_URL=http://localhost:4874/jsdelivr +export ADAPTOR_GITHUB_URL=http://localhost:4874/github +``` + +See `tooling/adaptor_cache/README.md` for the full command list, how to read the +cache logs, and the caveats. **Never** put this URL in your global `~/.npmrc`: +it proxies npm closely enough that `npm install` would appear to work, while +resolving against a week-stale packument and writing `localhost:4874` into +`package-lock.json`. + ### Problems with Apple Silicon You might run into some errors when running the docker containers on Apple diff --git a/assets/js/collaborative-editor/api/adaptors.ts b/assets/js/collaborative-editor/api/adaptors.ts new file mode 100644 index 00000000000..7d1abab12bc --- /dev/null +++ b/assets/js/collaborative-editor/api/adaptors.ts @@ -0,0 +1,23 @@ +/** + * HTTP client for the adaptor catalogue endpoint (`GET /adaptors/catalogue`). + * + * Session-cookie authenticated like every other route in this app; no + * project scoping, no CSRF header (plain GET). Zod validation of each + * catalogue entry happens downstream in AdaptorStore, not here. + */ + +export interface AdaptorCatalogueResponse { + data: unknown[]; +} + +export async function getAdaptorCatalogue(): Promise { + const response = await fetch('/adaptors/catalogue', { + credentials: 'same-origin', + }); + + if (!response.ok) { + throw new Error(response.statusText); + } + + return response.json() as Promise; +} diff --git a/assets/js/collaborative-editor/components/AdaptorIcon.tsx b/assets/js/collaborative-editor/components/AdaptorIcon.tsx index 51df8928377..918c157dbe3 100644 --- a/assets/js/collaborative-editor/components/AdaptorIcon.tsx +++ b/assets/js/collaborative-editor/components/AdaptorIcon.tsx @@ -1,6 +1,7 @@ -import useAdaptorIcons from '#/workflow-diagram/useAdaptorIcons'; +import { useContext, useMemo, useSyncExternalStore } from 'react'; -import { extractAdaptorName } from '../utils/adaptorUtils'; +import { StoreContext } from '../contexts/StoreProvider'; +import { extractAdaptorName, extractPackageName } from '../utils/adaptorUtils'; interface AdaptorIconProps { name: string; @@ -13,11 +14,35 @@ const sizeClasses = { lg: 'h-12 w-12', }; +// Reads the square icon URL for `name` directly from StoreContext, so callers +// that mock the `hooks/useAdaptors` module (e.g. FullScreenIDE tests) still get +// the existing placeholder fallback instead of crashing on a missing mock. +function useStoreIconUrl(name: string): string | null { + const context = useContext(StoreContext); + const adaptorStore = context?.adaptorStore ?? null; + const packageName = extractPackageName(name); + + const selectIconUrl = useMemo(() => { + if (!adaptorStore) return () => null; + return adaptorStore.withSelector(state => { + const found = state.adaptors.find(a => a.name === packageName); + return found?.icon_urls?.square ?? null; + }); + }, [adaptorStore, packageName]); + + const noopSubscribe = useMemo(() => () => () => {}, []); + + return useSyncExternalStore( + adaptorStore?.subscribe ?? noopSubscribe, + selectIconUrl + ); +} + export function AdaptorIcon({ name, size = 'md' }: AdaptorIconProps) { - const adaptorIconsData = useAdaptorIcons(); const displayName = extractAdaptorName(name) ?? null; + const iconUrl = useStoreIconUrl(name); - if (!adaptorIconsData || !displayName) { + if (!displayName) { return (
diff --git a/assets/js/collaborative-editor/components/AdaptorSelectionModal.tsx b/assets/js/collaborative-editor/components/AdaptorSelectionModal.tsx index 9a18635fc9a..cbaea21edb6 100644 --- a/assets/js/collaborative-editor/components/AdaptorSelectionModal.tsx +++ b/assets/js/collaborative-editor/components/AdaptorSelectionModal.tsx @@ -3,7 +3,12 @@ import { useEffect, useMemo, useState } from 'react'; import { useKeyboardShortcut } from '../keyboard'; -import { useAdaptors } from '../hooks/useAdaptors'; +import { + useAdaptorCommands, + useAdaptors, + useAdaptorsError, + useAdaptorsLoading, +} from '../hooks/useAdaptors'; import type { Adaptor } from '../types/adaptor'; import { getAdaptorDisplayName } from '../utils/adaptorUtils'; @@ -14,7 +19,7 @@ interface AdaptorSelectionModalProps { isOpen: boolean; onClose: () => void; onSelect: (adaptorSpec: string) => void; - projectAdaptors?: Adaptor[]; + adaptorsInUse?: Adaptor[]; } interface AdaptorWithDisplayName extends Adaptor { @@ -25,9 +30,14 @@ export function AdaptorSelectionModal({ isOpen, onClose, onSelect, - projectAdaptors = [], + adaptorsInUse = [], }: AdaptorSelectionModalProps) { const allAdaptors = useAdaptors(); + const isLoading = useAdaptorsLoading(); + const catalogueError = useAdaptorsError(); + const { requestAdaptors } = useAdaptorCommands(); + const catalogueLoading = isLoading && allAdaptors.length === 0; + const catalogueFailed = !!catalogueError && allAdaptors.length === 0; const [searchQuery, setSearchQuery] = useState(''); const [focusedIndex, setFocusedIndex] = useState(0); @@ -60,17 +70,17 @@ export function AdaptorSelectionModal({ [allAdaptors] ); - const projectAdaptorsWithDisplayNames = useMemo< + const adaptorsInUseWithDisplayNames = useMemo< AdaptorWithDisplayName[] >(() => { - return projectAdaptors.map(adaptor => ({ + return adaptorsInUse.map(adaptor => ({ ...adaptor, displayName: getAdaptorDisplayName(adaptor.name, { titleCase: true, fallback: adaptor.name, }), })); - }, [projectAdaptors]); + }, [adaptorsInUse]); const allAdaptorsWithDisplayNames = useMemo(() => { return allAdaptors.map(adaptor => ({ @@ -89,10 +99,10 @@ export function AdaptorSelectionModal({ // Filter project adaptors const filteredProject = searchQuery - ? projectAdaptorsWithDisplayNames.filter(adaptor => + ? adaptorsInUseWithDisplayNames.filter(adaptor => adaptor.displayName.toLowerCase().includes(lowerQuery) ) - : projectAdaptorsWithDisplayNames; + : adaptorsInUseWithDisplayNames; // Filter all adaptors and exclude duplicates from project adaptors const projectAdaptorNames = new Set(filteredProject.map(a => a.name)); @@ -130,7 +140,7 @@ export function AdaptorSelectionModal({ }; }, [ searchQuery, - projectAdaptorsWithDisplayNames, + adaptorsInUseWithDisplayNames, allAdaptorsWithDisplayNames, httpAdaptor, ]); @@ -142,7 +152,7 @@ export function AdaptorSelectionModal({ const handleRowClick = (adaptor: AdaptorWithDisplayName) => { // Construct full adaptor spec with semantic version - const adaptorSpec = `${adaptor.name}@${adaptor.latest}`; + const adaptorSpec = `${adaptor.name}@${adaptor.latest_version}`; // Immediately select and close (Figma design - no Continue button) onSelect(adaptorSpec); @@ -216,67 +226,96 @@ export function AdaptorSelectionModal({ >
- - {showingHttpFallback && ( -
-

- No adaptor found{' '} - for "{searchQuery}" -

-

- Try the HTTP adaptor below to connect to any system with - a REST API. -

-
- )} + {catalogueLoading ? ( +
+ + Loading adaptors... +
+ ) : catalogueFailed ? ( +
+

Couldn't load adaptors. Please try again.

+ +
+ ) : ( + + {showingHttpFallback && ( +
+

+ No adaptor found{' '} + for "{searchQuery}" +

+

+ Try the HTTP adaptor below to connect to any system + with a REST API. +

+
+ )} - {filteredProjectAdaptors.length > 0 && ( - - {filteredProjectAdaptors.map(adaptor => ( - } - onClick={() => handleRowClick(adaptor)} - focused={isAdaptorFocused(adaptor, 0)} - /> - ))} - - )} + {filteredProjectAdaptors.length > 0 && ( + + {filteredProjectAdaptors.map(adaptor => ( + } + onClick={() => handleRowClick(adaptor)} + focused={isAdaptorFocused(adaptor, 0)} + /> + ))} + + )} - {filteredAllAdaptors.length > 0 && ( - 0 - ? 'All adaptors' - : 'Available adaptors' - } - > - {filteredAllAdaptors.map(adaptor => ( - } - onClick={() => handleRowClick(adaptor)} - focused={isAdaptorFocused( - adaptor, - filteredProjectAdaptors.length - )} - /> - ))} - - )} - + {filteredAllAdaptors.length > 0 && ( + 0 + ? 'All adaptors' + : 'Available adaptors' + } + > + {filteredAllAdaptors.map(adaptor => ( + } + onClick={() => handleRowClick(adaptor)} + focused={isAdaptorFocused( + adaptor, + filteredProjectAdaptors.length + )} + /> + ))} + + )} + + )}
diff --git a/assets/js/collaborative-editor/components/AdaptorSelector.tsx b/assets/js/collaborative-editor/components/AdaptorSelector.tsx index ab63dd35159..29dfdd923b8 100644 --- a/assets/js/collaborative-editor/components/AdaptorSelector.tsx +++ b/assets/js/collaborative-editor/components/AdaptorSelector.tsx @@ -20,8 +20,8 @@ interface AdaptorSelectorProps { updateJob: (jobId: string, updates: Partial) => void; /** Setter to control configure modal state */ setIsConfigureModalOpen: (open: boolean) => void; - /** Available project adaptors */ - projectAdaptors: Adaptor[]; + /** Adaptors currently in use in this workflow */ + adaptorsInUse: Adaptor[]; /** Optional callback before adaptor change (for form sync in JobForm) */ onAdaptorChangeStart?: () => void; } @@ -45,7 +45,7 @@ export function AdaptorSelector({ job, updateJob, setIsConfigureModalOpen, - projectAdaptors, + adaptorsInUse, onAdaptorChangeStart, }: AdaptorSelectorProps) { const { @@ -90,7 +90,7 @@ export function AdaptorSelector({ isOpen={isOpen} onClose={handlePickerCloseGuarded} onSelect={handleAdaptorSelect} - projectAdaptors={projectAdaptors} + adaptorsInUse={adaptorsInUse} /> a.name === packageName); if (adaptor) { - const sortedVersions = sortVersionsDescending( - adaptor.versions.map(v => v.version) - ); + const sortedVersions = sortVersionsDescending(adaptor.versions); if (sortedVersions.length > 0 && sortedVersions[0]) { onVersionChange(sortedVersions[0]); @@ -420,9 +418,7 @@ export function ConfigureAdaptorModal({ return []; } - const sortedVersions = sortVersionsDescending( - adaptor.versions.map(v => v.version) - ); + const sortedVersions = sortVersionsDescending(adaptor.versions); return ['latest', ...sortedVersions]; }, [currentAdaptor, allAdaptors]); diff --git a/assets/js/collaborative-editor/components/diagram/WorkflowDiagram.tsx b/assets/js/collaborative-editor/components/diagram/WorkflowDiagram.tsx index 70f358778fc..12666342a16 100644 --- a/assets/js/collaborative-editor/components/diagram/WorkflowDiagram.tsx +++ b/assets/js/collaborative-editor/components/diagram/WorkflowDiagram.tsx @@ -11,7 +11,7 @@ import { import React, { useCallback, useEffect, useRef, useState } from 'react'; import tippy from 'tippy.js'; -import { useProjectAdaptors } from '#/collaborative-editor/hooks/useAdaptors'; +import { useAdaptorsInUse } from '#/collaborative-editor/hooks/useAdaptors'; import useConnect from '#/collaborative-editor/hooks/useConnect'; import { usePositions, @@ -201,8 +201,7 @@ export default function WorkflowDiagram(props: WorkflowDiagramProps) { position: { x: number; y: number }; } | null>(null); - // Fetch project adaptors for modal - const { projectAdaptors } = useProjectAdaptors(); + const { adaptorsInUse } = useAdaptorsInUse(); const updateSelection = useCallback( (id?: string | null) => { @@ -1024,7 +1023,7 @@ export default function WorkflowDiagram(props: WorkflowDiagramProps) { isOpen={pendingPlaceholder !== null} onClose={handleAdaptorModalClose} onSelect={handleAdaptorSelect} - projectAdaptors={projectAdaptors} + adaptorsInUse={adaptorsInUse} /> ); diff --git a/assets/js/collaborative-editor/components/ide/FullScreenIDE.tsx b/assets/js/collaborative-editor/components/ide/FullScreenIDE.tsx index 17c8887da9e..fb60aa7e492 100644 --- a/assets/js/collaborative-editor/components/ide/FullScreenIDE.tsx +++ b/assets/js/collaborative-editor/components/ide/FullScreenIDE.tsx @@ -27,7 +27,7 @@ import * as dataclipApi from '../../api/dataclips'; import { RENDER_MODES } from '../../constants/panel'; import { useCredentialModal } from '../../contexts/CredentialModalContext'; import { useMonacoRef } from '../../contexts/MonacoRefContext'; -import { useProjectAdaptors } from '../../hooks/useAdaptors'; +import { useAdaptorsInUse } from '../../hooks/useAdaptors'; import { useCredentials, useCredentialsCommands, @@ -398,7 +398,7 @@ export function FullScreenIDE({ const { projectCredentials, keychainCredentials } = useCredentials(); const { requestCredentials } = useCredentialsCommands(); - const { projectAdaptors, allAdaptors } = useProjectAdaptors(); + const { adaptorsInUse, allAdaptors } = useAdaptorsInUse(); const { updateJob } = useWorkflowActions(); // Credential modal is managed by the context @@ -413,19 +413,20 @@ export function FullScreenIDE({ // to be used by components that can't make use of 'latest' const currJobAdaptor = useMemo(() => { if (!currentJob?.adaptor) { - const latestCommon = projectAdaptors.find( + const latestCommon = adaptorsInUse.find( a => a.name === '@openfn/language-common' - )?.versions?.[0]?.version; + )?.latest_version; return `@openfn/language-common@${latestCommon || 'latest'}`; } const resolved = resolveAdaptor(currentJob.adaptor); if (resolved.version !== 'latest') return currentJob?.adaptor; - const latestVersion = projectAdaptors.find(a => a.name === resolved.package) - ?.versions?.[0]?.version; + const latestVersion = adaptorsInUse.find( + a => a.name === resolved.package + )?.latest_version; // If version not found, return original adaptor string if (!latestVersion) return currentJob.adaptor; return `${resolved.package}@${latestVersion}`; - }, [projectAdaptors, currentJob?.adaptor]); + }, [adaptorsInUse, currentJob?.adaptor]); // Run/Retry functionality for IDE Header const { canRun: canRunSnapshot, tooltipMessage: runTooltipMessage } = @@ -1380,7 +1381,7 @@ export function FullScreenIDE({ job={currentJob} updateJob={updateJob} setIsConfigureModalOpen={setIsConfigureModalOpen} - projectAdaptors={projectAdaptors} + adaptorsInUse={adaptorsInUse} /> )} diff --git a/assets/js/collaborative-editor/components/inspector/JobForm.tsx b/assets/js/collaborative-editor/components/inspector/JobForm.tsx index 201ed6593d3..53863e9db79 100644 --- a/assets/js/collaborative-editor/components/inspector/JobForm.tsx +++ b/assets/js/collaborative-editor/components/inspector/JobForm.tsx @@ -3,7 +3,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useAppForm } from '#/collaborative-editor/components/form'; import { useCredentialModal } from '#/collaborative-editor/contexts/CredentialModalContext'; -import { useProjectAdaptors } from '#/collaborative-editor/hooks/useAdaptors'; +import { useAdaptorsInUse } from '#/collaborative-editor/hooks/useAdaptors'; import { useCredentials, useCredentialsCommands, @@ -55,7 +55,7 @@ export function JobForm({ job }: JobFormProps) { const { updateJob } = useWorkflowActions(); const { projectCredentials, keychainCredentials } = useCredentials(); const { requestCredentials } = useCredentialsCommands(); - const { projectAdaptors, allAdaptors } = useProjectAdaptors(); + const { adaptorsInUse, allAdaptors } = useAdaptorsInUse(); const { isReadOnly } = useWorkflowReadOnly(); // Modal state for adaptor configuration @@ -395,7 +395,7 @@ export function JobForm({ job }: JobFormProps) { job={job} updateJob={updateJob} setIsConfigureModalOpen={setIsConfigureModalOpen} - projectAdaptors={projectAdaptors} + adaptorsInUse={adaptorsInUse} onAdaptorChangeStart={syncAdaptorToForm} />
diff --git a/assets/js/collaborative-editor/contexts/StoreProvider.tsx b/assets/js/collaborative-editor/contexts/StoreProvider.tsx index 9577a86c81b..c05e548d336 100644 --- a/assets/js/collaborative-editor/contexts/StoreProvider.tsx +++ b/assets/js/collaborative-editor/contexts/StoreProvider.tsx @@ -152,6 +152,12 @@ export const StoreProvider = ({ children }: StoreProviderProps) => { }; }); + // Fetch the adaptor catalogue over HTTP as soon as the store mounts, + // independent of Phoenix channel connection/document sync. + useEffect(() => { + void stores.adaptorStore.requestAdaptors(); + }, [stores.adaptorStore]); + // Bridge the SessionContextStore's `isNewWorkflow` flag up to SessionProvider // so the channel-join `action` stays honest across in-place reconnects. // diff --git a/assets/js/collaborative-editor/hooks/useAdaptors.ts b/assets/js/collaborative-editor/hooks/useAdaptors.ts index 208bdb790c3..245bc6646ab 100644 --- a/assets/js/collaborative-editor/hooks/useAdaptors.ts +++ b/assets/js/collaborative-editor/hooks/useAdaptors.ts @@ -10,7 +10,8 @@ import { useSyncExternalStore, useContext, useMemo } from 'react'; import { StoreContext } from '../contexts/StoreProvider'; import type { AdaptorStoreInstance } from '../stores/createAdaptorStore'; import type { Adaptor } from '../types/adaptor'; -import type { Job } from '../types/workflow'; +import type { Workflow } from '../types/workflow'; +import { extractPackageName } from '../utils/adaptorUtils'; /** * Main hook for accessing the AdaptorStore instance @@ -86,88 +87,88 @@ export const useAdaptor = (name: string): Adaptor | null => { }; /** - * Extracts adaptor package name from a full adaptor specifier - * e.g., "@openfn/language-common@1.0.0" -> "@openfn/language-common" + * Hook to read an adaptor's square-shape icon URL from the AdaptorStore. + * + * Accepts a full adaptor specifier (with or without version suffix). When no + * StoreProvider is mounted (e.g. the LiveView workflow-editor path), returns + * `null` rather than throwing so consumers fall back to their string label. */ -const getAdaptorPackageName = (adaptor: string | undefined): string | null => { - if (!adaptor) return null; - const match = adaptor.match(/^(@[^@]+)@/); - return match ? match[1] : null; +export const useAdaptorIconUrl = ( + adaptor: string | null | undefined +): string | null => { + const context = useContext(StoreContext); + const adaptorStore = context?.adaptorStore ?? null; + + const packageName = adaptor ? extractPackageName(adaptor) : null; + + const selectIconUrl = useMemo(() => { + if (!adaptorStore) return () => null; + return adaptorStore.withSelector(state => { + if (!packageName) return null; + const found = state.adaptors.find(a => a.name === packageName); + return found?.icon_urls?.square ?? null; + }); + }, [adaptorStore, packageName]); + + const noopSubscribe = useMemo(() => () => () => {}, []); + + return useSyncExternalStore( + adaptorStore?.subscribe ?? noopSubscribe, + selectIconUrl + ); }; /** - * Hook to get project-specific adaptors and all adaptors - * Returns both project adaptors and all adaptors from backend endpoint - * - * Project adaptors are merged from two sources: - * 1. Backend DB (saved jobs) - * 2. Y.Doc state (unsaved jobs in collaborative editor) - * - * This ensures newly added adaptors appear in projectAdaptors before saving. + * Hook to derive the subset of the adaptor catalogue that is referenced by jobs + * in the current Y.Doc workflow. Pure selector: the catalogue comes from the + * AdaptorStore's HTTP fetch, the jobs from the collaborative workflow store. */ -export const useProjectAdaptors = (): { - projectAdaptors: Adaptor[]; +export const useAdaptorsInUse = (): { + adaptorsInUse: Adaptor[]; allAdaptors: Adaptor[]; isLoading: boolean; } => { const context = useContext(StoreContext); if (!context) { - throw new Error('useProjectAdaptors must be used within a StoreProvider'); + throw new Error('useAdaptorsInUse must be used within a StoreProvider'); } const { adaptorStore, workflowStore } = context; - // Get adaptor state from adaptor store - const selectAdaptorData = adaptorStore.withSelector(state => ({ - backendProjectAdaptors: state.projectAdaptors || [], - allAdaptors: state.adaptors, - isLoading: state.isLoading, - })); + const selectAdaptors = adaptorStore.withSelector(state => state.adaptors); + const selectIsLoading = adaptorStore.withSelector(state => state.isLoading); + const selectJobs = workflowStore.withSelector(state => state.jobs); - const adaptorData = useSyncExternalStore( + const allAdaptors = useSyncExternalStore( adaptorStore.subscribe, - selectAdaptorData + selectAdaptors + ); + const isLoading = useSyncExternalStore( + adaptorStore.subscribe, + selectIsLoading + ); + const jobs: Workflow.Job[] = useSyncExternalStore( + workflowStore.subscribe, + selectJobs ); - // Get jobs from workflow store (Y.Doc state) - const selectJobs = workflowStore.withSelector(state => state.jobs); - const jobs: Job[] = useSyncExternalStore(workflowStore.subscribe, selectJobs); - - // Merge backend project adaptors with Y.Doc job adaptors - const projectAdaptors = useMemo(() => { - const { backendProjectAdaptors, allAdaptors } = adaptorData; - - // Get adaptor names already in backend list - const backendAdaptorNames = new Set( - backendProjectAdaptors.map(a => a.name) - ); + const adaptorsInUse = useMemo(() => { + if (jobs.length === 0) return []; - // Find adaptors used in Y.Doc jobs that aren't in backend list - const ydocAdaptorNames = new Set(); + const names = new Set(); for (const job of jobs) { - const packageName = getAdaptorPackageName(job.adaptor); - if (packageName && !backendAdaptorNames.has(packageName)) { - ydocAdaptorNames.add(packageName); - } + if (!job.adaptor) continue; + names.add(extractPackageName(job.adaptor)); } - // If no new adaptors from Y.Doc, return backend list as-is - if (ydocAdaptorNames.size === 0) { - return backendProjectAdaptors; - } - - // Find full adaptor objects from allAdaptors for Y.Doc adaptors - const ydocAdaptors = allAdaptors.filter(a => ydocAdaptorNames.has(a.name)); - - // Merge and sort - return [...backendProjectAdaptors, ...ydocAdaptors].sort((a, b) => - a.name.localeCompare(b.name) - ); - }, [adaptorData, jobs]); + return allAdaptors + .filter(a => names.has(a.name)) + .sort((a, b) => a.name.localeCompare(b.name)); + }, [allAdaptors, jobs]); return { - projectAdaptors, - allAdaptors: adaptorData.allAdaptors, - isLoading: adaptorData.isLoading, + adaptorsInUse, + allAdaptors, + isLoading, }; }; diff --git a/assets/js/collaborative-editor/stores/createAdaptorStore.ts b/assets/js/collaborative-editor/stores/createAdaptorStore.ts index 562147b5680..d379b398d77 100644 --- a/assets/js/collaborative-editor/stores/createAdaptorStore.ts +++ b/assets/js/collaborative-editor/stores/createAdaptorStore.ts @@ -12,25 +12,15 @@ * * ## Update Patterns: * - * ### Pattern 1: Channel Message → Immer → Notify (Server Updates) + * ### Pattern 1: Channel Signal → HTTP Re-fetch → Immer → Notify (Server Updates) * **When to use**: All server-initiated adaptor updates - * **Flow**: Channel message → validate with Zod → Immer update → React notification - * **Benefits**: Automatic validation, error handling, type safety - * - * ```typescript - * // Example: Handle server adaptor list update - * const handleAdaptorsUpdate = (rawData: unknown) => { - * const result = AdaptorsListSchema.safeParse(rawData); - * if (result.success) { - * state = produce(state, (draft) => { - * draft.adaptors = result.data; - * draft.lastUpdated = Date.now(); - * draft.error = null; - * }); - * notify(); - * } - * }; - * ``` + * **Flow**: `adaptors_updated` channel push (a name-only signal, no adaptor + * data) → re-fetch the catalogue over HTTP → validate the response with Zod + * → Immer update → React notification + * **Benefits**: Automatic validation, error handling, type safety. Every + * update goes through the same HTTP path as the initial load, so + * `handleAdaptorsReceived` (below) is the only place that writes + * `state.adaptors` from server data. * * ### Pattern 2: Direct Immer → Notify (Local State) * **When to use**: Loading states, errors, local UI state @@ -82,7 +72,7 @@ import type { PhoenixChannelProvider } from 'y-phoenix-channel'; import _logger from '#/utils/logger'; -import { channelRequest } from '../hooks/useChannel'; +import { getAdaptorCatalogue } from '../api/adaptors'; import { type Adaptor, type AdaptorState, @@ -96,6 +86,29 @@ import { wrapStoreWithDevTools } from './devtools'; const logger = _logger.ns('AdaptorStore').seal(); +// Deep-equality check tailored to the Adaptor shape so referential identity is +// preserved across no-op `adaptors_updated` pushes. +function adaptorsEqual(a: Adaptor, b: Adaptor): boolean { + if (a === b) return true; + if ( + a.name !== b.name || + a.repository !== b.repository || + a.latest_version !== b.latest_version || + a.icon_urls.square !== b.icon_urls.square || + a.icon_urls.rectangle !== b.icon_urls.rectangle || + a.versions.length !== b.versions.length + ) { + return false; + } + for (let i = 0; i < a.versions.length; i++) { + const aVer = a.versions[i]; + const bVer = b.versions[i]; + if (aVer === undefined || bVer === undefined) return false; + if (aVer !== bVer) return false; + } + return true; +} + // sorts adaptors coming into the adaptor store // 1. sorts the versions for every adaptor // 2. sorts the adaptors themselves by name @@ -104,7 +117,7 @@ export function sortAdaptors(adaptors: AdaptorsList = []) { for (const adaptor of adaptors) { // spreading because it could be read-only. const versions = [...(adaptor.versions || [])].sort((a, b) => - b.version.localeCompare(a.version) + b.localeCompare(a) ); sortedAdaptors.push({ ...adaptor, versions }); } @@ -119,7 +132,6 @@ export const createAdaptorStore = (): AdaptorStore => { let state: AdaptorState = produce( { adaptors: [], - projectAdaptors: [], isLoading: false, error: null, lastUpdated: null, @@ -159,7 +171,7 @@ export const createAdaptorStore = (): AdaptorStore => { const withSelector = createWithSelector(getSnapshot); // ============================================================================= - // PATTERN 1: Channel Message → Immer → Notify (Server Updates) + // PATTERN 1: Channel Signal → HTTP Re-fetch → Immer → Notify (Server Updates) // ============================================================================= /** @@ -170,14 +182,36 @@ export const createAdaptorStore = (): AdaptorStore => { const result = AdaptorsListSchema.safeParse(rawData); if (result.success) { - const adaptors = sortAdaptors(result.data); - - state = produce(state, draft => { - draft.adaptors = adaptors; - draft.isLoading = false; - draft.error = null; - draft.lastUpdated = Date.now(); + const incoming = sortAdaptors(result.data); + const existing = state.adaptors; + const existingByName = new Map(existing.map(a => [a.name, a])); + + // Merge by name to preserve referential identity of unchanged adaptors so + // `withSelector` consumers don't re-render on no-op `adaptors_updated` + // pushes. + const merged: Adaptor[] = incoming.map(next => { + const prev = existingByName.get(next.name); + return prev && adaptorsEqual(prev, next) ? prev : next; }); + + const arrayUnchanged = + merged.length === existing.length && + merged.every((a, i) => a === existing[i]); + + if (arrayUnchanged) { + state = produce(state, draft => { + draft.isLoading = false; + draft.error = null; + draft.lastUpdated = Date.now(); + }); + } else { + state = produce(state, draft => { + draft.adaptors = merged; + draft.isLoading = false; + draft.error = null; + draft.lastUpdated = Date.now(); + }); + } notify('handleAdaptorsReceived'); } else { const errorMessage = `Invalid adaptors data: ${result.error.message}`; @@ -194,14 +228,6 @@ export const createAdaptorStore = (): AdaptorStore => { } }; - /** - * Handle real-time adaptors update from server - */ - const handleAdaptorsUpdated = (rawData: unknown) => { - // Same validation logic as handleAdaptorsReceived - handleAdaptorsReceived(rawData); - }; - // ============================================================================= // PATTERN 2: Direct Immer → Notify (Local State) // ============================================================================= @@ -241,22 +267,18 @@ export const createAdaptorStore = (): AdaptorStore => { // CHANNEL INTEGRATION // ============================================================================= - let channelProvider: PhoenixChannelProvider | null = null; - /** * Connect to Phoenix channel provider for real-time updates */ const connectChannel = (provider: PhoenixChannelProvider) => { - channelProvider = provider; - - const adaptorsListHandler = (message: unknown) => { - logger.debug('Received adaptors_list message', message); - handleAdaptorsReceived(message); - }; - - const adaptorsUpdatedHandler = (message: unknown) => { - logger.debug('Received adaptors_updated message', message); - handleAdaptorsUpdated(message); + // The push only signals that named adaptors changed; it carries no + // adaptor data. Always re-fetch the catalogue over HTTP rather than + // branching on which names changed -- a brand-new adaptor needs the + // fetch regardless, and 304 caching makes re-fetching a known one just + // as cheap. + const adaptorsUpdatedHandler = () => { + logger.debug('Received adaptors_updated signal, refreshing catalogue'); + void requestAdaptors(); }; // Set up channel listeners @@ -266,42 +288,31 @@ export const createAdaptorStore = (): AdaptorStore => { devtools.connect(); + // Refresh the catalogue on (re)connect so a user who leaves the editor + // open across an adaptor publish still sees the new version. The + // `adaptors_updated` push alone isn't a reliable substitute for this. void requestAdaptors(); - void requestProjectAdaptors(); return () => { devtools.disconnect(); if (provider.channel) { - provider.channel.off('adaptors_list', adaptorsListHandler); provider.channel.off('adaptors_updated', adaptorsUpdatedHandler); } - channelProvider = null; }; }; /** - * Request adaptors from server via channel + * Request the adaptor catalogue over HTTP. Independent of Phoenix channel + * connection/document sync so the picker can populate as soon as the app + * mounts. */ const requestAdaptors = async (): Promise => { - if (!channelProvider?.channel) { - logger.warn('Cannot request adaptors - no channel connected'); - setError('No connection available'); - return; - } - setLoading(true); clearError(); try { - const response = await channelRequest<{ adaptors: unknown }>( - channelProvider.channel, - 'request_adaptors', - {} - ); - - if (response.adaptors) { - handleAdaptorsReceived(response.adaptors); - } + const response = await getAdaptorCatalogue(); + handleAdaptorsReceived(response.data); } catch (error) { logger.error('Adaptor request failed', error); setError( @@ -310,56 +321,6 @@ export const createAdaptorStore = (): AdaptorStore => { } }; - /** - * Request project adaptors from server via channel - */ - const requestProjectAdaptors = async (): Promise => { - if (!channelProvider?.channel) { - logger.warn('Cannot request project adaptors - no channel connected'); - setError('No connection available'); - return; - } - - setLoading(true); - clearError(); - - try { - logger.debug('Requesting project adaptors'); - const response = await channelRequest( - channelProvider.channel, - 'request_project_adaptors', - {} - ); - - if (response && typeof response === 'object') { - const { project_adaptors, all_adaptors } = response as { - project_adaptors: unknown; - all_adaptors: unknown; - }; - - const projectResult = AdaptorsListSchema.safeParse(project_adaptors); - const allResult = AdaptorsListSchema.safeParse(all_adaptors); - - if (projectResult.success && allResult.success) { - state = produce(state, draft => { - draft.projectAdaptors = sortAdaptors(projectResult.data); - draft.adaptors = sortAdaptors(allResult.data); - draft.isLoading = false; - draft.error = null; - }); - notify('requestProjectAdaptors'); - } else { - const errorMessage = 'Invalid project adaptors data'; - logger.error(errorMessage, { projectResult, allResult }); - setError(errorMessage); - } - } - } catch (error) { - logger.error('Project adaptors request failed', error); - setError('Failed to request project adaptors'); - } - }; - // ============================================================================= // QUERY HELPERS // ============================================================================= @@ -370,7 +331,7 @@ export const createAdaptorStore = (): AdaptorStore => { const getLatestVersion = (adaptorName: string): string | null => { const adaptor = findAdaptorByName(adaptorName); - return adaptor?.latest || null; + return adaptor?.latest_version || null; }; const getVersions = (adaptorName: string) => { @@ -390,7 +351,6 @@ export const createAdaptorStore = (): AdaptorStore => { // Commands (CQS pattern) requestAdaptors, - requestProjectAdaptors, setAdaptors, setLoading, setError, @@ -403,13 +363,6 @@ export const createAdaptorStore = (): AdaptorStore => { // Internal methods (not part of public AdaptorStore interface) _connectChannel: connectChannel, - // Test helper to set project adaptors directly - _setProjectAdaptors: (adaptors: Adaptor[]) => { - state = produce(state, draft => { - draft.projectAdaptors = sortAdaptors(adaptors); - }); - notify('_setProjectAdaptors'); - }, }; }; diff --git a/assets/js/collaborative-editor/types/adaptor.ts b/assets/js/collaborative-editor/types/adaptor.ts index 58ff2a42efe..a682521a25c 100644 --- a/assets/js/collaborative-editor/types/adaptor.ts +++ b/assets/js/collaborative-editor/types/adaptor.ts @@ -13,10 +13,11 @@ import { z } from 'zod'; // ============================================================================= /** - * Individual adaptor version schema + * Square and rectangle icon URLs for an adaptor. Either may be unavailable. */ -export const AdaptorVersionSchema = z.object({ - version: z.string(), +export const AdaptorIconUrlsSchema = z.object({ + square: z.string().nullable(), + rectangle: z.string().nullable(), }); /** @@ -24,9 +25,10 @@ export const AdaptorVersionSchema = z.object({ */ export const AdaptorSchema = z.object({ name: z.string(), - versions: z.array(AdaptorVersionSchema), - repo: z.string(), - latest: z.string(), + latest_version: z.string(), + versions: z.array(z.string()), + repository: z.string().nullable(), + icon_urls: AdaptorIconUrlsSchema, }); /** @@ -38,11 +40,6 @@ export const AdaptorsListSchema = z.array(AdaptorSchema); // TYPESCRIPT TYPES (Compile-time) // ============================================================================= -/** - * Individual adaptor version - */ -export type AdaptorVersion = z.infer; - /** * Single adaptor with all its versions and metadata */ @@ -60,9 +57,6 @@ export interface AdaptorState { /** Current list of available adaptors */ adaptors: AdaptorsList; - /** Project-specific adaptors used across workflows */ - projectAdaptors: AdaptorsList; - /** Loading state for initial fetch */ isLoading: boolean; @@ -80,9 +74,6 @@ export interface AdaptorCommands { /** Request adaptors list from server */ requestAdaptors: () => Promise; - /** Request project-specific adaptors from server */ - requestProjectAdaptors: () => Promise; - /** Manually set adaptors (for testing/fallback) */ setAdaptors: (adaptors: AdaptorsList) => void; @@ -116,7 +107,7 @@ export interface AdaptorQueries { getLatestVersion: (adaptorName: string) => string | null; /** Get all versions for adaptor */ - getVersions: (adaptorName: string) => AdaptorVersion[]; + getVersions: (adaptorName: string) => string[]; } /** diff --git a/assets/js/collaborative-editor/types/index.ts b/assets/js/collaborative-editor/types/index.ts index 2cd94ed59a1..57a3bf8cdb0 100644 --- a/assets/js/collaborative-editor/types/index.ts +++ b/assets/js/collaborative-editor/types/index.ts @@ -12,7 +12,6 @@ export type { AdaptorState, AdaptorStore, AdaptorsList, - AdaptorVersion, } from './adaptor'; export type { Job, JobCreate, JobUpdate } from './job'; export type { AwarenessUser, Session } from './session'; diff --git a/assets/js/workflow-diagram/components/MiniMapNode.tsx b/assets/js/workflow-diagram/components/MiniMapNode.tsx index 3c788d260a2..5e1752e4675 100644 --- a/assets/js/workflow-diagram/components/MiniMapNode.tsx +++ b/assets/js/workflow-diagram/components/MiniMapNode.tsx @@ -2,9 +2,9 @@ import { ClockIcon, GlobeAltIcon } from '@heroicons/react/24/outline'; import type { MiniMapNodeProps } from '@xyflow/react'; import { memo } from 'react'; +import { useAdaptorIconUrl } from '#/collaborative-editor/hooks/useAdaptors'; + import { useWorkflowStore } from '../../workflow-store/store'; -import useAdaptorIcons from '../useAdaptorIcons'; -import getAdaptorName from '../util/get-adaptor-name'; type Trigger = { id: string; @@ -54,11 +54,12 @@ const MiniMapNode = ({ const storeData = useWorkflowStore(); const jobs = propJobs ?? storeData.jobs; const triggers = propTriggers ?? storeData.triggers; - const adaptorIconsData = useAdaptorIcons(); // Check if this node is a trigger by looking it up in the triggers array const trigger = triggers.find((trigger: Trigger) => trigger.id === id); const isTrigger = !!trigger; + const job = jobs.find((job: Job) => job.id === id); + const icon = useAdaptorIconUrl(job?.adaptor); // For triggers, we'll use the appropriate icon if (isTrigger) { @@ -94,14 +95,6 @@ const MiniMapNode = ({ ); } - // For jobs, we'll use the adaptor icon if available - const job = jobs.find((job: Job) => job.id === id); - const adaptor = job?.adaptor ? getAdaptorName(job.adaptor) : null; - const icon = - adaptor && adaptorIconsData && adaptor in adaptorIconsData - ? adaptorIconsData[adaptor]?.square - : null; - // Fallback to rectangle if no icon is available return ( diff --git a/assets/js/workflow-diagram/nodes/Job.tsx b/assets/js/workflow-diagram/nodes/Job.tsx index 1e71c0a7b17..9ce572a6a7b 100644 --- a/assets/js/workflow-diagram/nodes/Job.tsx +++ b/assets/js/workflow-diagram/nodes/Job.tsx @@ -1,8 +1,9 @@ import { Position, type NodeProps } from '@xyflow/react'; import { memo } from 'react'; +import { useAdaptorIconUrl } from '#/collaborative-editor/hooks/useAdaptors'; + import PathButton from '../components/PathButton'; -import useAdaptorIcons, { type AdaptorIconData } from '../useAdaptorIcons'; import getAdaptorName from '../util/get-adaptor-name'; import Node from './Node'; @@ -22,10 +23,9 @@ const JobNode = ({ ], ]; - const adaptorIconsData = useAdaptorIcons(); - const adaptor = getAdaptorName(props.data?.adaptor); - const icon = getAdaptorIcon(adaptor, adaptorIconsData); + const iconUrl = useAdaptorIconUrl(props.data?.adaptor); + const icon = iconUrl ? {adaptor} : adaptor; return ( ; - } else { - return adaptor; - } - } catch { - return adaptor; - } -} - export default memo(JobNode); diff --git a/assets/test/collaborative-editor/api/adaptors.test.ts b/assets/test/collaborative-editor/api/adaptors.test.ts new file mode 100644 index 00000000000..e0a9469c5dc --- /dev/null +++ b/assets/test/collaborative-editor/api/adaptors.test.ts @@ -0,0 +1,43 @@ +/** + * Tests for the adaptor catalogue HTTP client + */ + +import { afterEach, describe, expect, test, vi } from 'vitest'; + +import { getAdaptorCatalogue } from '../../../js/collaborative-editor/api/adaptors'; + +describe('getAdaptorCatalogue', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + test('fetches the catalogue from the same-origin route with credentials', async () => { + const mockData = [{ name: '@openfn/language-http' }]; + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ data: mockData }), + }); + vi.stubGlobal('fetch', fetchMock); + + const result = await getAdaptorCatalogue(); + + expect(fetchMock).toHaveBeenCalledWith('/adaptors/catalogue', { + credentials: 'same-origin', + }); + expect(result).toEqual({ data: mockData }); + }); + + test('throws when the response is not ok', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + ok: false, + statusText: 'Internal Server Error', + }) + ); + + await expect(getAdaptorCatalogue()).rejects.toThrow( + 'Internal Server Error' + ); + }); +}); diff --git a/assets/test/collaborative-editor/components/AdaptorIcon.test.tsx b/assets/test/collaborative-editor/components/AdaptorIcon.test.tsx new file mode 100644 index 00000000000..f50f0484d01 --- /dev/null +++ b/assets/test/collaborative-editor/components/AdaptorIcon.test.tsx @@ -0,0 +1,87 @@ +/** + * Tests for AdaptorIcon component + * + * Verifies that icon URLs are read from the AdaptorStore (icon_urls.square) + * with the existing first-letter placeholder fallback when no URL is present + * or the adaptor is not in the store. + */ + +import { render, screen } from '@testing-library/react'; +import { describe, expect, it } from 'vitest'; + +import { AdaptorIcon } from '../../../js/collaborative-editor/components/AdaptorIcon'; +import { + StoreContext, + type StoreContextValue, +} from '../../../js/collaborative-editor/contexts/StoreProvider'; +import { createAdaptorStore } from '../../../js/collaborative-editor/stores/createAdaptorStore'; +import type { Adaptor } from '../../../js/collaborative-editor/types/adaptor'; + +function renderWithAdaptors(ui: React.ReactElement, adaptors: Adaptor[]) { + const adaptorStore = createAdaptorStore(); + adaptorStore.setAdaptors(adaptors); + + const stores = { + adaptorStore, + credentialStore: {} as StoreContextValue['credentialStore'], + metadataStore: {} as StoreContextValue['metadataStore'], + awarenessStore: {} as StoreContextValue['awarenessStore'], + workflowStore: {} as StoreContextValue['workflowStore'], + sessionContextStore: {} as StoreContextValue['sessionContextStore'], + historyStore: {} as StoreContextValue['historyStore'], + uiStore: {} as StoreContextValue['uiStore'], + editorPreferencesStore: {} as StoreContextValue['editorPreferencesStore'], + aiAssistantStore: {} as StoreContextValue['aiAssistantStore'], + } satisfies StoreContextValue; + + return render( + {ui} + ); +} + +describe('AdaptorIcon', () => { + it('renders icon_urls.square as an when populated in the store', () => { + const url = '/adaptor-icons/salesforce/square-abc.png'; + renderWithAdaptors( + , + [ + { + name: '@openfn/language-salesforce', + versions: ['2.0.0'], + repository: 'https://example.com', + latest_version: '2.0.0', + icon_urls: { square: url, rectangle: null }, + }, + ] + ); + + const img = screen.getByAltText('salesforce'); + expect(img.tagName).toBe('IMG'); + expect(img.getAttribute('src')).toContain(url); + }); + + it('renders the first-letter placeholder when icon_urls.square is null', () => { + renderWithAdaptors( + , + [ + { + name: '@openfn/language-salesforce', + versions: ['2.0.0'], + repository: 'https://example.com', + latest_version: '2.0.0', + icon_urls: { square: null, rectangle: '/some-rectangle.png' }, + }, + ] + ); + + expect(screen.queryByRole('img')).toBeNull(); + expect(screen.getByText('S')).toBeInTheDocument(); + }); + + it('renders the first-letter placeholder when the adaptor is not in the store', () => { + renderWithAdaptors(, []); + + expect(screen.queryByRole('img')).toBeNull(); + expect(screen.getByText('H')).toBeInTheDocument(); + }); +}); diff --git a/assets/test/collaborative-editor/components/AdaptorSelectionModal.test.tsx b/assets/test/collaborative-editor/components/AdaptorSelectionModal.test.tsx index b01e538bb81..c312176f321 100644 --- a/assets/test/collaborative-editor/components/AdaptorSelectionModal.test.tsx +++ b/assets/test/collaborative-editor/components/AdaptorSelectionModal.test.tsx @@ -21,15 +21,17 @@ vi.mock('#/workflow-diagram/useAdaptorIcons', () => ({ const mockProjectAdaptors: Adaptor[] = [ { name: '@openfn/language-http', - latest: '1.0.0', - versions: [{ version: '1.0.0' }, { version: '0.9.0' }], - repo: 'git+https://github.com/openfn/adaptors.git', + latest_version: '1.0.0', + versions: ['1.0.0', '0.9.0'], + repository: 'git+https://github.com/openfn/adaptors.git', + icon_urls: { square: null, rectangle: null }, }, { name: '@openfn/language-salesforce', - latest: '2.1.0', - versions: [{ version: '2.1.0' }, { version: '2.0.0' }], - repo: 'git+https://github.com/openfn/adaptors.git', + latest_version: '2.1.0', + versions: ['2.1.0', '2.0.0'], + repository: 'git+https://github.com/openfn/adaptors.git', + icon_urls: { square: null, rectangle: null }, }, ]; @@ -37,38 +39,39 @@ const mockAllAdaptors: Adaptor[] = [ ...mockProjectAdaptors, { name: '@openfn/language-dhis2', - latest: '3.2.1', - versions: [{ version: '3.2.1' }, { version: '3.2.0' }], - repo: 'git+https://github.com/openfn/adaptors.git', + latest_version: '3.2.1', + versions: ['3.2.1', '3.2.0'], + repository: 'git+https://github.com/openfn/adaptors.git', + icon_urls: { square: null, rectangle: null }, }, { name: '@openfn/language-common', - latest: '2.0.0', - versions: [{ version: '2.0.0' }, { version: '1.9.0' }], - repo: 'git+https://github.com/openfn/adaptors.git', + latest_version: '2.0.0', + versions: ['2.0.0', '1.9.0'], + repository: 'git+https://github.com/openfn/adaptors.git', + icon_urls: { square: null, rectangle: null }, }, ]; // Mock store context with proper structure -function createMockStoreContext() { +function createMockStoreContext( + isLoading = false, + error: string | null = null +) { + const snapshot = { + adaptors: isLoading || error ? [] : mockAllAdaptors, + projectAdaptors: mockProjectAdaptors, + isLoading, + error, + }; return { adaptorStore: { subscribe: vi.fn(() => vi.fn()), - getSnapshot: vi.fn(() => ({ - adaptors: mockAllAdaptors, - projectAdaptors: mockProjectAdaptors, - isLoading: false, - error: null, - })), - withSelector: vi.fn( - selector => () => - selector({ - adaptors: mockAllAdaptors, - projectAdaptors: mockProjectAdaptors, - isLoading: false, - error: null, - }) - ), + getSnapshot: vi.fn(() => snapshot), + withSelector: vi.fn(selector => () => selector(snapshot)), + requestAdaptors: vi.fn(), + setAdaptors: vi.fn(), + clearError: vi.fn(), }, credentialStore: { subscribe: vi.fn(() => vi.fn()), @@ -125,7 +128,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> ); @@ -147,6 +150,47 @@ describe('AdaptorSelectionModal', () => { screen.queryByPlaceholderText('Search for an adaptor to connect...') ).not.toBeInTheDocument(); }); + + it('shows a loading state instead of the search list while the catalogue is loading', () => { + renderWithProviders( + , + createMockStoreContext(true) + ); + + expect(screen.getByTestId('adaptor-list-loading')).toBeInTheDocument(); + expect( + screen.queryByPlaceholderText('Search for an adaptor to connect...') + ).not.toBeInTheDocument(); + }); + + it('shows an error state with a retry affordance when the catalogue fetch fails', () => { + const mockStoreContext = createMockStoreContext(false, 'Server error'); + + renderWithProviders( + , + mockStoreContext + ); + + expect(screen.getByTestId('adaptor-list-error')).toBeInTheDocument(); + expect( + screen.queryByPlaceholderText('Search for an adaptor to connect...') + ).not.toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Retry' })); + expect( + mockStoreContext.adaptorStore.requestAdaptors + ).toHaveBeenCalledTimes(1); + }); }); describe('adaptor display', () => { @@ -156,7 +200,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> ); @@ -172,7 +216,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> ); @@ -187,7 +231,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={[]} + adaptorsInUse={[]} /> ); @@ -203,7 +247,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> ); @@ -220,7 +264,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> ); @@ -243,7 +287,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> ); @@ -263,7 +307,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> ); @@ -287,7 +331,7 @@ describe('AdaptorSelectionModal', () => { isOpen={isOpen} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> @@ -329,7 +373,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> ); @@ -349,7 +393,7 @@ describe('AdaptorSelectionModal', () => { isOpen={true} onClose={onClose} onSelect={onSelect} - projectAdaptors={mockProjectAdaptors} + adaptorsInUse={mockProjectAdaptors} /> ); diff --git a/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx b/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx index 0d9ef4cc236..c6cd02b7623 100644 --- a/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx +++ b/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx @@ -38,26 +38,24 @@ vi.mock('#/workflow-diagram/useAdaptorIcons', () => ({ const mockProjectAdaptors: Adaptor[] = [ { name: '@openfn/language-http', - latest: '1.5.0', - versions: [ - { version: '1.5.0' }, - { version: '1.0.0' }, - { version: '0.9.0' }, - ], + latest_version: '1.5.0', + versions: ['1.5.0', '1.0.0', '0.9.0'], + repository: 'https://github.com/openfn/language-http', + icon_urls: { square: null, rectangle: null }, }, { name: '@openfn/language-salesforce', - latest: '2.1.0', - versions: [ - { version: '2.1.0' }, - { version: '2.0.0' }, - { version: '1.9.0' }, - ], + latest_version: '2.1.0', + versions: ['2.1.0', '2.0.0', '1.9.0'], + repository: 'https://github.com/openfn/language-salesforce', + icon_urls: { square: null, rectangle: null }, }, { name: '@openfn/language-common', - latest: '2.0.0', - versions: [{ version: '2.0.0' }], + latest_version: '2.0.0', + versions: ['2.0.0'], + repository: 'https://github.com/openfn/language-common', + icon_urls: { square: null, rectangle: null }, }, ]; @@ -414,15 +412,10 @@ describe('ConfigureAdaptorModal', () => { // Create adaptor with versions that need semantic sorting const adaptorWithManyVersions: Adaptor = { name: '@openfn/language-test', - latest: '10.0.0', - versions: [ - { version: '2.0.0' }, - { version: '10.0.0' }, - { version: '1.9.0' }, - { version: '9.0.0' }, - { version: '1.10.0' }, - ], - repo: 'https://github.com/openfn/language-test', + latest_version: '10.0.0', + versions: ['2.0.0', '10.0.0', '1.9.0', '9.0.0', '1.10.0'], + repository: 'https://github.com/openfn/language-test', + icon_urls: { square: null, rectangle: null }, }; renderWithProviders( diff --git a/assets/test/collaborative-editor/components/ide/FullScreenIDE.docs-panel.test.tsx b/assets/test/collaborative-editor/components/ide/FullScreenIDE.docs-panel.test.tsx index 30e15910e60..002a8895719 100644 --- a/assets/test/collaborative-editor/components/ide/FullScreenIDE.docs-panel.test.tsx +++ b/assets/test/collaborative-editor/components/ide/FullScreenIDE.docs-panel.test.tsx @@ -268,10 +268,11 @@ vi.mock('../../../../js/collaborative-editor/hooks/useCredentials', () => ({ // Mock adaptor hooks vi.mock('../../../../js/collaborative-editor/hooks/useAdaptors', () => ({ - useProjectAdaptors: () => ({ - projectAdaptors: [], + useAdaptorsInUse: () => ({ + adaptorsInUse: [], allAdaptors: [], }), + useAdaptorsLoading: () => false, })); // Mock LiveView actions diff --git a/assets/test/collaborative-editor/components/ide/FullScreenIDE.keyboard.test.tsx b/assets/test/collaborative-editor/components/ide/FullScreenIDE.keyboard.test.tsx index 4d78ef55770..ce7092de91b 100644 --- a/assets/test/collaborative-editor/components/ide/FullScreenIDE.keyboard.test.tsx +++ b/assets/test/collaborative-editor/components/ide/FullScreenIDE.keyboard.test.tsx @@ -363,14 +363,15 @@ vi.mock('../../../../js/collaborative-editor/hooks/useCredentials', () => ({ // Mock adaptor hooks vi.mock('../../../../js/collaborative-editor/hooks/useAdaptors', () => ({ - useProjectAdaptors: () => ({ - projectAdaptors: [], + useAdaptorsInUse: () => ({ + adaptorsInUse: [], allAdaptors: [], }), useAdaptors: () => ({ adaptors: [], loading: false, }), + useAdaptorsLoading: () => false, })); // Mock awareness hooks diff --git a/assets/test/collaborative-editor/components/ide/FullScreenIDE.test.tsx b/assets/test/collaborative-editor/components/ide/FullScreenIDE.test.tsx index b89142dfa8b..a10ecb5c024 100644 --- a/assets/test/collaborative-editor/components/ide/FullScreenIDE.test.tsx +++ b/assets/test/collaborative-editor/components/ide/FullScreenIDE.test.tsx @@ -327,11 +327,18 @@ vi.mock('../../../../js/collaborative-editor/hooks/useAdaptors', () => ({ loading: false, error: null, }), - useProjectAdaptors: () => ({ - projectAdaptors: [], + useAdaptorsInUse: () => ({ + adaptorsInUse: [], allAdaptors: [], }), useAdaptors: () => [], + useAdaptorsLoading: () => false, + useAdaptorsError: () => null, + useAdaptorCommands: () => ({ + requestAdaptors: vi.fn(), + setAdaptors: vi.fn(), + clearError: vi.fn(), + }), })); // Mock credentials hooks diff --git a/assets/test/collaborative-editor/contexts/StoreProvider.test.tsx b/assets/test/collaborative-editor/contexts/StoreProvider.test.tsx index 9c8c28d5fd2..973b605db80 100644 --- a/assets/test/collaborative-editor/contexts/StoreProvider.test.tsx +++ b/assets/test/collaborative-editor/contexts/StoreProvider.test.tsx @@ -13,6 +13,7 @@ import { useContext } from 'react'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import * as Y from 'yjs'; +import * as adaptorsApi from '../../../js/collaborative-editor/api/adaptors'; import { StoreContext, StoreProvider, @@ -34,7 +35,10 @@ import { // TEST SETUP & FIXTURES // ============================================================================= +vi.mock('../../../js/collaborative-editor/api/adaptors'); + const mockUseSession = vi.spyOn(useSessionModule, 'useSession'); +const getAdaptorCatalogueMock = vi.mocked(adaptorsApi.getAdaptorCatalogue); const createMockSessionState = ( overrides?: Partial @@ -75,6 +79,8 @@ const createMockUserData = () => ({ describe('StoreProvider', () => { beforeEach(() => { mockUseSession.mockReturnValue(createMockSessionState()); + getAdaptorCatalogueMock.mockReset(); + getAdaptorCatalogueMock.mockResolvedValue({ data: [] }); }); afterEach(() => { @@ -121,6 +127,23 @@ describe('StoreProvider', () => { }); }); + // =========================================================================== + // ADAPTOR CATALOGUE FETCH-ON-MOUNT TESTS + // =========================================================================== + + describe('adaptor catalogue fetch', () => { + test('fetches the catalogue over HTTP on mount, independent of channel connection', () => { + // Default beforeEach session state has no provider/channel. + render( + +
+ + ); + + expect(getAdaptorCatalogueMock).toHaveBeenCalledTimes(1); + }); + }); + // =========================================================================== // STORE INDEPENDENCE TESTS // =========================================================================== diff --git a/assets/test/collaborative-editor/createAdaptorStore.test.ts b/assets/test/collaborative-editor/createAdaptorStore.test.ts index e976b221e17..5efb968c030 100644 --- a/assets/test/collaborative-editor/createAdaptorStore.test.ts +++ b/assets/test/collaborative-editor/createAdaptorStore.test.ts @@ -1,40 +1,21 @@ /** * Tests for createAdaptorStore * - * This test suite covers all aspects of the AdaptorStore: - * - Core store interface (subscribe/getSnapshot) - * - State management commands (setLoading, setError, etc.) - * - Channel integration and message handling - * - Query helpers (findAdaptorByName, getLatestVersion, etc.) - * - Error handling and validation + * Covers the core store interface (subscribe/getSnapshot), state management + * commands, HTTP-backed `requestAdaptors`, Phoenix channel `adaptors_updated` + * live-update handling, and query helpers. */ -/** - * Test Fixtures - * - * This file uses Vitest 3.x fixtures for cleaner test setup and automatic cleanup. - * - * Available fixtures: - * - store: AdaptorStore instance (auto cleanup) - * - mockChannel: Mock Phoenix channel - * - mockProvider: Mock Phoenix channel provider (depends on mockChannel) - * - connectedStore: Store with channel connected (auto cleanup) - * - * Usage: - * adaptorTest("test name", async ({ connectedStore }) => { - * const { store, provider } = connectedStore; - * // test logic - cleanup automatic - * }); - */ - -import { describe, test, expect } from 'vitest'; +import { describe, test, expect, vi, beforeEach } from 'vitest'; +import * as adaptorsApi from '../../js/collaborative-editor/api/adaptors'; import { createAdaptorStore } from '../../js/collaborative-editor/stores/createAdaptorStore'; import type { AdaptorStoreInstance } from '../../js/collaborative-editor/stores/createAdaptorStore'; import { mockAdaptorsList, mockAdaptor, + mockAdaptorGmail, invalidAdaptorData, } from './fixtures/adaptorData.js'; import { @@ -42,54 +23,31 @@ import { createMockPhoenixChannelProvider, waitForCondition, } from './mocks/phoenixChannel.js'; -import { - createMockChannelPushOk, - createMockChannelPushError, -} from './__helpers__/channelMocks'; import type { MockPhoenixChannel, MockPhoenixChannelProvider, } from './mocks/phoenixChannel.js'; +vi.mock('../../js/collaborative-editor/api/adaptors'); + +const getAdaptorCatalogueMock = vi.mocked(adaptorsApi.getAdaptorCatalogue); + // Define fixture types interface AdaptorTestFixtures { store: AdaptorStoreInstance; mockChannel: MockPhoenixChannel; mockProvider: MockPhoenixChannelProvider; - connectedStore: { - store: AdaptorStoreInstance; - provider: MockPhoenixChannelProvider; - cleanup: () => void; - }; } -// Vitest 3.x fixtures for cleaner test setup and automatic cleanup const adaptorTest = test.extend({ store: async ({}, use) => { - const store = createAdaptorStore(); - await use(store); - // Automatic cleanup - store doesn't need explicit cleanup + await use(createAdaptorStore()); }, - mockChannel: async ({}, use) => { - const channel = createMockPhoenixChannel(); - await use(channel); - // Channel cleanup happens automatically + await use(createMockPhoenixChannel()); }, - mockProvider: async ({ mockChannel }, use) => { - const provider = createMockPhoenixChannelProvider(mockChannel); - await use(provider); - }, - - connectedStore: async ({ store, mockProvider }, use) => { - // Setup: connect channel to store - const cleanup = store._connectChannel(mockProvider as any); - - await use({ store, provider: mockProvider, cleanup }); - - // Automatic cleanup - cleanup(); + await use(createMockPhoenixChannelProvider(mockChannel)); }, }); @@ -99,312 +57,329 @@ function getSortedAdaptors(adaptors: any[]) { .sort((a, b) => a.name.localeCompare(b.name)) .map(adaptor => ({ ...adaptor, - versions: [...adaptor.versions].sort((a, b) => - b.version.localeCompare(a.version) - ), + versions: [...adaptor.versions].sort((a, b) => b.localeCompare(a)), })); } -describe('createAdaptorStore', () => { - describe('initialization', () => { - test('getSnapshot returns initial state', () => { - const store = createAdaptorStore(); - const initialState = store.getSnapshot(); +beforeEach(() => { + getAdaptorCatalogueMock.mockReset(); +}); - expect(initialState.adaptors).toEqual([]); - expect(initialState.isLoading).toBe(false); - expect(initialState.error).toBe(null); - expect(initialState.lastUpdated).toBe(null); +describe('createAdaptorStore', () => { + test('initializes with default state', () => { + const store = createAdaptorStore(); + expect(store.getSnapshot()).toEqual({ + adaptors: [], + isLoading: false, + error: null, + lastUpdated: null, }); }); - describe('subscriptions', () => { - test('subscribe/unsubscribe functionality works correctly', () => { - const store = createAdaptorStore(); - let callCount = 0; - - const listener = () => { - callCount++; - }; - - // Subscribe to changes - const unsubscribe = store.subscribe(listener); + test('subscribe/unsubscribe and multiple subscribers behave correctly', () => { + const store = createAdaptorStore(); + let count1 = 0; + let count2 = 0; + const unsubscribe1 = store.subscribe(() => count1++); + const unsubscribe2 = store.subscribe(() => count2++); + + store.setLoading(true); + expect(count1).toBe(1); + expect(count2).toBe(1); + + unsubscribe2(); + store.clearError(); + expect(count1).toBe(2); + expect(count2).toBe(1); // unsubscribed, no longer notified + + unsubscribe1(); + store.setLoading(false); + expect(count1).toBe(2); // unsubscribed, no longer notified + }); - // Trigger a state change - store.setLoading(true); + test('withSelector returns a referentially stable value until its slice changes', () => { + const store = createAdaptorStore(); + const selectAdaptors = store.withSelector(state => state.adaptors); + const selectIsLoading = store.withSelector(state => state.isLoading); - expect(callCount).toBe(1); + const adaptorsBefore = selectAdaptors(); + const loadingBefore = selectIsLoading(); - // Unsubscribe and trigger change - unsubscribe(); - store.clearError(); + store.setLoading(true); - // Listener should not be called after unsubscribe - expect(callCount).toBe(1); - }); + expect(selectAdaptors()).toBe(adaptorsBefore); // unrelated slice unchanged + expect(selectIsLoading()).not.toBe(loadingBefore); + }); - test('withSelector creates memoized selector with referential stability', () => { - const store = createAdaptorStore(); + test('setLoading/setError/clearError/setAdaptors transition state as expected', () => { + const store = createAdaptorStore(); - const selectAdaptors = store.withSelector(state => state.adaptors); - const selectIsLoading = store.withSelector(state => state.isLoading); + store.setLoading(true); + expect(store.getSnapshot().isLoading).toBe(true); - // Initial calls - const adaptors1 = selectAdaptors(); - const loading1 = selectIsLoading(); + store.setError('Test error message'); + let state = store.getSnapshot(); + expect(state.error).toBe('Test error message'); + expect(state.isLoading).toBe(false); // setError clears loading - // Change unrelated state - adaptors selector should return same reference - store.setLoading(true); - const adaptors3 = selectAdaptors(); - const loading3 = selectIsLoading(); + store.clearError(); + expect(store.getSnapshot().error).toBeNull(); - // Unrelated state change should not affect memoized selector - expect(adaptors1).toBe(adaptors3); - // Related state change should return new value - expect(loading1).not.toBe(loading3); - }); + const timestamp = Date.now(); + store.setAdaptors(mockAdaptorsList); + state = store.getSnapshot(); + expect(state.adaptors).toEqual(mockAdaptorsList); + expect(state.error).toBeNull(); + expect(state.lastUpdated).toBeGreaterThanOrEqual(timestamp); + }); - test('handles multiple subscribers correctly', () => { + describe('requestAdaptors (HTTP)', () => { + test('fetches, validates, sorts and stores a valid catalogue', async () => { + getAdaptorCatalogueMock.mockResolvedValue({ data: mockAdaptorsList }); const store = createAdaptorStore(); - let listener1Count = 0; - let listener2Count = 0; + await store.requestAdaptors(); - const unsubscribe1 = store.subscribe(() => { - listener1Count++; - }); - const unsubscribe2 = store.subscribe(() => { - listener2Count++; - }); + const state = store.getSnapshot(); + expect(state.adaptors).toEqual(getSortedAdaptors(mockAdaptorsList)); + expect(state.isLoading).toBe(false); + expect(state.error).toBeNull(); + expect(state.lastUpdated).toBeGreaterThan(0); + }); - // Trigger change - store.setLoading(true); + test('records a validation error and leaves adaptors empty on invalid entries', async () => { + getAdaptorCatalogueMock.mockResolvedValue({ + data: [invalidAdaptorData.missingName], + }); + const store = createAdaptorStore(); - expect(listener1Count).toBe(1); - expect(listener2Count).toBe(1); + await store.requestAdaptors(); - // Unsubscribe middle listener - unsubscribe2(); + const state = store.getSnapshot(); + expect(state.adaptors).toHaveLength(0); + expect(state.isLoading).toBe(false); + expect(state.error).toContain('Invalid adaptors data'); + }); - // Trigger another change - store.setError('test'); + test('records an error when the fetch itself rejects', async () => { + getAdaptorCatalogueMock.mockRejectedValue(new Error('Server error')); + const store = createAdaptorStore(); - // Unsubscribed listener should not be called - expect(listener2Count).toBe(1); + await store.requestAdaptors(); - // Cleanup - unsubscribe1(); + const state = store.getSnapshot(); + expect(state.adaptors).toHaveLength(0); + expect(state.error).toContain('Failed to request adaptors'); + expect(state.isLoading).toBe(false); }); - }); - describe('state management', () => { - test('handles state transitions for loading, error, and data correctly', () => { + test('sets isLoading synchronously while the request is in flight', async () => { + let resolveFetch: (value: { + data: typeof mockAdaptorsList; + }) => void = () => {}; + getAdaptorCatalogueMock.mockReturnValue( + new Promise(resolve => { + resolveFetch = resolve; + }) + ); const store = createAdaptorStore(); - let notificationCount = 0; - - store.subscribe(() => { - notificationCount++; - }); - // Test loading state transitions - store.setLoading(true); + const pending = store.requestAdaptors(); expect(store.getSnapshot().isLoading).toBe(true); - expect(notificationCount).toBe(1); - store.setLoading(false); + resolveFetch({ data: mockAdaptorsList }); + await pending; + expect(store.getSnapshot().isLoading).toBe(false); - expect(notificationCount).toBe(2); + }); - // Test error state transitions - store.setLoading(true); - const errorMessage = 'Test error message'; - store.setError(errorMessage); - let state = store.getSnapshot(); - expect(state.error).toBe(errorMessage); - expect(state.isLoading).toBe(false); // Setting error clears loading + test('clears a stale error from a previous failed fetch as soon as a retry starts', async () => { + getAdaptorCatalogueMock.mockRejectedValueOnce(new Error('Server error')); + const store = createAdaptorStore(); + await store.requestAdaptors(); + expect(store.getSnapshot().error).toContain('Failed to request adaptors'); - store.clearError(); + getAdaptorCatalogueMock.mockResolvedValueOnce({ data: mockAdaptorsList }); + const pending = store.requestAdaptors(); expect(store.getSnapshot().error).toBeNull(); - // Test adaptors state updates - const timestamp = Date.now(); - store.setAdaptors(mockAdaptorsList); - state = store.getSnapshot(); - expect(state.adaptors).toEqual(mockAdaptorsList); - expect(state.error).toBeNull(); - expect(state.lastUpdated).toBeGreaterThanOrEqual(timestamp); - - // Test rapid state updates maintain consistency - store.setLoading(true); - store.setError('error 1'); - store.clearError(); - store.setAdaptors(mockAdaptorsList); - store.setLoading(false); - store.setError('error 2'); - store.clearError(); - - // Final state should be consistent - const finalState = store.getSnapshot(); - expect(finalState.adaptors).toEqual(mockAdaptorsList); - expect(finalState.isLoading).toBe(false); - expect(finalState.error).toBeNull(); - expect(finalState.lastUpdated).toBeGreaterThan(0); + await pending; + expect(store.getSnapshot().error).toBeNull(); }); }); - describe('Phoenix channel integration', () => { - describe('requestAdaptors', () => { - adaptorTest( - 'processes valid and invalid data via channel', - async ({ mockChannel, mockProvider }) => { - // Test successful response with valid data - const store1 = createAdaptorStore(); - mockChannel.push = createMockChannelPushOk({ - adaptors: mockAdaptorsList, - }); - store1._connectChannel(mockProvider as any); - await store1.requestAdaptors(); - - let state = store1.getSnapshot(); - const expectedSortedAdaptors = getSortedAdaptors(mockAdaptorsList); - - expect(state.adaptors).toEqual(expectedSortedAdaptors); - expect(state.isLoading).toBe(false); - expect(state.error).toBeNull(); - expect(state.lastUpdated).toBeGreaterThan(0); - expect(state.adaptors).toHaveLength(mockAdaptorsList.length); - - // Test invalid data handling with fresh store - const store2 = createAdaptorStore(); - mockChannel.push = createMockChannelPushOk({ - adaptors: [invalidAdaptorData.missingName], - }); - store2._connectChannel(mockProvider as any); - await store2.requestAdaptors(); - - state = store2.getSnapshot(); - expect(state.adaptors).toHaveLength(0); - expect(state.isLoading).toBe(false); - expect(state.error).toContain('Invalid adaptors data'); - } - ); + describe('channel connection and adaptors_updated events', () => { + adaptorTest( + 'connectChannel refreshes the catalogue over HTTP so a reconnect picks up new adaptors', + async ({ store, mockProvider }) => { + getAdaptorCatalogueMock.mockResolvedValue({ data: mockAdaptorsList }); - adaptorTest( - 'handles error response and no connection', - async ({ store, mockChannel, mockProvider }) => { - // Test error response - mockChannel.push = createMockChannelPushError( - 'Server error', - 'server_error' - ); - store._connectChannel(mockProvider as any); - await store.requestAdaptors(); - - let state = store.getSnapshot(); - expect(state.adaptors).toHaveLength(0); - expect(state.error).toContain('Failed to request adaptors'); - expect(state.isLoading).toBe(false); - - // Test no channel connection - const storeWithoutChannel = createAdaptorStore(); - await storeWithoutChannel.requestAdaptors(); - - state = storeWithoutChannel.getSnapshot(); - expect(state.error).toContain('No connection available'); - expect(state.isLoading).toBe(false); - } - ); - }); + const cleanup = store._connectChannel(mockProvider as any); - describe('channel connection and events', () => { - adaptorTest( - 'connects channel, loads adaptors, and processes real-time updates', - async ({ store, mockChannel, mockProvider }) => { - // Setup mock to return adaptors on initial request - mockChannel.push = createMockChannelPushOk({ - adaptors: mockAdaptorsList, - }); - - // Connect to channel - const cleanup = store._connectChannel(mockProvider as any); - - // Wait for initial adaptors to be loaded - await waitForCondition(() => store.getSnapshot().adaptors.length > 0); - - // Verify initial load with sorting - let state = store.getSnapshot(); - const expectedSortedAdaptors = getSortedAdaptors(mockAdaptorsList); - expect(state.adaptors).toEqual(expectedSortedAdaptors); - - // Test real-time updates via adaptors_updated event - const updatedAdaptors = [mockAdaptor]; - const mockChannelWithTest = mockChannel as typeof mockChannel & { - _test: { emit: (event: string, message: unknown) => void }; - }; - mockChannelWithTest._test.emit('adaptors_updated', updatedAdaptors); - - // Wait for the update to be processed - await waitForCondition( - () => store.getSnapshot().adaptors.length === 1 - ); - - state = store.getSnapshot(); - const expectedUpdatedAdaptors = getSortedAdaptors(updatedAdaptors); - expect(state.adaptors).toEqual(expectedUpdatedAdaptors); - - // Cleanup - cleanup(); - } - ); - }); + await waitForCondition(() => store.getSnapshot().adaptors.length > 0); + + expect(getAdaptorCatalogueMock).toHaveBeenCalledTimes(1); + expect(store.getSnapshot().adaptors).toEqual( + getSortedAdaptors(mockAdaptorsList) + ); + cleanup(); + } + ); + + adaptorTest( + 'an adaptors_updated push for a brand-new adaptor re-fetches over HTTP and adds it to the catalogue', + async ({ store, mockChannel, mockProvider }) => { + getAdaptorCatalogueMock.mockResolvedValueOnce({ + data: mockAdaptorsList, + }); + const cleanup = store._connectChannel(mockProvider as any); + + await waitForCondition(() => store.getSnapshot().adaptors.length > 0); + expect(store.getSnapshot().adaptors).toEqual( + getSortedAdaptors(mockAdaptorsList) + ); - describe('error handling', () => { - test('handles invalid channel provider', async () => { - const store = createAdaptorStore(); + const catalogueWithGmail = [...mockAdaptorsList, mockAdaptorGmail]; + getAdaptorCatalogueMock.mockResolvedValueOnce({ + data: catalogueWithGmail, + }); - // Test with null provider - expect(() => store._connectChannel(null as any)).toThrow(TypeError); + mockChannel._test.emit('adaptors_updated', { + names: ['@openfn/language-gmail'], + }); - // Test with undefined provider - expect(() => store._connectChannel(undefined as any)).toThrow( - TypeError + await waitForCondition(() => + store + .getSnapshot() + .adaptors.some(a => a.name === '@openfn/language-gmail') ); - }); + + expect(getAdaptorCatalogueMock).toHaveBeenCalledTimes(2); + expect(store.getSnapshot().adaptors).toEqual( + getSortedAdaptors(catalogueWithGmail) + ); + expect(store.getSnapshot().error).toBeNull(); + cleanup(); + } + ); + + adaptorTest( + 'an adaptors_updated push for an already-shown adaptor re-fetches and updates its version list', + async ({ store, mockChannel, mockProvider }) => { + getAdaptorCatalogueMock.mockResolvedValueOnce({ + data: mockAdaptorsList, + }); + const cleanup = store._connectChannel(mockProvider as any); + + await waitForCondition(() => store.getSnapshot().adaptors.length > 0); + + const bumpedHttp = { + ...mockAdaptor, + versions: ['2.2.0', ...mockAdaptor.versions], + latest_version: '2.2.0', + }; + const catalogueWithBump = mockAdaptorsList.map(a => + a.name === mockAdaptor.name ? bumpedHttp : a + ); + getAdaptorCatalogueMock.mockResolvedValueOnce({ + data: catalogueWithBump, + }); + + mockChannel._test.emit('adaptors_updated', { + names: ['@openfn/language-http'], + }); + + await waitForCondition( + () => + store.findAdaptorByName('@openfn/language-http')?.latest_version !== + '2.1.0' + ); + + expect(getAdaptorCatalogueMock).toHaveBeenCalledTimes(2); + expect(store.getSnapshot().adaptors).toEqual( + getSortedAdaptors(catalogueWithBump) + ); + expect(store.getSnapshot().error).toBeNull(); + cleanup(); + } + ); + + test('throws when connecting with a null/undefined provider', () => { + const store = createAdaptorStore(); + expect(() => store._connectChannel(null as any)).toThrow(TypeError); + expect(() => store._connectChannel(undefined as any)).toThrow(TypeError); }); }); describe('query helpers', () => { - test('findAdaptorByName returns correct adaptor', () => { + test('findAdaptorByName/getLatestVersion/getVersions look up by name', () => { const store = createAdaptorStore(); store.setAdaptors(mockAdaptorsList); - const foundAdaptor = store.findAdaptorByName('@openfn/language-http'); - expect(foundAdaptor).toEqual(mockAdaptor); + expect(store.findAdaptorByName('@openfn/language-http')).toEqual( + mockAdaptor + ); + expect( + store.findAdaptorByName('@openfn/language-nonexistent') + ).toBeNull(); + + expect(store.getLatestVersion('@openfn/language-http')).toBe('2.1.0'); + expect(store.getLatestVersion('@openfn/language-nonexistent')).toBeNull(); - const notFound = store.findAdaptorByName('@openfn/language-nonexistent'); - expect(notFound).toBeNull(); + expect(store.getVersions('@openfn/language-http')).toEqual( + mockAdaptor.versions + ); + expect(store.getVersions('@openfn/language-nonexistent')).toHaveLength(0); }); + }); - test('getLatestVersion returns correct version', () => { + describe('handleAdaptorsReceived merge-by-name', () => { + test('preserves referential identity across identical loads', async () => { + getAdaptorCatalogueMock.mockResolvedValue({ data: mockAdaptorsList }); const store = createAdaptorStore(); - store.setAdaptors(mockAdaptorsList); + await store.requestAdaptors(); + + const selectAdaptors = store.withSelector(state => state.adaptors); + const firstRef = selectAdaptors(); - const latestVersion = store.getLatestVersion('@openfn/language-http'); - expect(latestVersion).toBe('2.1.0'); + await store.requestAdaptors(); - const notFound = store.getLatestVersion('@openfn/language-nonexistent'); - expect(notFound).toBeNull(); + const secondRef = selectAdaptors(); + expect(secondRef).toBe(firstRef); + secondRef.forEach((adaptor, i) => { + expect(adaptor).toBe(firstRef[i]); + }); }); - test('getVersions returns correct versions array', () => { + test('replaces only the changed entry when one adaptor mutates', async () => { + getAdaptorCatalogueMock.mockResolvedValue({ data: mockAdaptorsList }); const store = createAdaptorStore(); - store.setAdaptors(mockAdaptorsList); + await store.requestAdaptors(); - const versions = store.getVersions('@openfn/language-http'); - expect(versions).toEqual(mockAdaptor.versions); + const beforeByName = new Map( + store.getSnapshot().adaptors.map(a => [a.name, a]) + ); - const notFound = store.getVersions('@openfn/language-nonexistent'); - expect(notFound).toHaveLength(0); + const target = mockAdaptorsList[0]!; + const mutated = mockAdaptorsList.map(a => + a.name === target.name + ? { + ...a, + versions: ['99.0.0', ...a.versions], + latest_version: '99.0.0', + } + : a + ); + getAdaptorCatalogueMock.mockResolvedValue({ data: mutated }); + await store.requestAdaptors(); + + for (const adaptor of store.getSnapshot().adaptors) { + if (adaptor.name === target.name) { + expect(adaptor).not.toBe(beforeByName.get(adaptor.name)); + } else { + expect(adaptor).toBe(beforeByName.get(adaptor.name)); + } + } }); }); }); diff --git a/assets/test/collaborative-editor/fixtures/adaptorData.ts b/assets/test/collaborative-editor/fixtures/adaptorData.ts index e12adb3e1e5..e086ba67a8a 100644 --- a/assets/test/collaborative-editor/fixtures/adaptorData.ts +++ b/assets/test/collaborative-editor/fixtures/adaptorData.ts @@ -9,18 +9,17 @@ import { sortAdaptors } from '#/collaborative-editor/stores/createAdaptorStore'; import type { Adaptor, - AdaptorVersion, AdaptorsList, } from '../../../js/collaborative-editor/types/adaptor'; /** * Sample adaptor versions for testing */ -export const mockAdaptorVersions: AdaptorVersion[] = [ - { version: '2.1.0' }, - { version: '2.0.5' }, - { version: '2.0.0' }, - { version: '1.9.5' }, +export const mockAdaptorVersions: string[] = [ + '2.1.0', + '2.0.5', + '2.0.0', + '1.9.5', ]; /** @@ -29,8 +28,9 @@ export const mockAdaptorVersions: AdaptorVersion[] = [ export const mockAdaptor: Adaptor = { name: '@openfn/language-http', versions: mockAdaptorVersions, - repo: 'https://github.com/OpenFn/adaptors/tree/main/packages/http', - latest: '2.1.0', + repository: 'https://github.com/OpenFn/adaptors/tree/main/packages/http', + latest_version: '2.1.0', + icon_urls: { square: null, rectangle: null }, }; /** @@ -38,39 +38,35 @@ export const mockAdaptor: Adaptor = { */ export const mockAdaptorDhis2: Adaptor = { name: '@openfn/language-dhis2', - versions: [{ version: '4.2.1' }, { version: '4.2.0' }, { version: '4.1.3' }], - repo: 'https://github.com/OpenFn/adaptors/tree/main/packages/dhis2', - latest: '4.2.1', + versions: ['4.2.1', '4.2.0', '4.1.3'], + repository: 'https://github.com/OpenFn/adaptors/tree/main/packages/dhis2', + latest_version: '4.2.1', + icon_urls: { square: null, rectangle: null }, }; export const mockAdaptorSalesforce: Adaptor = { name: '@openfn/language-salesforce', - versions: [ - { version: '3.5.2' }, - { version: '3.5.1' }, - { version: '3.5.0' }, - { version: '3.4.9' }, - ], - repo: 'https://github.com/OpenFn/adaptors/tree/main/packages/salesforce', - latest: '3.5.2', + versions: ['3.5.2', '3.5.1', '3.5.0', '3.4.9'], + repository: + 'https://github.com/OpenFn/adaptors/tree/main/packages/salesforce', + latest_version: '3.5.2', + icon_urls: { square: null, rectangle: null }, }; export const mockAdaptorGmail: Adaptor = { name: '@openfn/language-gmail', - versions: [{ version: '1.2.0' }, { version: '1.1.0' }, { version: '1.0.0' }], - repo: 'https://github.com/OpenFn/adaptors/tree/main/packages/gmail', - latest: '1.2.0', + versions: ['1.2.0', '1.1.0', '1.0.0'], + repository: 'https://github.com/OpenFn/adaptors/tree/main/packages/gmail', + latest_version: '1.2.0', + icon_urls: { square: null, rectangle: null }, }; export const mockAdaptorCommon: Adaptor = { name: '@openfn/language-common', - versions: [ - { version: '2.0.0' }, - { version: '1.15.0' }, - { version: '1.14.0' }, - ], - repo: 'https://github.com/OpenFn/adaptors/tree/main/packages/common', - latest: '2.0.0', + versions: ['2.0.0', '1.15.0', '1.14.0'], + repository: 'https://github.com/OpenFn/adaptors/tree/main/packages/common', + latest_version: '2.0.0', + icon_urls: { square: null, rectangle: null }, }; /** @@ -94,32 +90,29 @@ export const invalidAdaptorData = { missingName: { // name missing versions: mockAdaptorVersions, - repo: 'https://github.com/test', - latest: '1.0.0', + repository: 'https://github.com/test', + latest_version: '1.0.0', }, invalidVersions: { name: '@openfn/language-test', versions: 'invalid', // should be array - repo: 'https://github.com/test', - latest: '1.0.0', + repository: 'https://github.com/test', + latest_version: '1.0.0', }, missingLatest: { name: '@openfn/language-test', versions: mockAdaptorVersions, - repo: 'https://github.com/test', - // latest missing + repository: 'https://github.com/test', + // latest_version missing }, invalidVersionStructure: { name: '@openfn/language-test', - versions: [ - { version: '1.0.0' }, - { invalidField: 'invalid' }, // wrong structure - ], - repo: 'https://github.com/test', - latest: '1.0.0', + versions: ['1.0.0', { invalidField: 'invalid' }], // wrong structure + repository: 'https://github.com/test', + latest_version: '1.0.0', }, }; @@ -136,13 +129,12 @@ export function createMockAdaptor(overrides: Partial = {}): Adaptor { /** * Helper to create adaptors list with specific number of items */ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ export function createMockAdaptorsList(count: number): AdaptorsList { return Array.from({ length: count }, (_, i) => ({ name: `@openfn/language-test-${i}`, - versions: [{ version: `${i}.1.0` }, { version: `${i}.0.0` }], - repo: `https://github.com/test/adaptor-${i}`, - latest: `${i}.1.0`, + versions: [`${i}.1.0`, `${i}.0.0`], + repository: `https://github.com/test/adaptor-${i}`, + latest_version: `${i}.1.0`, + icon_urls: { square: null, rectangle: null }, })); } -/* eslint-enable @typescript-eslint/restrict-template-expressions */ diff --git a/assets/test/collaborative-editor/useAdaptors.test.tsx b/assets/test/collaborative-editor/useAdaptors.test.tsx index 8727805a33c..6a66b56a0a1 100644 --- a/assets/test/collaborative-editor/useAdaptors.test.tsx +++ b/assets/test/collaborative-editor/useAdaptors.test.tsx @@ -14,8 +14,8 @@ import { useAdaptorCommands, useAdaptors, useAdaptorsError, + useAdaptorsInUse, useAdaptorsLoading, - useProjectAdaptors, } from '../../js/collaborative-editor/hooks/useAdaptors'; import { createSessionStore } from '../../js/collaborative-editor/stores/createSessionStore'; @@ -291,7 +291,7 @@ describe('useAdaptors hooks', () => { await waitFor(() => { expect(httpAdaptor.current).not.toBe(null); expect(httpAdaptor.current?.name).toBe('@openfn/language-http'); - expect(httpAdaptor.current?.latest).toBe('2.1.0'); + expect(httpAdaptor.current?.latest_version).toBe('2.1.0'); }); expect(nonExistent.current).toBe(null); @@ -376,162 +376,128 @@ describe('useAdaptors hooks', () => { }); }); - describe('useProjectAdaptors', () => { + describe('useAdaptorsInUse', () => { test('requires StoreProvider context', () => { - expect(() => renderHook(() => useProjectAdaptors())).toThrow( - 'useProjectAdaptors must be used within a StoreProvider' + expect(() => renderHook(() => useAdaptorsInUse())).toThrow( + 'useAdaptorsInUse must be used within a StoreProvider' ); }); - test('returns projectAdaptors, allAdaptors, and isLoading', async () => { + test('derives adaptors in use from Y.Doc jobs', async () => { const { wrapper, stores } = createWrapper(); - const { result } = renderHook(() => useProjectAdaptors(), { wrapper }); + const { result } = renderHook(() => useAdaptorsInUse(), { wrapper }); - // Initially empty - expect(result.current.projectAdaptors).toEqual([]); - expect(result.current.allAdaptors).toEqual([]); - expect(result.current.isLoading).toBe(false); - - // Set all adaptors (simulates backend response) - const allAdaptors = [mockAdaptor, mockAdaptorGmail, mockAdaptorCommon]; - act(() => { - stores.adaptorStore.setAdaptors(allAdaptors); - }); - - await waitFor(() => { - expect(result.current.allAdaptors).toHaveLength(3); - }); - }); - - test('merges Y.Doc job adaptors into projectAdaptors', async () => { - const { wrapper, stores } = createWrapper(); - const { result } = renderHook(() => useProjectAdaptors(), { wrapper }); - - // Set all available adaptors const allAdaptors = [mockAdaptor, mockAdaptorGmail, mockAdaptorCommon]; act(() => { stores.adaptorStore.setAdaptors(allAdaptors); }); - // Add a job to the workflow store that uses Gmail adaptor act(() => { stores.workflowStore._setJobsForTesting([ { id: 'job-1', - name: 'Test Job', - adaptor: '@openfn/language-gmail@latest', - body: 'fn(state => state)', + name: 'HTTP Job', + adaptor: '@openfn/language-http@2.1.0', + body: '', + }, + { + id: 'job-2', + name: 'Common Job', + adaptor: '@openfn/language-common@2.0.0', + body: '', }, ]); }); - // projectAdaptors should now include Gmail from Y.Doc await waitFor(() => { - const projectAdaptorNames = result.current.projectAdaptors.map( - a => a.name - ); - expect(projectAdaptorNames).toContain('@openfn/language-gmail'); + expect(result.current.adaptorsInUse).toHaveLength(2); }); + + const names = result.current.adaptorsInUse.map(a => a.name); + expect(names).toEqual([ + '@openfn/language-common', + '@openfn/language-http', + ]); + + // adaptorsInUse entries are the same object references as the catalogue entries, not copies. + const catalogue = result.current.allAdaptors; + for (const a of result.current.adaptorsInUse) { + const fromCatalogue = catalogue.find(c => c.name === a.name); + expect(a).toBe(fromCatalogue); + } }); - test('does not duplicate adaptors already in backend projectAdaptors', async () => { + test('returns empty list when workflow has no jobs', async () => { const { wrapper, stores } = createWrapper(); - const { result } = renderHook(() => useProjectAdaptors(), { wrapper }); + const { result } = renderHook(() => useAdaptorsInUse(), { wrapper }); - // Set all available adaptors and project adaptors (simulates backend where HTTP is already saved) const allAdaptors = [mockAdaptor, mockAdaptorGmail, mockAdaptorCommon]; act(() => { stores.adaptorStore.setAdaptors(allAdaptors); - // Directly set projectAdaptors via internal state modification - stores.adaptorStore._setProjectAdaptors([mockAdaptor]); - }); - - // Add a job that uses HTTP (already in projectAdaptors from backend) - act(() => { - stores.workflowStore._setJobsForTesting([ - { - id: 'job-1', - name: 'Test Job', - adaptor: '@openfn/language-http@2.1.0', - body: 'fn(state => state)', - }, - ]); }); await waitFor(() => { - // Should only have HTTP once, not duplicated - const httpCount = result.current.projectAdaptors.filter( - a => a.name === '@openfn/language-http' - ).length; - expect(httpCount).toBe(1); + expect(result.current.allAdaptors).toHaveLength(3); }); + + expect(result.current.adaptorsInUse).toEqual([]); }); - test('handles jobs without adaptor field', async () => { + test('ignores jobs referencing adaptors absent from the catalogue', async () => { const { wrapper, stores } = createWrapper(); - const { result } = renderHook(() => useProjectAdaptors(), { wrapper }); + const { result } = renderHook(() => useAdaptorsInUse(), { wrapper }); - // Set all available adaptors - const allAdaptors = [mockAdaptor, mockAdaptorGmail]; act(() => { - stores.adaptorStore.setAdaptors(allAdaptors); + stores.adaptorStore.setAdaptors([mockAdaptor]); }); - // Add a job without an adaptor field (should not crash) act(() => { stores.workflowStore._setJobsForTesting([ { id: 'job-1', - name: 'Test Job', - adaptor: undefined as any, - body: 'fn(state => state)', + name: 'Unknown', + adaptor: '@openfn/language-unknown@1.0.0', + body: '', + }, + { + id: 'job-2', + name: 'HTTP', + adaptor: '@openfn/language-http@2.0.0', + body: '', }, ]); }); - // Should not throw and projectAdaptors should be empty (no backend project adaptors set) await waitFor(() => { - expect(result.current.projectAdaptors).toEqual([]); + expect(result.current.adaptorsInUse).toHaveLength(1); }); + + expect(result.current.adaptorsInUse[0]?.name).toBe( + '@openfn/language-http' + ); }); - test('sorts merged projectAdaptors alphabetically', async () => { + test('matches version-suffixed adaptor specs against catalogue package names', async () => { const { wrapper, stores } = createWrapper(); - const { result } = renderHook(() => useProjectAdaptors(), { wrapper }); + const { result } = renderHook(() => useAdaptorsInUse(), { wrapper }); - // Set all available adaptors - const allAdaptors = [mockAdaptor, mockAdaptorGmail, mockAdaptorCommon]; act(() => { - stores.adaptorStore.setAdaptors(allAdaptors); - // Set HTTP as already in project from backend - stores.adaptorStore._setProjectAdaptors([mockAdaptor]); + stores.adaptorStore.setAdaptors([mockAdaptor]); }); - // Add jobs using Gmail and Common (not in backend projectAdaptors) act(() => { stores.workflowStore._setJobsForTesting([ { id: 'job-1', - name: 'Gmail Job', - adaptor: '@openfn/language-gmail@latest', - body: '', - }, - { - id: 'job-2', - name: 'Common Job', - adaptor: '@openfn/language-common@2.0.0', + name: 'HTTP', + adaptor: '@openfn/language-http@2.0.0', body: '', }, ]); }); await waitFor(() => { - expect(result.current.projectAdaptors).toHaveLength(3); - const names = result.current.projectAdaptors.map(a => a.name); - // Should be sorted: common, gmail, http - expect(names).toEqual([ - '@openfn/language-common', - '@openfn/language-gmail', + expect(result.current.adaptorsInUse.map(a => a.name)).toEqual([ '@openfn/language-http', ]); }); diff --git a/assets/test/workflow-diagram/components/MiniMapNode.test.tsx b/assets/test/workflow-diagram/components/MiniMapNode.test.tsx new file mode 100644 index 00000000000..8aa20964a22 --- /dev/null +++ b/assets/test/workflow-diagram/components/MiniMapNode.test.tsx @@ -0,0 +1,120 @@ +/** + * MiniMapNode Component Tests + * + * Verifies that the minimap renders job icons sourced from the AdaptorStore + * via useAdaptorIconUrl, with the rect-only placeholder fallback when the + * URL is null. Trigger rendering is unaffected. + */ + +import { render } from '@testing-library/react'; +import type { MiniMapNodeProps } from '@xyflow/react'; +import { describe, expect, test } from 'vitest'; + +import { + StoreContext, + type StoreContextValue, +} from '../../../js/collaborative-editor/contexts/StoreProvider'; +import { createAdaptorStore } from '../../../js/collaborative-editor/stores/createAdaptorStore'; +import type { Adaptor } from '../../../js/collaborative-editor/types/adaptor'; +import MiniMapNode from '../../../js/workflow-diagram/components/MiniMapNode'; + +type Job = { id: string; adaptor?: string }; +type Trigger = { id: string; type: 'webhook' | 'cron' | 'kafka' }; + +function renderInSvg( + nodeProps: MiniMapNodeProps, + jobs: Job[], + triggers: Trigger[], + adaptors: Adaptor[] +) { + const adaptorStore = createAdaptorStore(); + adaptorStore.setAdaptors(adaptors); + + const stores = { + adaptorStore, + credentialStore: {} as StoreContextValue['credentialStore'], + metadataStore: {} as StoreContextValue['metadataStore'], + awarenessStore: {} as StoreContextValue['awarenessStore'], + workflowStore: {} as StoreContextValue['workflowStore'], + sessionContextStore: {} as StoreContextValue['sessionContextStore'], + historyStore: {} as StoreContextValue['historyStore'], + uiStore: {} as StoreContextValue['uiStore'], + editorPreferencesStore: {} as StoreContextValue['editorPreferencesStore'], + aiAssistantStore: {} as StoreContextValue['aiAssistantStore'], + } satisfies StoreContextValue; + + return render( + + + + + + ); +} + +describe('MiniMapNode - job icon', () => { + const baseNodeProps: MiniMapNodeProps = { + x: 0, + y: 0, + width: 120, + height: 120, + selected: false, + borderRadius: 0, + className: '', + shapeRendering: 'auto', + }; + + test('renders an when the adaptor is seeded with icon_urls.square', () => { + const url = '/adaptor-icons/http/square-1.png'; + const { container } = renderInSvg( + { ...baseNodeProps, id: 'job-1' }, + [{ id: 'job-1', adaptor: '@openfn/language-http@1.0.0' }], + [], + [ + { + name: '@openfn/language-http', + versions: ['1.0.0'], + repository: 'https://example.com', + latest_version: '1.0.0', + icon_urls: { square: url, rectangle: null }, + }, + ] + ); + + const image = container.querySelector('image'); + expect(image).not.toBeNull(); + expect(image!.getAttribute('href')).toBe(url); + }); + + test('renders no when icon_urls.square is null', () => { + const { container } = renderInSvg( + { ...baseNodeProps, id: 'job-1' }, + [{ id: 'job-1', adaptor: '@openfn/language-http@1.0.0' }], + [], + [ + { + name: '@openfn/language-http', + versions: ['1.0.0'], + repository: 'https://example.com', + latest_version: '1.0.0', + icon_urls: { square: null, rectangle: null }, + }, + ] + ); + + expect(container.querySelector('image')).toBeNull(); + expect(container.querySelector('rect')).not.toBeNull(); + }); + + test('webhook trigger still renders the GlobeAltIcon (regression guard)', () => { + const { container } = renderInSvg( + { ...baseNodeProps, id: 'trigger-1' }, + [], + [{ id: 'trigger-1', type: 'webhook' }], + [] + ); + + // GlobeAltIcon renders an svg inside a foreignObject for triggers + expect(container.querySelector('foreignObject svg')).not.toBeNull(); + }); +}); diff --git a/assets/test/workflow-diagram/nodes/Job.test.tsx b/assets/test/workflow-diagram/nodes/Job.test.tsx new file mode 100644 index 00000000000..4c853a382d5 --- /dev/null +++ b/assets/test/workflow-diagram/nodes/Job.test.tsx @@ -0,0 +1,95 @@ +/** + * Job Node Component Tests + * + * Verifies that job nodes read their adaptor icons from the AdaptorStore + * via useAdaptorIconUrl, with graceful string-label fallback when the URL + * is null OR no StoreProvider is mounted (LiveView workflow-editor path). + */ + +import { render } from '@testing-library/react'; +import { ReactFlowProvider } from '@xyflow/react'; +import { describe, expect, test } from 'vitest'; + +import { + StoreContext, + type StoreContextValue, +} from '../../../js/collaborative-editor/contexts/StoreProvider'; +import { createAdaptorStore } from '../../../js/collaborative-editor/stores/createAdaptorStore'; +import type { Adaptor } from '../../../js/collaborative-editor/types/adaptor'; +import JobNode from '../../../js/workflow-diagram/nodes/Job'; + +function renderJob(adaptor: string, adaptors: Adaptor[] | null) { + const data = { name: 'My Job', adaptor }; + + const tree = ( + + + + ); + + if (adaptors === null) { + return render(tree); + } + + const adaptorStore = createAdaptorStore(); + adaptorStore.setAdaptors(adaptors); + + const stores = { + adaptorStore, + credentialStore: {} as StoreContextValue['credentialStore'], + metadataStore: {} as StoreContextValue['metadataStore'], + awarenessStore: {} as StoreContextValue['awarenessStore'], + workflowStore: {} as StoreContextValue['workflowStore'], + sessionContextStore: {} as StoreContextValue['sessionContextStore'], + historyStore: {} as StoreContextValue['historyStore'], + uiStore: {} as StoreContextValue['uiStore'], + editorPreferencesStore: {} as StoreContextValue['editorPreferencesStore'], + aiAssistantStore: {} as StoreContextValue['aiAssistantStore'], + } satisfies StoreContextValue; + + return render( + {tree} + ); +} + +describe('JobNode - adaptor icon', () => { + test('renders an with icon_urls.square when the adaptor is seeded', () => { + const url = '/adaptor-icons/http/square-deadbeef.png'; + const { container } = renderJob('@openfn/language-http@1.0.0', [ + { + name: '@openfn/language-http', + versions: ['1.0.0'], + repository: 'https://example.com', + latest_version: '1.0.0', + icon_urls: { square: url, rectangle: null }, + }, + ]); + + const img = container.querySelector('img'); + expect(img).not.toBeNull(); + expect(img?.getAttribute('src')).toContain(url); + expect(img?.getAttribute('alt')).toBe('http'); + }); + + test('falls back to the adaptor string label when icon_urls.square is null', () => { + const { container } = renderJob('@openfn/language-http@1.0.0', [ + { + name: '@openfn/language-http', + versions: ['1.0.0'], + repository: 'https://example.com', + latest_version: '1.0.0', + icon_urls: { square: null, rectangle: '/rect.png' }, + }, + ]); + + expect(container.querySelector('img')).toBeNull(); + expect(container.textContent).toContain('http'); + }); + + test('does not throw and falls back to label when no StoreProvider is mounted', () => { + const { container } = renderJob('@openfn/language-http@1.0.0', null); + + expect(container.querySelector('img')).toBeNull(); + expect(container.textContent).toContain('http'); + }); +}); diff --git a/bin/adaptor_cache b/bin/adaptor_cache new file mode 100755 index 00000000000..e6aec611332 --- /dev/null +++ b/bin/adaptor_cache @@ -0,0 +1,233 @@ +#!/usr/bin/env bash + +# ============================================================================= +# Lightning Adaptor Cache +# ============================================================================= +# +# A local caching reverse proxy in front of the three upstreams the +# Lightning.Adaptors.* subsystem reads from: registry.npmjs.org, +# cdn.jsdelivr.net and raw.githubusercontent.com. See +# tooling/adaptor_cache/README.md. +# +# A forward proxy would not work: Lightning talks to these upstreams through +# Tesla over the Finch adapter, and neither honours HTTP_PROXY/HTTPS_PROXY. +# Each upstream does have a configurable base URL, which is what this reverse +# proxy plugs into. +# +# ============================================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +COMPOSE_DIR="${PROJECT_ROOT}/tooling/adaptor_cache" +COMPOSE_FILE="${COMPOSE_DIR}/docker-compose.yml" + +# Fixed project name, not derived from the directory, so every git worktree +# shares one cache instead of each spinning up its own. +PROJECT_NAME="adaptor-cache" + +PORT="${ADAPTOR_CACHE_PORT:-4874}" +BASE_URL="http://localhost:${PORT}" + +compose() { + ADAPTOR_CACHE_PORT="${PORT}" \ + docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" "$@" +} + +show_help() { + cat <<'EOF' +Lightning Adaptor Cache + +A local caching reverse proxy in front of the three upstreams the +Lightning.Adaptors.* subsystem reads from: registry.npmjs.org, +cdn.jsdelivr.net and raw.githubusercontent.com. See +tooling/adaptor_cache/README.md. + +Usage: bin/adaptor_cache + + up Start the proxy and print the export lines + down Stop the proxy, keeping the cache on disk + status Show container state and reachability + purge Stop the proxy AND drop the cache volume + logs Tail the access log (cache=HIT / cache=MISS) + check Probe all three prefixes, prove MISS then HIT + --help This message + +Environment variables: + + ADAPTOR_CACHE_PORT Host port to bind (default: 4874) +EOF +} + +require_docker() { + if ! command -v docker >/dev/null 2>&1; then + echo "adaptor_cache: docker not found on PATH." >&2 + exit 1 + fi + + if ! docker compose version >/dev/null 2>&1; then + echo "adaptor_cache: 'docker compose' is not available." >&2 + exit 1 + fi + + if [[ ! -f "${COMPOSE_FILE}" ]]; then + echo "adaptor_cache: missing ${COMPOSE_FILE}" >&2 + exit 1 + fi +} + +wait_for_healthz() { + local _attempt + for _attempt in $(seq 1 30); do + if curl -fsS -o /dev/null --max-time 2 "${BASE_URL}/_healthz"; then + return 0 + fi + sleep 1 + done + + echo "adaptor_cache: proxy did not come up on ${BASE_URL} after 30s." >&2 + echo "adaptor_cache: run 'bin/adaptor_cache logs' to see why." >&2 + return 1 +} + +print_exports() { + cat <&1)"; then + printf ' %-9s FAIL could not reach %s\n' "${label}" "${url}" + printf ' %s\n' "${headers}" + return 1 + fi + + status="$(printf '%s' "${headers}" | tr -d '\r' | awk 'NR==1 {print $2}')" + cache_1="$(printf '%s' "${headers}" | tr -d '\r' \ + | awk -F': ' 'tolower($1)=="x-cache-status" {print $2}' | tail -1)" + + headers="$(curl -sS -o /dev/null -D - --max-time 30 "${url}")" + cache_2="$(printf '%s' "${headers}" | tr -d '\r' \ + | awk -F': ' 'tolower($1)=="x-cache-status" {print $2}' | tail -1)" + + printf ' %-9s %-4s first=%-12s second=%-12s %s\n' \ + "${label}" "${status:-???}" "${cache_1:--}" "${cache_2:--}" "${url}" + + if [[ "${cache_2}" != "HIT" ]]; then + printf ' ^ expected the second request to be a HIT\n' + return 1 + fi + + return 0 +} + +cmd_up() { + require_docker + compose up -d + wait_for_healthz + echo "adaptor_cache: up on ${BASE_URL}" + print_exports +} + +cmd_down() { + require_docker + compose down + echo "adaptor_cache: down. The cache volume is intact; 'purge' drops it." +} + +cmd_purge() { + require_docker + compose down -v + echo "adaptor_cache: down and cache volume removed." +} + +cmd_status() { + require_docker + compose ps + echo + if curl -fsS -o /dev/null --max-time 2 "${BASE_URL}/_healthz"; then + echo "adaptor_cache: reachable at ${BASE_URL}" + else + echo "adaptor_cache: NOT reachable at ${BASE_URL}" + fi +} + +cmd_logs() { + require_docker + if [[ $# -gt 0 ]]; then + compose logs "$@" + else + compose logs -f --tail 100 nginx + fi +} + +cmd_check() { + require_docker + + if ! curl -fsS -o /dev/null --max-time 2 "${BASE_URL}/_healthz"; then + echo "adaptor_cache: not running. Start it with 'bin/adaptor_cache up'." >&2 + exit 1 + fi + + echo "adaptor_cache: probing all three prefixes twice each" + echo + + local failed=0 + + probe npm "${BASE_URL}/npm/-/v1/search?text=@openfn&size=1" || failed=1 + probe jsdelivr \ + "${BASE_URL}/jsdelivr/npm/@openfn/language-common/package.json" || failed=1 + probe github \ + "${BASE_URL}/github/OpenFn/adaptors/main/packages/common/assets/square.png" \ + || failed=1 + + echo + if [[ "${failed}" -ne 0 ]]; then + echo "adaptor_cache: check FAILED" >&2 + exit 1 + fi + + echo "adaptor_cache: check passed — all three prefixes cache." +} + +main() { + local command="${1:-help}" + [[ $# -gt 0 ]] && shift || true + + case "${command}" in + up) cmd_up "$@" ;; + down) cmd_down "$@" ;; + purge) cmd_purge "$@" ;; + status) cmd_status "$@" ;; + logs) cmd_logs "$@" ;; + check) cmd_check "$@" ;; + help | --help | -h) show_help ;; + *) + echo "adaptor_cache: unknown command '${command}'" >&2 + echo >&2 + show_help >&2 + exit 1 + ;; + esac +} + +main "$@" diff --git a/config/dev.exs b/config/dev.exs index 3fd698f53dd..4d55304f67c 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -147,18 +147,6 @@ config :philter, allowed_hosts: ["localhost"] config :lightning, Lightning.AuthProviders.OauthHTTPClient.PinnedAdapter, allowed_hosts: ["localhost"] -# Lightning.Adaptors.NPM upstream URLs — explicit override for clarity in dev. -# Each key is read by a single sub-module: -# * registry_url → NPM.Registry (npm search + packument) -# * jsdelivr_url → NPM.Schema (configuration-schema.json fetch) -# * github_url → NPM.GitHub (raw icon GETs) -# * github_ref → NPM.GitHub (git ref under OpenFn/adaptors) -config :lightning, Lightning.Adaptors.NPM, - registry_url: "https://registry.npmjs.org", - github_url: "https://raw.githubusercontent.com", - github_ref: "main", - jsdelivr_url: "https://cdn.jsdelivr.net" - config :git_hooks, # In local dev (with a real .git repo) we auto-install hooks. # In Docker builds the .git directory is not present (or incomplete), diff --git a/config/test.exs b/config/test.exs index 301e472d4b5..6a05ec32858 100644 --- a/config/test.exs +++ b/config/test.exs @@ -101,7 +101,7 @@ config :lightning, Lightning.Mailer, adapter: Swoosh.Adapters.Test config :lightning, Lightning.AdaptorRegistry, use_cache: "test/fixtures/adaptor_registry_cache.json" -# Phase A Adaptors.Supervisor config for test boot. +# Adaptors.Supervisor config for test boot. # # - `:strategy` — the production `Lightning.Adaptors.Supervisor` mounted in # `application.ex` would default to `Lightning.Adaptors.NPM` and try to diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index fd69d2eb9bb..959b7e1cbb0 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -6,11 +6,12 @@ defmodule Lightning.Adaptors do `Lightning.Adaptors.Scheduler`, and version resolution to `Lightning.Adaptors.Repo`. No logic lives here. - All functions come in a dual-arity shape: the zero-/single-arg form + Most functions come in a dual-arity shape: the zero-/single-arg form passes the compile-time default supervisor name `@sup`; the extra-arity form accepts an explicit supervisor name for test isolation. - `resolve_version/2` is the single exception — it has no sup arity because - it reads the global Repo directly. + `resolve_version/2`, `catalogue/0`, and `catalogue_stamp/0` are + exceptions — they read the global Repo directly, not a running + supervisor process, so there is nothing to swap for test isolation. """ alias Lightning.Adaptors.Config @@ -51,6 +52,20 @@ defmodule Lightning.Adaptors do {:ok, Path.t()} | {:error, term()} def icon(sup, pkg, shape), do: Store.icon(sup, pkg, shape) + @doc """ + Full catalogue for the active source: every adaptor's `name`, + `latest_version`, `repository`, icon fields, and full version list. + Reads `Repo` directly, like `resolve_version/2`. + """ + @spec catalogue() :: [Repo.catalogue_entry()] + def catalogue, do: Repo.catalogue(AdaptorsSupervisor.source(@sup)) + + @doc """ + ETag basis for `catalogue/0` — see `Repo.catalogue_stamp/1`. + """ + @spec catalogue_stamp() :: {DateTime.t() | nil, non_neg_integer()} + def catalogue_stamp, do: Repo.catalogue_stamp(AdaptorsSupervisor.source(@sup)) + @spec resolve_version(String.t(), String.t()) :: {:ok, String.t()} | {:error, :not_found} def resolve_version(name, requested) when requested in ["latest", "local"] do diff --git a/lib/lightning/adaptors/channel_broadcaster.ex b/lib/lightning/adaptors/channel_broadcaster.ex index 85b598374c0..09dbf14f995 100644 --- a/lib/lightning/adaptors/channel_broadcaster.ex +++ b/lib/lightning/adaptors/channel_broadcaster.ex @@ -3,17 +3,15 @@ defmodule Lightning.Adaptors.ChannelBroadcaster do Burst-coalesced fan-out of adaptor changes to connected sessions. Subscribes to `:source_topic` (the cache-coherence topic shared with - `Lightning.Adaptors.Invalidator`) and republishes a single pre-rendered - envelope to `:client_topic` at most once per 250ms leading-edge window. + `Lightning.Adaptors.Invalidator`) and republishes a single envelope of + changed names to `:client_topic` at most once per 250ms leading-edge + window. Two-topic separation: the source topic is the cache-coherence audience; the client topic is the display-freshness audience (`WorkflowChannel` - subscribers). This bridges them: `Lightning.Adaptors.packages/1` is - rendered once per burst and fanned out by PubSub rather than once per - session (§6.5c). - - No within-callback fan-out in `:flush` — `Phoenix.PubSub.broadcast/3` - is a single call that reaches all subscribers in one hop (§10 #19). + subscribers). This bridges them: the payload tells a session "these + adaptors changed, go refetch" — not what changed about them, so + `:flush` never has to touch the cache or render anything. """ use GenServer @@ -35,9 +33,7 @@ defmodule Lightning.Adaptors.ChannelBroadcaster do Required opts: * `:name` — registered GenServer name. * `:source_topic` — PubSub topic to subscribe to (cache-coherence). - * `:client_topic` — PubSub topic to broadcast the rendered envelope to. - * `:sup` — supervisor instance name; forwarded to - `Lightning.Adaptors.packages/1` for per-instance isolation. + * `:client_topic` — PubSub topic to broadcast the changed names to. """ @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts) do @@ -56,36 +52,33 @@ defmodule Lightning.Adaptors.ChannelBroadcaster do {:ok, %{ client_topic: Keyword.fetch!(opts, :client_topic), - sup: Keyword.fetch!(opts, :sup), - timer: nil + timer: nil, + names: MapSet.new() }} end @impl true # First message of a burst: arm the leading-edge timer. - def handle_info({:changed, _name, _source}, %{timer: nil} = state) do + def handle_info({:changed, name, _source}, %{timer: nil} = state) do timer = Process.send_after(self(), :flush, @debounce_ms) - {:noreply, %{state | timer: timer}} + {:noreply, %{state | timer: timer, names: MapSet.put(state.names, name)}} end - # Subsequent messages within the debounce window: drop on the floor. - def handle_info({:changed, _name, _source}, state) do - {:noreply, state} + # Subsequent messages within the debounce window: accumulate, don't flush. + def handle_info({:changed, name, _source}, state) do + {:noreply, %{state | names: MapSet.put(state.names, name)}} end - def handle_info(:flush, %{client_topic: topic, sup: sup} = state) do - case Lightning.Adaptors.packages(sup) do - {:ok, pkgs} -> - Phoenix.PubSub.broadcast( - Lightning.PubSub, - topic, - %{event: "adaptors_updated", payload: %{adaptors: pkgs}} - ) - - {:error, _} -> - :ok - end - - {:noreply, %{state | timer: nil}} + def handle_info(:flush, %{client_topic: topic, names: names} = state) do + Phoenix.PubSub.broadcast( + Lightning.PubSub, + topic, + %{ + event: "adaptors_updated", + payload: %{names: Enum.sort(names)} + } + ) + + {:noreply, %{state | timer: nil, names: MapSet.new()}} end end diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index d5ed5e44c29..a65adeccbdf 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -3,9 +3,7 @@ defmodule Lightning.Adaptors.NPM do Production implementation of `Lightning.Adaptors.Strategy` that talks to the public NPM registry and the OpenFn adaptors monorepo on GitHub. - Consolidates the legacy `Lightning.AdaptorRegistry`, - `Mix.Tasks.Lightning.InstallSchemas`, and - `Mix.Tasks.Lightning.InstallAdaptorIcons` into one stateless module: + Implements the four `Lightning.Adaptors.Strategy` callbacks: * `c:list_adaptors/0` — single search-API call returning `name + latest_version` for every `@openfn/language-*` package. diff --git a/lib/lightning/adaptors/package_name.ex b/lib/lightning/adaptors/package_name.ex new file mode 100644 index 00000000000..a629fa74c9d --- /dev/null +++ b/lib/lightning/adaptors/package_name.ex @@ -0,0 +1,64 @@ +defmodule Lightning.Adaptors.PackageName do + @moduledoc """ + NPM-style package-name parsing and worker wire-shape recomposition for + the `Lightning.Adaptors.*` subsystem. + + This module is the single source of truth for the legacy + `AdaptorRegistry.resolve_adaptor/1` and `resolve_package_name/1` + contracts, ported to read through the `Lightning.Adaptors` facade. + + `parse/1` splits `"name@version"` strings; `to_wire/1` resolves the + `latest` literal through `Lightning.Adaptors.resolve_version/2`, + preserves `"name@local"` as a literal regardless of source, and emits + `"name@local"` under a `:local` strategy source. + """ + + alias Lightning.Adaptors + alias Lightning.Adaptors.Config + + @package_name_regex ~r/(@?[\/\d\n\w-]+)(?:@([\d\.\w-]+))?$/ + + @spec parse(nil) :: {nil, nil} + def parse(nil), do: {nil, nil} + + @spec parse(String.t()) :: {String.t() | nil, String.t() | nil} + def parse(package_name) when is_binary(package_name) do + case Regex.run(@package_name_regex, package_name) do + [_, name, version] -> {name, version} + [_, _name] -> {package_name, nil} + _ -> {nil, nil} + end + end + + @spec to_wire(String.t() | nil) :: String.t() + def to_wire(adaptor) do + case parse(adaptor) do + {nil, nil} -> "" + {name, version} -> recompose(name, version, adaptor) + end + end + + defp recompose(name, "local", _original), do: "#{name}@local" + + defp recompose(name, version, original) do + case Config.current_source() do + :local -> + "#{name}@local" + + _ -> + case version do + "latest" -> + case Adaptors.resolve_version(name, "latest") do + {:ok, resolved} -> "#{name}@#{resolved}" + {:error, _} -> "#{name}@latest" + end + + nil -> + original + + _concrete -> + original + end + end + end +end diff --git a/lib/lightning/adaptors/repo.ex b/lib/lightning/adaptors/repo.ex index f15f2ad690d..e53a40a8cd5 100644 --- a/lib/lightning/adaptors/repo.ex +++ b/lib/lightning/adaptors/repo.ex @@ -10,9 +10,7 @@ defmodule Lightning.Adaptors.Repo do Every read helper takes the desired `:source` (`:npm | :local`) explicitly; the module itself stays source-agnostic. Callers resolve - the active source via `Lightning.Adaptors.Config.current_source/0` - (see §4.4 source-tagging invariant and §6.4 in - `.context/lightning/adaptors/REWRITE-2026-05.md`). + the active source via `Lightning.Adaptors.Config.current_source/0`. `upsert_adaptor/1` is the only writer the Scheduler uses. It is idempotent, transactional, and diff-aware: `checked_at` advances on @@ -42,6 +40,17 @@ defmodule Lightning.Adaptors.Repo do icon_rectangle_sha256: binary() | nil } + @type catalogue_entry :: %{ + name: String.t(), + latest_version: String.t(), + repository: String.t() | nil, + versions: [String.t()], + icon_square_ext: String.t() | nil, + icon_rectangle_ext: String.t() | nil, + icon_square_sha256: binary() | nil, + icon_rectangle_sha256: binary() | nil + } + @version_row_fields ~w(adaptor_id version integrity tarball_url size_bytes dependencies peer_dependencies published_at deprecated)a @@ -267,6 +276,69 @@ defmodule Lightning.Adaptors.Repo do ) end + @doc """ + Full catalogue projection for a source: every adaptor's `name`, + `latest_version`, `repository`, icon fields, and full version list. + """ + @spec catalogue(source()) :: [catalogue_entry()] + def catalogue(source) do + adaptors = + Lightning.Repo.all( + from a in Adaptor, + where: a.source == ^source, + order_by: [asc: a.name], + select: %{ + name: a.name, + latest_version: a.latest_version, + repository: a.repository, + icon_square_ext: a.icon_square_ext, + icon_rectangle_ext: a.icon_rectangle_ext, + icon_square_sha256: a.icon_square_sha256, + icon_rectangle_sha256: a.icon_rectangle_sha256 + } + ) + + versions_by_name = + Lightning.Repo.all( + from v in AdaptorVersion, + join: a in Adaptor, + on: v.adaptor_id == a.id, + where: a.source == ^source, + order_by: [asc: v.inserted_at, asc: v.version], + select: {a.name, v.version} + ) + |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + + Enum.map(adaptors, fn adaptor -> + Map.put(adaptor, :versions, Map.get(versions_by_name, adaptor.name, [])) + end) + end + + @doc """ + ETag basis for the catalogue: `{timestamp, version_row_count}` for + `source`, where `timestamp` is the later of + `MAX(adaptors.updated_at)` and `MAX(adaptor_versions.inserted_at)`, + or `nil` when the source has no rows. + + `version_row_count` is carried alongside the timestamp because a + removed version doesn't move either max — deleting rows only ever + lowers the count. + """ + @spec catalogue_stamp(source()) :: {DateTime.t() | nil, non_neg_integer()} + def catalogue_stamp(source) do + Lightning.Repo.one( + from a in Adaptor, + left_join: v in AdaptorVersion, + on: v.adaptor_id == a.id, + where: a.source == ^source, + select: + {type( + fragment("GREATEST(?, ?)", max(a.updated_at), max(v.inserted_at)), + :utc_datetime_usec + ), count(v.id)} + ) + end + defp upsert_adaptor_row(repo, nil, attrs, _now) do %Adaptor{} |> Adaptor.changeset(attrs) diff --git a/lib/lightning/adaptors/repo_adaptor.ex b/lib/lightning/adaptors/repo_adaptor.ex index 469620df5fe..1b66bff0f67 100644 --- a/lib/lightning/adaptors/repo_adaptor.ex +++ b/lib/lightning/adaptors/repo_adaptor.ex @@ -4,9 +4,8 @@ defmodule Lightning.Adaptors.Repo.Adaptor do metadata projection used by the picker and Scheduler. Source-tagged via `:source` (`:npm | :local`) so the same package - name can coexist across sources; the unique index is `[:name, :source]` - (see §4.4 source-tagging invariant in - `.context/lightning/adaptors/REWRITE-2026-05.md`). + name can coexist across sources; the unique index is + `[:name, :source]`. Mirrors `Lightning.Adaptors.Strategy.adaptor_record` minus `:versions`, which lives on `Lightning.Adaptors.Repo.AdaptorVersion`. diff --git a/lib/lightning/adaptors/repo_adaptor_version.ex b/lib/lightning/adaptors/repo_adaptor_version.ex index ec048b81806..d0e05df3001 100644 --- a/lib/lightning/adaptors/repo_adaptor_version.ex +++ b/lib/lightning/adaptors/repo_adaptor_version.ex @@ -6,8 +6,7 @@ defmodule Lightning.Adaptors.Repo.AdaptorVersion do `deprecated`). Belongs to `Lightning.Adaptors.Repo.Adaptor` and cascade-deletes with - its parent. Mirrors `Lightning.Adaptors.Strategy.version_record` (see - §6.1 and §6.4 in `.context/lightning/adaptors/REWRITE-2026-05.md`). + its parent. Mirrors `Lightning.Adaptors.Strategy.version_record`. """ use Ecto.Schema diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 6077fc48d2a..3f275119765 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -271,10 +271,11 @@ defmodule Lightning.Adaptors.Scheduler do :touched else case strategy.fetch_adaptor(name) do - {:ok, record} -> - Logger.debug( - "Adaptors[#{state.source}]: fetched #{name}@#{record.version}" - ) + # latest_version is bound in the head rather than read inside the + # Logger call. A log message is only built when its level is enabled, + # so a field read inside one isn't exercised. + {:ok, %{latest_version: version} = record} -> + Logger.debug("Adaptors[#{state.source}]: fetched #{name}@#{version}") {:fetched, record} @@ -326,23 +327,12 @@ defmodule Lightning.Adaptors.Scheduler do |> merge_icon(:square, package_icons, state.source) |> merge_icon(:rectangle, package_icons, state.source) - try do - {:ok, _} = AdaptorsRepo.upsert_adaptor(record_with_icons) - - Phoenix.PubSub.broadcast( - Lightning.PubSub, - state.source_topic, - {:changed, name, state.source} - ) - - Logger.debug("Adaptors[#{state.source}]: persisted #{name}") - :ok - rescue - e -> - Logger.error( - "Scheduler: upsert_adaptor(#{name}) failed: #{Exception.message(e)}" - ) + case upsert_and_broadcast(record_with_icons, name, state) do + :ok -> + Logger.debug("Adaptors[#{state.source}]: persisted #{name}") + :ok + {:error, _reason} -> :error end end @@ -386,10 +376,9 @@ defmodule Lightning.Adaptors.Scheduler do Map.put(record, :"icon_#{shape}_etag", etag) end - # Top up icons on rows that currently have NULL on at least one shape. - # Runs after the main upsert pass on every tick — cheap, scoped to - # rows with gaps, and self-correcting after a strategy outage or a - # past bug like the one that left every row iconless. + # Tops up icons on rows currently missing at least one shape. Runs + # after the main upsert pass on every tick — cheap, scoped to rows + # with gaps, and self-correcting after a strategy outage. defp heal_missing_icons(icons, _state) when map_size(icons) == 0, do: 0 defp heal_missing_icons(icons, state) do @@ -512,30 +501,19 @@ defmodule Lightning.Adaptors.Scheduler do defp force_refresh_one(strategy, name, state) do case strategy.fetch_adaptor(name) do - {:ok, record} -> + {:ok, %{latest_version: version} = record} -> record_with_source = Map.put(record, :source, state.source) - try do - {:ok, _} = AdaptorsRepo.upsert_adaptor(record_with_source) - - Phoenix.PubSub.broadcast( - Lightning.PubSub, - state.source_topic, - {:changed, name, state.source} - ) - - Logger.info( - "Adaptors[#{state.source}]: refresh_package(#{name}) ok version=#{record.version}" - ) - - :ok - rescue - e -> - Logger.error( - "Scheduler: upsert_adaptor(#{name}) failed: #{Exception.message(e)}" + case upsert_and_broadcast(record_with_source, name, state) do + :ok -> + Logger.info( + "Adaptors[#{state.source}]: refresh_package(#{name}) ok version=#{version}" ) - {:error, {:upsert_failed, Exception.message(e)}} + :ok + + {:error, _reason} = error -> + error end {:error, reason} -> @@ -547,6 +525,28 @@ defmodule Lightning.Adaptors.Scheduler do end end + # The rescue deliberately covers only the upsert and the broadcast. Callers + # log their own success line outside it, so a mistake in that line crashes + # rather than being reported back as a failed upsert. + defp upsert_and_broadcast(record, name, state) do + {:ok, _} = AdaptorsRepo.upsert_adaptor(record) + + Phoenix.PubSub.broadcast( + Lightning.PubSub, + state.source_topic, + {:changed, name, state.source} + ) + + :ok + rescue + e -> + Logger.error( + "Scheduler: upsert_adaptor(#{name}) failed: #{Exception.message(e)}" + ) + + {:error, {:upsert_failed, Exception.message(e)}} + end + # Project a list of adaptor rows to the prior-etag map shape expected # by `Strategy.fetch_icons/1`: `%{name => %{shape => etag}}`. Rows # whose etags are both nil are skipped entirely (no empty inner map); @@ -576,7 +576,6 @@ defmodule Lightning.Adaptors.Scheduler do defp maybe_put_shape_etag(map, shape, etag) when is_binary(etag), do: Map.put(map, shape, etag) - # Count :not_modified sentinels across all shapes in the icons map. # Used in the tick summary log. defp count_not_modified(icons) do Enum.reduce(icons, 0, fn {_name, shapes}, acc -> diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index a1d23c781d9..c72dbc42c4c 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -13,8 +13,7 @@ defmodule Lightning.Adaptors.Store do Each cache key carries the active `:source` (`:npm | :local`) read via `Lightning.Adaptors.Supervisor.source/1`, so the same package name can - coexist across deployment modes without manual scrubbing (see §4.4 of - `.context/adaptors/REWRITE-2026-05.md`). + coexist across deployment modes without manual scrubbing. ## Commit vs ignore diff --git a/lib/lightning/adaptors/supervisor.ex b/lib/lightning/adaptors/supervisor.ex index c1691630854..2cc47205ddb 100644 --- a/lib/lightning/adaptors/supervisor.ex +++ b/lib/lightning/adaptors/supervisor.ex @@ -3,11 +3,11 @@ defmodule Lightning.Adaptors.Supervisor do Per-instance supervisor for the `Lightning.Adaptors.*` subsystem. The entire subsystem boots, crashes, and is supervised as a unit - under `:rest_for_one`. `Cachex` is the load-bearing root: if it - crashes, the supervisor restarts it and cascades to its dependents - (`Task.Supervisor`, plus the broadcaster/scheduler children added in - later phases) so they re-bind to the fresh Cachex name on the way - back up. + under `:rest_for_one`. `Cachex` is the first child: if it crashes, + the supervisor restarts it and every child listed after it + (`Task.Supervisor`, `Invalidator`, `NodeMonitor`, `ChannelBroadcaster`, + and the `HighlanderPG`-wrapped `Scheduler`) so they re-bind to the + fresh Cachex name on the way back up. No registered name, Cachex table name, PubSub topic, `Task.Supervisor` name, or `HighlanderPG` lock key is hardcoded. Every name is derived @@ -107,8 +107,7 @@ defmodule Lightning.Adaptors.Supervisor do {Lightning.Adaptors.ChannelBroadcaster, name: channel_broadcaster_name(name), source_topic: source_topic, - client_topic: client_topic, - sup: name}, + client_topic: client_topic}, Supervisor.child_spec( {HighlanderPG, child: scheduler_child, diff --git a/lib/lightning/ai_assistant/ai_assistant.ex b/lib/lightning/ai_assistant/ai_assistant.ex index 28e00592f18..e84214afe5d 100644 --- a/lib/lightning/ai_assistant/ai_assistant.ex +++ b/lib/lightning/ai_assistant/ai_assistant.ex @@ -553,7 +553,7 @@ defmodule Lightning.AiAssistant do %{ session | expression: expression, - adaptor: Lightning.AdaptorRegistry.resolve_adaptor(adaptor) + adaptor: Lightning.Adaptors.PackageName.to_wire(adaptor) } end diff --git a/lib/lightning/config/bootstrap.ex b/lib/lightning/config/bootstrap.ex index 67b35b38ba3..f947a8a4ef4 100644 --- a/lib/lightning/config/bootstrap.ex +++ b/lib/lightning/config/bootstrap.ex @@ -268,6 +268,28 @@ defmodule Lightning.Config.Bootstrap do local_adaptors_repos: if(use_local_adaptors_repos?, do: local_adaptors_repos, else: []) + # Upstreams for the NPM strategy. Each key reaches exactly one sub-module + # through Lightning.Adaptors.Config.strategy_opts/1: registry_url is the + # npm search and packument endpoint (NPM.Registry), jsdelivr_url serves + # configuration schemas (NPM.Schema), and github_url plus github_ref locate + # the raw icon files under OpenFn/adaptors (NPM.GitHub). The defaults match + # the @default_* attributes in those modules. + # + # Point them at `bin/adaptor_cache` to serve all three from a local disk + # cache while working on adaptors. + config :lightning, Lightning.Adaptors.NPM, + registry_url: + env!("ADAPTOR_REGISTRY_URL", :string, "https://registry.npmjs.org"), + jsdelivr_url: + env!("ADAPTOR_JSDELIVR_URL", :string, "https://cdn.jsdelivr.net"), + github_url: + env!( + "ADAPTOR_GITHUB_URL", + :string, + "https://raw.githubusercontent.com" + ), + github_ref: env!("ADAPTOR_GITHUB_REF", :string, "main") + config :lightning, schemas_path: env!( diff --git a/lib/lightning/credentials.ex b/lib/lightning/credentials.ex index 055cbfe378c..f9c49b17fe8 100644 --- a/lib/lightning/credentials.ex +++ b/lib/lightning/credentials.ex @@ -587,15 +587,12 @@ defmodule Lightning.Credentials do """ @spec get_schema(String.t()) :: Credentials.Schema.t() def get_schema(schema_name) do - {:ok, schemas_path} = Application.fetch_env(:lightning, :schemas_path) - - File.read("#{schemas_path}/#{schema_name}.json") - |> case do - {:ok, raw_json} -> - Credentials.Schema.new(raw_json, schema_name) + case Lightning.Adaptors.schema(schema_name) do + {:ok, schema_body} -> + Credentials.Schema.new(schema_body, schema_name) {:error, reason} -> - raise "Error reading credential schema. Got: #{reason |> inspect()}" + raise "Error reading credential schema. Got: #{inspect(reason)}" end end diff --git a/lib/lightning_web/channels/run_with_options.ex b/lib/lightning_web/channels/run_with_options.ex index b79a71634a6..15c2e4cf4e0 100644 --- a/lib/lightning_web/channels/run_with_options.ex +++ b/lib/lightning_web/channels/run_with_options.ex @@ -1,7 +1,7 @@ defmodule LightningWeb.RunWithOptions do @moduledoc false - alias Lightning.AdaptorRegistry + alias Lightning.Adaptors.PackageName alias Lightning.Run alias Lightning.Workflows.Snapshot.Edge alias Lightning.Workflows.Snapshot.Job @@ -41,7 +41,7 @@ defmodule LightningWeb.RunWithOptions do def render(%Job{} = job) do %{ "id" => job.id, - "adaptor" => AdaptorRegistry.resolve_adaptor(job.adaptor), + "adaptor" => PackageName.to_wire(job.adaptor), "credential_id" => get_credential_id(job), "body" => job.body, "name" => job.name diff --git a/lib/lightning_web/channels/workflow_channel.ex b/lib/lightning_web/channels/workflow_channel.ex index e53ff7fb886..e0301698ddc 100644 --- a/lib/lightning_web/channels/workflow_channel.ex +++ b/lib/lightning_web/channels/workflow_channel.ex @@ -32,7 +32,6 @@ defmodule LightningWeb.WorkflowChannel do alias Lightning.VersionControl alias Lightning.VersionControl.VersionControlUsageLimiter alias Lightning.Workflows - alias Lightning.Workflows.Job alias Lightning.Workflows.Snapshot alias Lightning.Workflows.WorkflowRelease alias Lightning.Workflows.WorkflowReleases @@ -92,6 +91,11 @@ defmodule LightningWeb.WorkflowChannel do "workflow:collaborate:#{workflow_id}" ) + Phoenix.PubSub.subscribe( + Lightning.PubSub, + Lightning.Adaptors.Supervisor.client_topic(Lightning.Adaptors) + ) + {:ok, assign(socket, workflow_id: workflow_id, @@ -119,40 +123,11 @@ defmodule LightningWeb.WorkflowChannel do @impl true def handle_in("request_adaptors", _payload, socket) do async_task(socket, "request_adaptors", fn -> - adaptors = Lightning.AdaptorRegistry.all() - %{adaptors: adaptors} - end) - end + adaptors = + list_all_packages() + |> Enum.map(&with_icon_urls/1) - @impl true - def handle_in("request_project_adaptors", _payload, socket) do - project = socket.assigns.project - - async_task(socket, "request_project_adaptors", fn -> - project_adaptor_names = - from(j in Job, - join: w in assoc(j, :workflow), - where: w.project_id == ^project.id, - select: j.adaptor, - distinct: true - ) - |> Lightning.Repo.all() - |> Enum.sort() - - all_adaptors = Lightning.AdaptorRegistry.all() - - project_adaptors = - all_adaptors - |> Enum.filter(fn adaptor -> - Enum.any?(project_adaptor_names, fn used_adaptor -> - String.starts_with?(used_adaptor, adaptor.name) - end) - end) - - %{ - project_adaptors: project_adaptors, - all_adaptors: all_adaptors - } + %{adaptors: adaptors} end) end @@ -993,6 +968,12 @@ defmodule LightningWeb.WorkflowChannel do {:noreply, socket} end + @impl true + def handle_info(%{event: "adaptors_updated", payload: payload}, socket) do + push(socket, "adaptors_updated", payload) + {:noreply, socket} + end + @impl true def handle_info( %{event: "webhook_auth_methods_updated", payload: webhook_auth_methods}, @@ -1267,6 +1248,30 @@ defmodule LightningWeb.WorkflowChannel do {:noreply, socket} end + defp list_all_packages do + case Lightning.Adaptors.packages() do + {:ok, pkgs} -> pkgs + {:error, _} -> [] + end + end + + defp with_icon_urls(adaptor) do + Map.put(adaptor, :icon_urls, icon_urls_for(adaptor.name)) + end + + defp icon_urls_for(name) do + case Lightning.Adaptors.icon_meta(name) do + {:ok, meta} -> + %{ + square: LightningWeb.AdaptorIconURL.build(name, meta, :square), + rectangle: LightningWeb.AdaptorIconURL.build(name, meta, :rectangle) + } + + {:error, :not_found} -> + %{square: nil, rectangle: nil} + end + end + defp handle_async_event("request_run_steps", socket_ref, reply) do unwrapped_reply = unwrap_run_steps_reply(reply) reply(socket_ref, unwrapped_reply) @@ -1275,7 +1280,6 @@ defmodule LightningWeb.WorkflowChannel do defp handle_async_event(event, socket_ref, reply) when event in [ "request_adaptors", - "request_project_adaptors", "request_credentials", "request_metadata", "request_current_user", diff --git a/lib/lightning_web/components/layouts/settings.html.heex b/lib/lightning_web/components/layouts/settings.html.heex index 2fabf7de552..4b8226ab2f2 100644 --- a/lib/lightning_web/components/layouts/settings.html.heex +++ b/lib/lightning_web/components/layouts/settings.html.heex @@ -73,6 +73,13 @@ <.icon name="hero-circle-stack" class="h-5 w-5 shrink-0" /> Collections + + <.icon name="hero-wrench-screwdriver" class="h-5 w-5 shrink-0" /> + Maintenance + put_resp_header("etag", etag) + |> put_resp_header("cache-control", "private, no-cache") + |> put_resp_header("vary", "Cookie") + + if get_req_header(conn, "if-none-match") == [etag] do + send_resp(conn, 304, "") + else + json(conn, %{data: Enum.map(Adaptors.catalogue(), &render_entry/1)}) + end + end + + defp render_entry(entry) do + %{ + name: entry.name, + latest_version: entry.latest_version, + versions: entry.versions, + repository: entry.repository, + icon_urls: %{ + square: AdaptorIconURL.build(entry.name, entry, :square), + rectangle: AdaptorIconURL.build(entry.name, entry, :rectangle) + } + } + end + + defp etag_for({nil, 0}), do: ~s("empty") + + defp etag_for({%DateTime{} = stamp, count}), + do: ~s("#{DateTime.to_iso8601(stamp)}-#{count}") +end diff --git a/lib/lightning_web/controllers/adaptor_icon_controller.ex b/lib/lightning_web/controllers/adaptor_icon_controller.ex new file mode 100644 index 00000000000..4da1d83df4a --- /dev/null +++ b/lib/lightning_web/controllers/adaptor_icon_controller.ex @@ -0,0 +1,149 @@ +defmodule LightningWeb.AdaptorIconURL do + @moduledoc """ + Single source of truth for content-addressable adaptor-icon URLs. + + Called from `LightningWeb.AdaptorIconController` for redirect targets and + from `WorkflowChannel`'s `request_adaptors` payload. + + `sha8` is the first 4 raw bytes of the icon's sha256, hex-encoded + to 8 lowercase characters, yielding a deterministic content-addressable + path segment. + """ + + @doc """ + Build a content-addressable icon URL for `name`/`shape`. + + Returns `nil` when the adaptor row has no ext or sha256 for the + requested shape — i.e. when no icon is available. + """ + @spec build(String.t(), map(), :square | :rectangle) :: String.t() | nil + def build(name, meta, shape) do + with ext when not is_nil(ext) <- Map.get(meta, :"icon_#{shape}_ext"), + sha when not is_nil(sha) <- Map.get(meta, :"icon_#{shape}_sha256") do + sha8 = sha |> binary_part(0, 4) |> Base.encode16(case: :lower) + + "/adaptors/icons/#{URI.encode(name, &URI.char_unreserved?/1)}/" <> + "#{shape}-#{sha8}.#{ext}" + else + _ -> nil + end + end +end + +defmodule LightningWeb.AdaptorIconController do + @moduledoc """ + Serves content-addressable adaptor icons. + + Route: `/adaptors/icons/:name/:shape-:sha8.:ext` + + `sha8` is the first 4 raw bytes of the stored sha256 hex-encoded to + 8 lowercase characters. The controller compares `sha8` against the + DB-projected metadata and responds with one of: + + - **200** — sha matches; serves bytes with a 1-year immutable cache. + - **302** — sha is stale but the adaptor still has an icon; redirects + to the canonical (current-sha) URL with `Cache-Control: no-store` + on the redirect itself. + - **404** — adaptor unknown, ext mismatch, bad shape, or no icon. + """ + + use LightningWeb, :controller + + alias Lightning.Adaptors + + @immutable_cache "public, max-age=31536000, immutable" + + # Router-shaped params: a single `:filename` segment of the form + # `-.` because Phoenix path matchers permit only one + # dynamic segment per path component. We split here and delegate to the + # 4-key clause below, which is also what the unit tests call directly. + @filename_regex ~r/\A(?[a-z]+)-(?[A-Fa-f0-9]+)\.(?[A-Za-z0-9]+)\z/ + + @doc false + def show(conn, %{"name" => name, "filename" => filename}) do + case Regex.named_captures(@filename_regex, filename) do + %{"shape" => shape, "sha8" => sha8, "ext" => ext} -> + show(conn, %{ + "name" => name, + "shape" => shape, + "sha8" => sha8, + "ext" => ext + }) + + _ -> + send_resp(conn, 404, "") + end + end + + def show( + conn, + %{"name" => name, "shape" => shape, "sha8" => sha8, "ext" => ext} + ) + when shape in ~w(square rectangle) do + case Adaptors.icon_meta(name) do + {:error, :not_found} -> + send_resp(conn, 404, "") + + {:ok, meta} -> + cond do + ext_for_shape(meta, shape) != ext -> + send_resp(conn, 404, "") + + not has_icon?(meta, shape) -> + send_resp(conn, 404, "") + + sha_matches?(meta, shape, sha8) -> + serve_bytes(conn, name, shape, ext) + + true -> + redirect_to_current(conn, name, meta, shape) + end + end + end + + def show(conn, _params), do: send_resp(conn, 404, "") + + defp serve_bytes(conn, name, shape, ext) do + case Adaptors.icon(name, String.to_existing_atom(shape)) do + {:ok, path} -> + conn + |> put_resp_content_type(content_type_for(ext)) + |> put_resp_header("cache-control", @immutable_cache) + |> send_file(200, path) + + {:error, _} -> + send_resp(conn, 404, "") + end + end + + defp redirect_to_current(conn, name, meta, shape) do + url = + LightningWeb.AdaptorIconURL.build( + name, + meta, + String.to_existing_atom(shape) + ) + + conn + |> put_resp_header("cache-control", "no-store") + |> put_resp_header("location", url) + |> send_resp(302, "") + end + + defp has_icon?(meta, shape), do: not is_nil(ext_for_shape(meta, shape)) + + defp ext_for_shape(meta, shape), do: Map.get(meta, :"icon_#{shape}_ext") + + defp sha_matches?(meta, shape, sha8) do + case Map.get(meta, :"icon_#{shape}_sha256") do + <> -> + Base.encode16(prefix, case: :lower) == String.downcase(sha8) + + _ -> + false + end + end + + defp content_type_for("png"), do: "image/png" + defp content_type_for("svg"), do: "image/svg+xml" +end diff --git a/lib/lightning_web/live/maintenance_live/index.ex b/lib/lightning_web/live/maintenance_live/index.ex new file mode 100644 index 00000000000..e00ab587fe3 --- /dev/null +++ b/lib/lightning_web/live/maintenance_live/index.ex @@ -0,0 +1,85 @@ +defmodule LightningWeb.MaintenanceLive.Index do + @moduledoc """ + Superuser-only maintenance page for on-demand operations against + `Lightning.Adaptors`. + + Exposes two actions: "Refresh Adaptor Registry" (`refresh_now/0`) and + "Refresh Adaptor Icons" (`refresh_icons/0`). Both are fire-and-forget: the + user gets a flash and the actual work happens asynchronously on the leader + node. + """ + use LightningWeb, :live_view + + alias Lightning.Policies.Permissions + alias Lightning.Policies.Users + + @impl true + def mount(_params, _session, socket) do + if superuser?(socket) do + {:ok, + assign(socket, + active_menu_item: :maintenance, + page_title: "Maintenance" + ), layout: {LightningWeb.Layouts, :settings}} + else + {:ok, + socket + |> put_flash(:nav, :no_access) + |> push_navigate(to: "/projects")} + end + end + + @impl true + def handle_event("refresh_adaptors", _params, socket) do + if superuser?(socket) do + socket = + case Lightning.Adaptors.refresh_now() do + :ok -> + put_flash(socket, :info, "Adaptor refresh queued.") + + {:error, reason} -> + put_flash(socket, :error, "Refresh failed: #{inspect(reason)}") + end + + {:noreply, socket} + else + {:noreply, + socket + |> put_flash(:nav, :no_access) + |> push_navigate(to: "/projects")} + end + end + + def handle_event("refresh_icons", _params, socket) do + if superuser?(socket) do + socket = + case Lightning.Adaptors.refresh_icons() do + {:ok, %{updated: updated, unchanged: unchanged}} -> + put_flash( + socket, + :info, + "Icon refresh complete — #{updated} updated, #{unchanged} unchanged." + ) + + {:error, reason} -> + put_flash(socket, :error, "Icon refresh failed: #{inspect(reason)}") + end + + {:noreply, socket} + else + {:noreply, + socket + |> put_flash(:nav, :no_access) + |> push_navigate(to: "/projects")} + end + end + + defp superuser?(socket) do + Permissions.can?( + Users, + :access_admin_space, + socket.assigns.current_user, + {} + ) + end +end diff --git a/lib/lightning_web/live/maintenance_live/index.html.heex b/lib/lightning_web/live/maintenance_live/index.html.heex new file mode 100644 index 00000000000..eb018839448 --- /dev/null +++ b/lib/lightning_web/live/maintenance_live/index.html.heex @@ -0,0 +1,53 @@ + + <:header> + + <:title>Maintenance + + + +
+
+
+

+ Refresh Adaptor Registry +

+

+ Re-fetch the list of available adaptors and their versions. +

+
+ <.button + theme="primary" + phx-click="refresh_adaptors" + id="refresh-adaptors-button" + > + Run + +
+ +
+
+

+ Refresh Adaptor Icons +

+

+ Re-fetch icons for every adaptor and replace rows whose icon has + changed. Does not touch adaptor metadata or version history. +

+
+ <.button + theme="primary" + phx-click="refresh_icons" + id="refresh-icons-button" + > + Run + +
+
+ + diff --git a/lib/lightning_web/router.ex b/lib/lightning_web/router.ex index d873ebcc6b6..5a60abcc5a1 100644 --- a/lib/lightning_web/router.ex +++ b/lib/lightning_web/router.ex @@ -67,6 +67,10 @@ defmodule LightningWeb.Router do get "/authenticate/:provider/callback", OidcController, :new get "/oauth/:provider/callback", OauthController, :new + + get "/adaptors/icons/:name/:filename", + AdaptorIconController, + :show end ## JSON API @@ -122,6 +126,13 @@ defmodule LightningWeb.Router do :runs end + ## Adaptor catalogue (cookie-authenticated JSON) + scope "/", LightningWeb do + pipe_through [:authenticated_json, :require_authenticated_user] + + get "/adaptors/catalogue", AdaptorController, :index + end + ## Collections scope "/collections", LightningWeb do pipe_through [:authenticated_api] @@ -240,6 +251,8 @@ defmodule LightningWeb.Router do live "/settings/audit", AuditLive.Index, :index + live "/settings/maintenance", MaintenanceLive.Index, :index + live "/settings/authentication", AuthProvidersLive.Index, :edit live "/settings/authentication/new", AuthProvidersLive.Index, :new diff --git a/lib/mix/tasks/lightning.refresh_adaptors.ex b/lib/mix/tasks/lightning.refresh_adaptors.ex new file mode 100644 index 00000000000..74707fec9b7 --- /dev/null +++ b/lib/mix/tasks/lightning.refresh_adaptors.ex @@ -0,0 +1,59 @@ +defmodule Mix.Tasks.Lightning.RefreshAdaptors do + @shortdoc "On-demand adaptor metadata refresh" + @moduledoc """ + Trigger an immediate adaptor refresh from the command line. + + Use cases: + + * Dev re-scan — force a re-scan after adding local adaptors + * Ops force-pull — pull latest metadata without waiting for the scheduler tick + + ## Usage + + mix lightning.refresh_adaptors + mix lightning.refresh_adaptors --name @openfn/language-http + + The first form calls `Lightning.Adaptors.refresh_now/0`, refreshing all + adaptors. The second form calls `Lightning.Adaptors.refresh_package/1` + to force a single-adaptor refresh, bypassing the ledger diff. + + Both forms block until completion. The Scheduler is wrapped in + `HighlanderPG` and registered globally, so the call routes through + Erlang distribution to whichever node currently holds the lease — the + CLI can be run from any node in the cluster. + + ## Exit codes + + * `0` — success + * `1` — package name not found (possible typo) + * `2` — other error + """ + + use Mix.Task + + @impl Mix.Task + def run(argv) do + Mix.Task.run("app.start") + + {opts, _args} = OptionParser.parse!(argv, strict: [name: :string]) + + result = + case opts[:name] do + nil -> Lightning.Adaptors.refresh_now() + pkg -> Lightning.Adaptors.refresh_package(pkg) + end + + case result do + :ok -> + Mix.shell().info("Adaptors refreshed successfully.") + + {:error, :not_found} -> + Mix.shell().error("Package not found. Check the name and try again.") + exit({:shutdown, 1}) + + {:error, reason} -> + Mix.shell().error("Refresh failed: #{inspect(reason)}") + exit({:shutdown, 2}) + end + end +end diff --git a/priv/repo/migrations/20260827084128_add_adaptor_catalogue_indexes.exs b/priv/repo/migrations/20260827084128_add_adaptor_catalogue_indexes.exs new file mode 100644 index 00000000000..bd6f5a6542d --- /dev/null +++ b/priv/repo/migrations/20260827084128_add_adaptor_catalogue_indexes.exs @@ -0,0 +1,8 @@ +defmodule Lightning.Repo.Migrations.AddAdaptorCatalogueIndexes do + use Ecto.Migration + + def change do + create index(:adaptors, [:updated_at]) + create index(:adaptor_versions, [:inserted_at]) + end +end diff --git a/test/integration/web_and_worker_test.exs b/test/integration/web_and_worker_test.exs index d01baa4aa03..e8a2c5de729 100644 --- a/test/integration/web_and_worker_test.exs +++ b/test/integration/web_and_worker_test.exs @@ -121,6 +121,13 @@ defmodule Lightning.WebAndWorkerTest do @tag :integration @tag timeout: 20_000 test "the whole thing", %{conn: conn, user: user} do + # Seed a concrete version so `PackageName.to_wire/1` resolves + # `@latest` without hitting the live NPM registry. + Lightning.AdaptorTestHelpers.seed_adaptor_package( + "@openfn/language-http", + "3.1.12" + ) + project = insert(:project) # Create credential with body for main environment diff --git a/test/lightning/adaptors/channel_broadcaster_test.exs b/test/lightning/adaptors/channel_broadcaster_test.exs index 002f0e19978..de6c447da2f 100644 --- a/test/lightning/adaptors/channel_broadcaster_test.exs +++ b/test/lightning/adaptors/channel_broadcaster_test.exs @@ -1,9 +1,8 @@ defmodule Lightning.Adaptors.ChannelBroadcasterTest do @moduledoc """ - Tests `:flush` via `Lightning.Adaptors.packages/1` (the 2-arity facade), - not `packages/0`, because each test spins up its own isolated supervisor - instance. The Batch 7 review should confirm that `/1` and `/0` are - behaviourally identical in production (both delegate to `Store.packages/1`). + Exercises `Lightning.Adaptors.ChannelBroadcaster` through its real + message interface: broadcasting `{:changed, name, source}` tuples onto + `:source_topic` and asserting what it forwards to `:client_topic`. """ use ExUnit.Case, async: true @@ -13,170 +12,101 @@ defmodule Lightning.Adaptors.ChannelBroadcasterTest do setup do sup = :"cb_test_#{System.unique_integer([:positive])}" - # The supervisor's :rest_for_one child list starts the - # ChannelBroadcaster automatically — registered under - # `channel_broadcaster_name(sup)`. + # :rest_for_one starts the ChannelBroadcaster automatically, registered + # under `channel_broadcaster_name(sup)`. start_supervised!( {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} ) - # Stop the auto-started Invalidator — these tests pre-populate the - # Cachex `{:packages, source}` key directly to exercise the - # ChannelBroadcaster's `:flush` path in isolation. The Invalidator - # subscribes to the same source_topic and would race the broadcaster - # by deleting the cached entry before the flush window expires. - :ok = Supervisor.terminate_child(sup, Lightning.Adaptors.Invalidator) - source_topic = AdaptorsSupervisor.source_topic(sup) client_topic = AdaptorsSupervisor.client_topic(sup) cb_name = AdaptorsSupervisor.channel_broadcaster_name(sup) - cache = AdaptorsSupervisor.cache_name(sup) - source = AdaptorsSupervisor.source(sup) - - packages = [%{name: "@openfn/language-http", latest_version: "1.0.0"}] - Cachex.put!(cache, {:packages, source}, {:ok, packages}) :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, client_topic) - {:ok, - sup: sup, - cb_name: cb_name, - source_topic: source_topic, - cache: cache, - source: source, - packages: packages} + {:ok, cb_name: cb_name, source_topic: source_topic} end - describe "start_link/1" do - test "registers under the :name opt", %{cb_name: cb_name} do - assert is_pid(Process.whereis(cb_name)) - end + defp changed(source_topic, name) do + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + source_topic, + {:changed, name, :npm} + ) end - describe "handle_info/2 - {:changed, ...}" do - test "first message in idle state arms the 250ms timer", %{ - cb_name: cb_name, + describe "handle_info/2 - :flush" do + test "broadcasts just the changed adaptor's name", %{ source_topic: source_topic } do - Phoenix.PubSub.broadcast!( - Lightning.PubSub, - source_topic, - {:changed, "pkg", :npm} - ) - - %{timer: timer} = :sys.get_state(cb_name) - assert is_reference(timer) + changed(source_topic, "@openfn/language-http") + + assert_receive %{ + event: "adaptors_updated", + payload: %{names: ["@openfn/language-http"]} + }, + 500 end - test "subsequent messages within the window are dropped — one broadcast per burst", - %{source_topic: source_topic, packages: packages} do - for _ <- 1..5 do - Phoenix.PubSub.broadcast!( - Lightning.PubSub, - source_topic, - {:changed, "pkg", :npm} - ) - end + test "two adaptors changing within one debounce window are both named in a single broadcast", + %{source_topic: source_topic} do + changed(source_topic, "@openfn/language-salesforce") + changed(source_topic, "@openfn/language-http") + # Duplicate: must not appear twice in the output. + changed(source_topic, "@openfn/language-http") assert_receive %{ event: "adaptors_updated", - payload: %{adaptors: ^packages} + payload: %{ + names: [ + "@openfn/language-http", + "@openfn/language-salesforce" + ] + } }, 500 refute_receive %{event: "adaptors_updated"}, 100 - end - test "timer resets to nil after :flush fires", %{ - cb_name: cb_name, - source_topic: source_topic - } do - Phoenix.PubSub.broadcast!( - Lightning.PubSub, - source_topic, - {:changed, "pkg", :npm} - ) - - assert_receive %{event: "adaptors_updated"}, 500 - %{timer: timer} = :sys.get_state(cb_name) - assert timer == nil - end - end - - describe "handle_info/2 - :flush" do - test "broadcasts the envelope to client_topic with the correct shape", %{ - source_topic: source_topic, - packages: packages - } do - Phoenix.PubSub.broadcast!( - Lightning.PubSub, - source_topic, - {:changed, "pkg", :npm} - ) + # A second, separate burst must not still carry the first burst's + # names — the accumulator has to reset after :flush. + changed(source_topic, "@openfn/language-dhis2") assert_receive %{ event: "adaptors_updated", - payload: %{adaptors: ^packages} + payload: %{names: ["@openfn/language-dhis2"]} }, 500 end - - test "broadcasts with empty adaptors list when packages returns {:ok, []}", - %{ - cache: cache, - source: source, - source_topic: source_topic - } do - Cachex.put!(cache, {:packages, source}, {:ok, []}) - - Phoenix.PubSub.broadcast!( - Lightning.PubSub, - source_topic, - {:changed, "pkg", :npm} - ) - - assert_receive %{event: "adaptors_updated", payload: %{adaptors: []}}, 500 - end end describe "crash recovery" do test "supervisor restarts the GenServer; next {:changed} re-arms cleanly", %{ cb_name: cb_name, - source_topic: source_topic, - packages: packages + source_topic: source_topic } do original_pid = Process.whereis(cb_name) assert is_pid(original_pid) ref = Process.monitor(original_pid) - # Arm the timer, then kill the process mid-burst. - Phoenix.PubSub.broadcast!( - Lightning.PubSub, - source_topic, - {:changed, "pkg", :npm} - ) + changed(source_topic, "@openfn/language-http") Process.exit(original_pid, :kill) - # Confirm death before looking for the restarted process. assert_receive {:DOWN, ^ref, :process, ^original_pid, :killed}, 500 new_pid = await_registered(cb_name) assert is_pid(new_pid) assert new_pid != original_pid - # The new instance starts with timer: nil — one more {:changed} opens a - # fresh 250ms window and produces a clean broadcast. - Phoenix.PubSub.broadcast!( - Lightning.PubSub, - source_topic, - {:changed, "pkg", :npm} - ) + # The restarted GenServer starts with timer: nil and an empty + # names accumulator, so this reopens a fresh window. + changed(source_topic, "@openfn/language-http") assert_receive %{ event: "adaptors_updated", - payload: %{adaptors: ^packages} + payload: %{names: ["@openfn/language-http"]} }, 500 end @@ -188,12 +118,7 @@ defmodule Lightning.Adaptors.ChannelBroadcasterTest do task = Task.async(fn -> for _ <- 1..50 do - Phoenix.PubSub.broadcast!( - Lightning.PubSub, - source_topic, - {:changed, "pkg", :npm} - ) - + changed(source_topic, "@openfn/language-http") Process.sleep(10) end end) @@ -204,8 +129,6 @@ defmodule Lightning.Adaptors.ChannelBroadcasterTest do count = drain_broadcasts() - # Leading-edge invariant: throttle produces some broadcasts (> 0) - # but far fewer than one per message (< 50). assert count > 0 and count < 50, "Expected leading-edge throttling (1..49), got #{count}" end diff --git a/test/lightning/adaptors/end_to_end_broadcast_test.exs b/test/lightning/adaptors/end_to_end_broadcast_test.exs index ed4e0d765c1..4e26c8295a1 100644 --- a/test/lightning/adaptors/end_to_end_broadcast_test.exs +++ b/test/lightning/adaptors/end_to_end_broadcast_test.exs @@ -1,17 +1,12 @@ defmodule Lightning.Adaptors.EndToEndBroadcastTest do @moduledoc """ - Phase A closeout — §6.5c integration smoke. - - A `{:changed, name, source}` broadcast on the per-instance source - topic (the cache-coherence audience that the `Scheduler` and - `Invalidator` share) must traverse the wired stack and arrive on - the per-instance client topic (the display-freshness audience that - `WorkflowChannel` subscribers listen to) as a single coalesced - `adaptors_updated` envelope. - - This is the only assertion that breaks if any of the four newly-wired - Supervisor children (Invalidator, NodeMonitor, ChannelBroadcaster, - Scheduler) is misconfigured for the boot path. + A `{:changed, name, source}` broadcast on the per-instance source topic + (shared by `Scheduler` and `Invalidator` for cache coherence) must reach + the per-instance client topic (which `WorkflowChannel` subscribers use + for display freshness) as a single coalesced `adaptors_updated` envelope. + + This is the only test that exercises the full wiring across Invalidator, + NodeMonitor, ChannelBroadcaster, and Scheduler. """ use Lightning.DataCase, async: false @@ -38,11 +33,10 @@ defmodule Lightning.Adaptors.EndToEndBroadcastTest do {:changed, "@openfn/language-test", :local} ) - # The ChannelBroadcaster fans out a map envelope (see - # `Lightning.Adaptors.ChannelBroadcaster.handle_info(:flush, _)`). - # The DB is empty in this case → `Store.packages/1` returns - # `{:ok, []}` → an empty-list envelope is broadcast. - assert_receive %{event: "adaptors_updated", payload: %{adaptors: _}}, + assert_receive %{ + event: "adaptors_updated", + payload: %{names: ["@openfn/language-test"]} + }, ChannelBroadcaster.debounce_ms() + 200 end end diff --git a/test/lightning/adaptors/highlander_integration_test.exs b/test/lightning/adaptors/highlander_integration_test.exs index bbcc3472dce..8dda75166ad 100644 --- a/test/lightning/adaptors/highlander_integration_test.exs +++ b/test/lightning/adaptors/highlander_integration_test.exs @@ -1,8 +1,8 @@ defmodule Lightning.Adaptors.HighlanderIntegrationTest do @moduledoc """ - §12.7 — verifies that the HighlanderPG-wrapped `Lightning.Adaptors.Scheduler` - actually behaves as a cluster singleton when two supervisor instances - compete for the same Postgres advisory lock. + Verifies that the HighlanderPG-wrapped `Lightning.Adaptors.Scheduler` + behaves as a cluster singleton when two supervisor instances compete for + the same Postgres advisory lock. Both supervisors share an explicit `:lock_key` so they race for the same `pg_try_advisory_lock` bucket, but each keeps its own derived @@ -86,8 +86,6 @@ defmodule Lightning.Adaptors.HighlanderIntegrationTest do leader_sup = if leader_global == gname_a, do: sup_a, else: sup_b - # Singleton invariant: only the leader's :global registration is - # populated cluster-wide. assert :global.whereis_name(surviving_global) == :undefined, "expected only the leader to have a globally-registered Scheduler" @@ -101,7 +99,6 @@ defmodule Lightning.Adaptors.HighlanderIntegrationTest do # its wrapped Scheduler under its own :global name. assert_eventually(is_pid(:global.whereis_name(surviving_global)), 3_000) - # Sanity: the formerly-leading :global name is gone. assert :global.whereis_name(leader_global) == :undefined end end diff --git a/test/lightning/adaptors/invalidator_test.exs b/test/lightning/adaptors/invalidator_test.exs index 4ac7feac8d1..365925a6822 100644 --- a/test/lightning/adaptors/invalidator_test.exs +++ b/test/lightning/adaptors/invalidator_test.exs @@ -6,8 +6,8 @@ defmodule Lightning.Adaptors.InvalidatorTest do setup do sup = :"inv_test_#{System.unique_integer([:positive])}" - # The supervisor's :rest_for_one child list starts the Invalidator - # automatically — registered under `invalidator_name(sup)`. + # :rest_for_one starts the Invalidator automatically, registered under + # `invalidator_name(sup)`. start_supervised!( {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} ) diff --git a/test/lightning/adaptors/node_monitor_test.exs b/test/lightning/adaptors/node_monitor_test.exs index c0b212550c0..f7888e1142d 100644 --- a/test/lightning/adaptors/node_monitor_test.exs +++ b/test/lightning/adaptors/node_monitor_test.exs @@ -11,8 +11,8 @@ defmodule Lightning.Adaptors.NodeMonitorTest do setup do sup = :"nm_test_#{System.unique_integer([:positive])}" - # The supervisor's :rest_for_one child list starts the NodeMonitor - # automatically — registered under `node_monitor_name(sup)`. + # :rest_for_one starts the NodeMonitor automatically, registered under + # `node_monitor_name(sup)`. start_supervised!( {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} ) diff --git a/test/lightning/adaptors/npm/github_test.exs b/test/lightning/adaptors/npm/github_test.exs index 3f09b0ae966..997a2acfea9 100644 --- a/test/lightning/adaptors/npm/github_test.exs +++ b/test/lightning/adaptors/npm/github_test.exs @@ -337,8 +337,7 @@ defmodule Lightning.Adaptors.NPM.GitHubTest do %{"@openfn/language-http" => %{square: etag}} ) - # explicit sentinel — distinct from "absent" (which would mean upstream - # had no such shape at all). + # Distinct from a missing key, which would mean upstream had no such asset at all. assert map["@openfn/language-http"][:square] == :not_modified end diff --git a/test/lightning/adaptors/npm/registry_test.exs b/test/lightning/adaptors/npm/registry_test.exs index 760b4891264..2132eb14d47 100644 --- a/test/lightning/adaptors/npm/registry_test.exs +++ b/test/lightning/adaptors/npm/registry_test.exs @@ -11,9 +11,8 @@ defmodule Lightning.Adaptors.NPM.RegistryTest do http_timeout: 1_000 ) - # Per-test Tesla adapter override — config/test.exs globally pins - # `Lightning.Tesla.Mock`, but we need the real Finch adapter so that - # Bypass actually receives requests over a socket. + # config/test.exs globally pins Lightning.Tesla.Mock; override to the real + # Finch adapter here so Bypass actually receives requests over a socket. prev_adapter = Application.get_env(:tesla, :adapter) Application.put_env( diff --git a/test/lightning/adaptors/npm_test.exs b/test/lightning/adaptors/npm_test.exs index ad2ea8ded67..4088fc33520 100644 --- a/test/lightning/adaptors/npm_test.exs +++ b/test/lightning/adaptors/npm_test.exs @@ -6,9 +6,8 @@ defmodule Lightning.Adaptors.NPMTest do @package "@openfn/language-http" @latest_version "2.1.0" - # Three Bypass servers: one for the npm registry, one for jsDelivr, - # one for raw.githubusercontent.com. Per-test config installs all three - # URLs onto the strategy_opts block. + # One Bypass server each for the npm registry, jsDelivr, and + # raw.githubusercontent.com; each URL is installed onto the strategy_opts block. setup do registry = Bypass.open() jsdelivr = Bypass.open() @@ -226,8 +225,6 @@ defmodule Lightning.Adaptors.NPMTest do end end - # ==================== Helpers ==================== - defp build_packument do %{ "name" => @package, diff --git a/test/lightning/adaptors/package_name_test.exs b/test/lightning/adaptors/package_name_test.exs new file mode 100644 index 00000000000..7f01aacc2b6 --- /dev/null +++ b/test/lightning/adaptors/package_name_test.exs @@ -0,0 +1,73 @@ +defmodule Lightning.Adaptors.PackageNameTest do + use Lightning.DataCase, async: false + + import Lightning.Factories + + alias Lightning.Adaptors.PackageName + + describe "parse/1" do + test "splits scoped name and semver version" do + assert PackageName.parse("@openfn/language-common@1.2.3") == + {"@openfn/language-common", "1.2.3"} + end + + test "splits unscoped name and version" do + assert PackageName.parse("foo@2.0.0") == {"foo", "2.0.0"} + end + + test "returns the name with nil version when no @version is given" do + assert PackageName.parse("@openfn/language-common") == + {"@openfn/language-common", nil} + end + + test "treats the @local literal as a version" do + assert PackageName.parse("@openfn/language-common@local") == + {"@openfn/language-common", "local"} + end + + test "treats the @latest literal as a version" do + assert PackageName.parse("@openfn/language-common@latest") == + {"@openfn/language-common", "latest"} + end + + test "returns {nil, nil} for nil input" do + assert PackageName.parse(nil) == {nil, nil} + end + + test "returns {nil, nil} for malformed input" do + assert PackageName.parse("") == {nil, nil} + end + end + + describe "to_wire/1" do + test "passes through concrete semver unchanged" do + assert PackageName.to_wire("@openfn/language-common@1.6.2") == + "@openfn/language-common@1.6.2" + end + + test "returns empty string for nil input" do + assert PackageName.to_wire(nil) == "" + end + + test "preserves @local literal regardless of source" do + assert PackageName.to_wire("@openfn/language-common@local") == + "@openfn/language-common@local" + end + + test "resolves @latest to the concrete latest_version from Adaptors.Repo" do + insert(:adaptor, + name: "@openfn/language-common", + source: :npm, + latest_version: "9.9.9" + ) + + assert PackageName.to_wire("@openfn/language-common@latest") == + "@openfn/language-common@9.9.9" + end + + test "falls back to @latest literal when adaptor is unknown" do + assert PackageName.to_wire("@openfn/never-existed@latest") == + "@openfn/never-existed@latest" + end + end +end diff --git a/test/lightning/adaptors/repo_catalogue_test.exs b/test/lightning/adaptors/repo_catalogue_test.exs new file mode 100644 index 00000000000..14b5a4f451e --- /dev/null +++ b/test/lightning/adaptors/repo_catalogue_test.exs @@ -0,0 +1,145 @@ +defmodule Lightning.Adaptors.RepoCatalogueTest do + use Lightning.DataCase, async: true + + alias Lightning.Adaptors.Repo, as: AdaptorRepo + + describe "catalogue/1" do + test "returns name, latest_version, repository, icon fields, and full version list" do + {:ok, _adaptor} = + AdaptorRepo.upsert_adaptor(%{ + name: "@openfn/language-http", + source: :npm, + latest_version: "2.0.0", + repository: "https://github.com/openfn/language-http", + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "square"), + icon_rectangle_ext: "svg", + icon_rectangle_sha256: :crypto.hash(:sha256, "rectangle"), + versions: [ + version_record("1.0.0"), + version_record("2.0.0") + ] + }) + + assert [entry] = AdaptorRepo.catalogue(:npm) + + assert entry.name == "@openfn/language-http" + assert entry.latest_version == "2.0.0" + assert entry.repository == "https://github.com/openfn/language-http" + assert entry.icon_square_ext == "png" + assert entry.icon_rectangle_ext == "svg" + assert Enum.sort(entry.versions) == ["1.0.0", "2.0.0"] + end + + test "is source-scoped" do + {:ok, _} = + AdaptorRepo.upsert_adaptor(%{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + versions: [version_record("1.0.0")] + }) + + assert AdaptorRepo.catalogue(:local) == [] + end + + test "returns an empty list for an adaptor with no versions" do + {:ok, _} = + AdaptorRepo.upsert_adaptor(%{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + versions: [] + }) + + assert [%{versions: []}] = AdaptorRepo.catalogue(:npm) + end + end + + describe "catalogue_stamp/1" do + test "returns a nil timestamp and zero count when the source has no rows" do + assert AdaptorRepo.catalogue_stamp(:npm) == {nil, 0} + end + + test "reflects the adaptor row's updated_at when there are no versions" do + {:ok, adaptor} = + AdaptorRepo.upsert_adaptor(%{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + versions: [] + }) + + assert {stamp, 0} = AdaptorRepo.catalogue_stamp(:npm) + assert stamp == adaptor.updated_at + end + + test "advances when a new version is published, without touching the adaptor row" do + {:ok, _adaptor} = + AdaptorRepo.upsert_adaptor(%{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + versions: [version_record("1.0.0")] + }) + + {before_stamp, _count} = AdaptorRepo.catalogue_stamp(:npm) + + {:ok, _adaptor} = + AdaptorRepo.upsert_adaptor(%{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + versions: [version_record("1.0.0"), version_record("1.1.0")] + }) + + {after_stamp, count} = AdaptorRepo.catalogue_stamp(:npm) + + assert DateTime.after?(after_stamp, before_stamp) + assert count == 2 + end + + test "changes when a version is removed from an adaptor that doesn't hold the current max" do + {:ok, _b} = + AdaptorRepo.upsert_adaptor(%{ + name: "@openfn/language-b", + source: :npm, + latest_version: "1.0.0", + versions: [version_record("1.0.0")] + }) + + {:ok, _a} = + AdaptorRepo.upsert_adaptor(%{ + name: "@openfn/language-a", + source: :npm, + latest_version: "1.0.0", + versions: [version_record("1.0.0")] + }) + + before_stamp = AdaptorRepo.catalogue_stamp(:npm) + + {:ok, _b} = + AdaptorRepo.upsert_adaptor(%{ + name: "@openfn/language-b", + source: :npm, + latest_version: "1.0.0", + versions: [] + }) + + assert AdaptorRepo.catalogue_stamp(:npm) != before_stamp + end + end + + defp version_record(version) do + %{ + version: version, + integrity: "sha512-#{version}", + tarball_url: "https://example.com/x/-/x-#{version}.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + end +end diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index bbf148dbffb..12eb6696352 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -501,55 +501,6 @@ defmodule Lightning.Adaptors.SchedulerTest do File.rm(icon_path) end - - test "fetches per-adaptor in parallel (multiple concurrent fetch_adaptor calls)", - %{sup: sup} do - test_pid = self() - barrier = :ets.new(:scheduler_test_barrier, [:public, :set]) - :ets.insert(barrier, {:in_flight, 0}) - :ets.insert(barrier, {:max_in_flight, 0}) - - names = - for i <- 1..6, - do: %{name: "@openfn/language-pkg#{i}", latest_version: "1.0.0"} - - expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> - {:ok, names} - end) - - stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn name -> - in_flight = :ets.update_counter(barrier, :in_flight, 1) - - :ets.update_element( - barrier, - :max_in_flight, - {2, max_seen(barrier, in_flight)} - ) - - # Hold long enough that the fan-out has time to overlap. - Process.sleep(80) - :ets.update_counter(barrier, :in_flight, -1) - send(test_pid, {:fetched, name}) - {:ok, adaptor_record(name: name)} - end) - - start_scheduler(sup) - - for _ <- 1..6 do - assert_receive {:fetched, _name}, 5_000 - end - - [{:max_in_flight, max_in_flight}] = :ets.lookup(barrier, :max_in_flight) - :ets.delete(barrier) - - assert max_in_flight > 1, - "expected concurrent fetch_adaptor calls, saw at most 1 in-flight" - end - end - - defp max_seen(barrier, current) do - [{:max_in_flight, prev}] = :ets.lookup(barrier, :max_in_flight) - max(prev, current) end describe "refresh_package/2" do @@ -559,7 +510,7 @@ defmodule Lightning.Adaptors.SchedulerTest do source_topic = AdaptorsSupervisor.source_topic(sup) stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> - send(test_pid, :init_tick_done) + send(test_pid, :init_list_adaptors_called) {:ok, []} end) @@ -576,7 +527,7 @@ defmodule Lightning.Adaptors.SchedulerTest do start_scheduler(sup) # Drain the init tick (table is empty → delay 0 → fires immediately). - assert_receive :init_tick_done, 2000 + assert_receive :init_list_adaptors_called, 2000 sched_name = AdaptorsSupervisor.global_scheduler_name(sup) @@ -590,7 +541,7 @@ defmodule Lightning.Adaptors.SchedulerTest do test_pid = self() stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> - send(test_pid, :init_tick_done) + send(test_pid, :init_list_adaptors_called) {:ok, []} end) @@ -601,7 +552,7 @@ defmodule Lightning.Adaptors.SchedulerTest do start_scheduler(sup) # Drain init tick before calling refresh_package. - assert_receive :init_tick_done, 2000 + assert_receive :init_list_adaptors_called, 2000 sched_name = AdaptorsSupervisor.global_scheduler_name(sup) @@ -615,7 +566,7 @@ defmodule Lightning.Adaptors.SchedulerTest do source_topic = AdaptorsSupervisor.source_topic(sup) stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> - send(test_pid, :init_tick_done) + send(test_pid, :init_list_adaptors_called) {:ok, []} end) @@ -636,15 +587,12 @@ defmodule Lightning.Adaptors.SchedulerTest do :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) start_scheduler(sup) - assert_receive :init_tick_done, 2000 + assert_receive :init_list_adaptors_called, 2000 assert_receive :icons_called, 2000 sched_name = AdaptorsSupervisor.global_scheduler_name(sup) assert :ok = Scheduler.refresh_package(sched_name, "@openfn/language-http") assert_receive {:changed, "@openfn/language-http", _}, 2000 - - # Give any (mistaken) extra fetch_icons call time to happen. - Process.sleep(100) end end @@ -724,7 +672,6 @@ defmodule Lightning.Adaptors.SchedulerTest do ) expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn opts -> - # Strategy receives the prior etag for this row's shape. assert Keyword.get(opts, :prior_etags) == %{ "@openfn/language-same" => %{square: etag} } @@ -899,7 +846,6 @@ defmodule Lightning.Adaptors.SchedulerTest do ) expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn opts -> - # Both rows contribute prior etags. assert Keyword.get(opts, :prior_etags) == %{ "@openfn/language-stale-etag" => %{square: stale_old_etag}, "@openfn/language-current-etag" => %{square: current_etag} diff --git a/test/lightning/adaptors/supervisor_integration_test.exs b/test/lightning/adaptors/supervisor_integration_test.exs index 608466cb860..52726311f4c 100644 --- a/test/lightning/adaptors/supervisor_integration_test.exs +++ b/test/lightning/adaptors/supervisor_integration_test.exs @@ -1,10 +1,10 @@ defmodule Lightning.Adaptors.SupervisorIntegrationTest do @moduledoc """ - Integration-level tests for `Lightning.Adaptors.Supervisor`: prove all - Phase A children boot under a single `start_supervised!` call and that - the `:rest_for_one` cascade pins §6.5a (Invalidator subscribes at init; - if Cachex restarts without Invalidator restarting, the cache goes - stale). + Integration-level tests for `Lightning.Adaptors.Supervisor`: prove the full + child list boots under a single `start_supervised!` call, and that the + `:rest_for_one` cascade restarts Invalidator when Cachex restarts (§6.5a). + Invalidator subscribes at init, so without that restart the cache would go + stale. """ use Lightning.DataCase, async: false diff --git a/test/lightning/ai_assistant/ai_assistant_test.exs b/test/lightning/ai_assistant/ai_assistant_test.exs index 4c9257e42fd..3d759dd0ab1 100644 --- a/test/lightning/ai_assistant/ai_assistant_test.exs +++ b/test/lightning/ai_assistant/ai_assistant_test.exs @@ -451,7 +451,7 @@ defmodule Lightning.AiAssistantTest do assert session.expression == job_1.body assert session.adaptor == - Lightning.AdaptorRegistry.resolve_adaptor(job_1.adaptor) + Lightning.Adaptors.PackageName.to_wire(job_1.adaptor) assert length(session.messages) == 1 message = hd(session.messages) @@ -979,7 +979,7 @@ defmodule Lightning.AiAssistantTest do assert updated_session.expression == expression assert updated_session.adaptor == - Lightning.AdaptorRegistry.resolve_adaptor(adaptor) + Lightning.Adaptors.PackageName.to_wire(adaptor) end end @@ -1103,7 +1103,7 @@ defmodule Lightning.AiAssistantTest do assert enriched.expression == job.body assert enriched.adaptor == - Lightning.AdaptorRegistry.resolve_adaptor(job.adaptor) + Lightning.Adaptors.PackageName.to_wire(job.adaptor) end test "adds run logs when follow_run_id is in meta", %{ @@ -1269,7 +1269,7 @@ defmodule Lightning.AiAssistantTest do assert enriched.expression == "console.log('test');" assert enriched.adaptor == - Lightning.AdaptorRegistry.resolve_adaptor( + Lightning.Adaptors.PackageName.to_wire( "@openfn/language-http@latest" ) end diff --git a/test/lightning/config/bootstrap_test.exs b/test/lightning/config/bootstrap_test.exs index 4873632f318..f97fa992aff 100644 --- a/test/lightning/config/bootstrap_test.exs +++ b/test/lightning/config/bootstrap_test.exs @@ -602,6 +602,40 @@ defmodule Lightning.Config.BootstrapTest do end end + describe "adaptors NPM upstream URLs" do + test "default to the production upstreams when nothing is set" do + Dotenvy.source([%{}]) + Bootstrap.configure() + + npm = get_env(:lightning, Lightning.Adaptors.NPM) + + assert npm[:registry_url] == "https://registry.npmjs.org" + assert npm[:jsdelivr_url] == "https://cdn.jsdelivr.net" + assert npm[:github_url] == "https://raw.githubusercontent.com" + assert npm[:github_ref] == "main" + end + + test "are overridden by the ADAPTOR_* env vars" do + Dotenvy.source([ + %{ + "ADAPTOR_REGISTRY_URL" => "http://localhost:4874/npm", + "ADAPTOR_JSDELIVR_URL" => "http://localhost:4874/jsdelivr", + "ADAPTOR_GITHUB_URL" => "http://localhost:4874/github", + "ADAPTOR_GITHUB_REF" => "some-feature-branch" + } + ]) + + Bootstrap.configure() + + npm = get_env(:lightning, Lightning.Adaptors.NPM) + + assert npm[:registry_url] == "http://localhost:4874/npm" + assert npm[:jsdelivr_url] == "http://localhost:4874/jsdelivr" + assert npm[:github_url] == "http://localhost:4874/github" + assert npm[:github_ref] == "some-feature-branch" + end + end + describe "per_workflow_claim_limit" do test "defaults to 50" do Dotenvy.source([%{}]) diff --git a/test/lightning/credentials/schema_test.exs b/test/lightning/credentials/schema_test.exs index e886aec7b6d..9dec9144f5e 100644 --- a/test/lightning/credentials/schema_test.exs +++ b/test/lightning/credentials/schema_test.exs @@ -2,6 +2,7 @@ defmodule Lightning.Credentials.SchemaTest do use Lightning.DataCase, async: true import ExUnit.CaptureLog + import Lightning.Factories import Mox alias Lightning.Credentials @@ -16,6 +17,17 @@ defmodule Lightning.Credentials.SchemaTest do :ok end + defp seed_adaptor_schema(name) do + # Persist the raw JSON binary (not a decoded map) so + # `Jason.decode!(_, objects: :ordered_objects)` downstream can + # preserve the schema's property order. + schema_body = + Path.join(["test", "fixtures", "schemas", "#{name}.json"]) + |> File.read!() + + insert(:adaptor, name: name, source: :npm, schema_data: schema_body) + end + setup do schema_map = """ @@ -305,7 +317,46 @@ defmodule Lightning.Credentials.SchemaTest do end end + describe "Credentials.get_schema/1" do + setup do + Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() + :ok + end + + test "preserves JSON property order from the persisted schema body" do + ordered_body = ~s({ + "properties": { + "zeta": {"type": "string"}, + "alpha": {"type": "string"}, + "mu": {"type": "string"} + }, + "type": "object" + }) + + insert(:adaptor, + name: "ordered-fixture", + source: :npm, + schema_data: ordered_body + ) + + Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() + + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + {:error, :unreachable} + end) + + schema = Credentials.get_schema("ordered-fixture") + + assert schema.fields == [:zeta, :alpha, :mu] + end + end + describe "validate/2" do + setup do + Enum.each(~w(godata postgresql http dhis2), &seed_adaptor_schema/1) + :ok + end + test "successfully validates field with json schema email format" do schema = Credentials.get_schema("godata") diff --git a/test/lightning/credentials_test.exs b/test/lightning/credentials_test.exs index ed394dc72b0..3a0df68b516 100644 --- a/test/lightning/credentials_test.exs +++ b/test/lightning/credentials_test.exs @@ -334,6 +334,12 @@ defmodule Lightning.CredentialsTest do end describe "create_credential/1" do + setup do + # create_credential/1 needs a schema on file for body casting to work. + Lightning.AdaptorTestHelpers.seed_credential_schema("postgresql") + :ok + end + test "fails if another cred exists with the same name for the same user" do user = insert(:user) @@ -491,6 +497,11 @@ defmodule Lightning.CredentialsTest do end describe "update_credential/2" do + setup do + Lightning.AdaptorTestHelpers.seed_credential_schema("postgresql") + :ok + end + test "updates an OAuth credential with new scopes" do user = insert(:user) oauth_client = insert(:oauth_client) diff --git a/test/lightning_web/channels/run_channel_test.exs b/test/lightning_web/channels/run_channel_test.exs index c49de5725d9..87c8c827c81 100644 --- a/test/lightning_web/channels/run_channel_test.exs +++ b/test/lightning_web/channels/run_channel_test.exs @@ -247,6 +247,18 @@ defmodule LightningWeb.RunChannelTest do setup :set_google_credential setup :create_socket_and_run + # `@latest` resolves via a direct `Repo.get_adaptor/2` read, so it's + # safe to seed here even though this file runs async: true. + setup do + insert(:adaptor, + name: "@openfn/language-common", + source: :npm, + latest_version: "1.6.2" + ) + + :ok + end + test "fetch:plan success", %{ socket: socket, run: run, diff --git a/test/lightning_web/channels/run_with_options_test.exs b/test/lightning_web/channels/run_with_options_test.exs index f5e0bb1bede..e64d091d4fd 100644 --- a/test/lightning_web/channels/run_with_options_test.exs +++ b/test/lightning_web/channels/run_with_options_test.exs @@ -1,5 +1,5 @@ defmodule LightningWeb.RunWithOptionsTest do - use Lightning.DataCase, async: true + use Lightning.DataCase, async: false import Lightning.Factories @@ -9,6 +9,23 @@ defmodule LightningWeb.RunWithOptionsTest do alias LightningWeb.RunWithOptions describe "rendering a run" do + setup do + # Clear the production Adaptors.Supervisor Cachex so each test's seeded + # rows are visible (Cachex persists across DB-sandbox boundaries). + cache = Lightning.Adaptors.Supervisor.cache_name(Lightning.Adaptors) + Cachex.clear(cache) + + # Seed @openfn/language-common so `@latest` resolves to a concrete + # semver via `Lightning.Adaptors.PackageName.to_wire/1`. + insert(:adaptor, + name: "@openfn/language-common", + source: :npm, + latest_version: "1.6.2" + ) + + :ok + end + test "renders a workflow using a snapshot" do user = insert(:user) @@ -119,14 +136,25 @@ defmodule LightningWeb.RunWithOptionsTest do expected_result end - @tag :tmp_dir - test "renders adaptors with @local when local_daptors_repo is configured", %{ - tmp_dir: tmp_dir - } do - Mox.stub(Lightning.MockConfig, :adaptor_registry, fn -> - [local_adaptors_repos: [tmp_dir]] + test "renders adaptors with @local when :local strategy source is active" do + prev = Application.get_env(:lightning, Lightning.Adaptors, []) + + Application.put_env( + :lightning, + Lightning.Adaptors, + Keyword.put(prev, :strategy, Lightning.Adaptors.Local) + ) + + on_exit(fn -> + Application.put_env(:lightning, Lightning.Adaptors, prev) end) + insert(:adaptor, + name: "@openfn/language-common", + source: :local, + latest_version: "local" + ) + user = insert(:user) {:ok, %{triggers: [trigger], jobs: [job]} = workflow} = diff --git a/test/lightning_web/channels/workflow_channel_test.exs b/test/lightning_web/channels/workflow_channel_test.exs index 52a94d73349..e2ea537642a 100644 --- a/test/lightning_web/channels/workflow_channel_test.exs +++ b/test/lightning_web/channels/workflow_channel_test.exs @@ -2700,89 +2700,114 @@ defmodule LightningWeb.WorkflowChannelTest do end describe "request_adaptors and request_credentials" do + setup do + # The production Adaptors.Supervisor's Cachex persists across tests; + # clear it so each test's seeded Adaptors.Repo rows are visible. + cache = Lightning.Adaptors.Supervisor.cache_name(Lightning.Adaptors) + Cachex.clear(cache) + + # Seed Adaptors.Repo rows so packages/0 returns a non-empty list. + # Individual tests insert additional rows for icon-meta assertions. + insert(:adaptor, name: "@openfn/language-salesforce", source: :npm) + insert(:adaptor, name: "@openfn/language-http", source: :npm) + :ok + end + test "handles multiple concurrent requests independently", %{ socket: socket } do ref_adaptors = push(socket, "request_adaptors", %{}) ref_credentials = push(socket, "request_credentials", %{}) - assert_reply ref_adaptors, :ok, %{adaptors: _} + assert_reply ref_adaptors, :ok, %{adaptors: adaptors} assert_reply ref_credentials, :ok, %{credentials: credentials} + assert is_list(adaptors) + assert adaptors != [] + assert Enum.all?(adaptors, &Map.has_key?(&1, :icon_urls)) + + assert Enum.all?(adaptors, fn a -> + Enum.sort(Map.keys(a.icon_urls)) == [:rectangle, :square] + end) + assert Map.has_key?(credentials, :project_credentials) assert Map.has_key?(credentials, :keychain_credentials) assert is_list(credentials.project_credentials) assert is_list(credentials.keychain_credentials) end - test "returns project-specific adaptors", %{socket: socket, project: project} do - # Create jobs with specific adaptors in this project - workflow = insert(:workflow, project: project) - - insert(:job, - workflow: workflow, - adaptor: "@openfn/language-salesforce@latest" + test "request_adaptors enriches records with icon_urls when meta present", + %{socket: socket} do + name = "@openfn/language-common" + square_sha = :crypto.strong_rand_bytes(32) + rectangle_sha = :crypto.strong_rand_bytes(32) + + insert(:adaptor, + name: name, + icon_square_ext: "png", + icon_square_sha256: square_sha, + icon_rectangle_ext: "svg", + icon_rectangle_sha256: rectangle_sha ) - insert(:job, workflow: workflow, adaptor: "@openfn/language-http@2.0.0") + ref = push(socket, "request_adaptors", %{}) + assert_reply ref, :ok, %{adaptors: adaptors} - ref = push(socket, "request_project_adaptors", %{}) + record = Enum.find(adaptors, &(&1.name == name)) + assert record, "expected legacy registry to include #{name}" - assert_reply ref, :ok, %{ - project_adaptors: project_adaptors, - all_adaptors: all_adaptors - } + {:ok, meta} = Lightning.Adaptors.icon_meta(name) - assert is_list(project_adaptors) - assert is_list(all_adaptors) + assert record.icon_urls.square == + LightningWeb.AdaptorIconURL.build(name, meta, :square) - # Verify project_adaptors contains only adaptors used in the project - project_adaptor_names = Enum.map(project_adaptors, & &1.name) - assert "@openfn/language-salesforce" in project_adaptor_names - assert "@openfn/language-http" in project_adaptor_names + assert record.icon_urls.rectangle == + LightningWeb.AdaptorIconURL.build(name, meta, :rectangle) - # Verify all_adaptors contains the full registry - assert length(all_adaptors) > 0 + assert is_binary(record.icon_urls.square) + assert is_binary(record.icon_urls.rectangle) end - test "returns empty project_adaptors for project with no jobs", %{ - socket: socket - } do - ref = push(socket, "request_project_adaptors", %{}) + test "request_adaptors emits nil icon_urls when row has no icon meta", + %{socket: socket} do + name = "@openfn/language-dhis2" + + insert(:adaptor, + name: name, + source: :npm, + icon_square_ext: nil, + icon_square_sha256: nil, + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil + ) - assert_reply ref, :ok, %{ - project_adaptors: project_adaptors, - all_adaptors: all_adaptors - } + ref = push(socket, "request_adaptors", %{}) + assert_reply ref, :ok, %{adaptors: adaptors} - assert project_adaptors == [] - assert is_list(all_adaptors) - assert length(all_adaptors) > 0 + record = Enum.find(adaptors, &(&1.name == name)) + assert record, "expected packages/0 to include #{name}" + assert record.icon_urls == %{square: nil, rectangle: nil} end - test "handles duplicate adaptors in project", %{ - socket: socket, - project: project - } do - workflow = insert(:workflow, project: project) + test "request_adaptors handles half-populated icon meta", %{socket: socket} do + name = "@openfn/language-commcare" + square_sha = :crypto.strong_rand_bytes(32) - # Create multiple jobs with the same adaptor - insert(:job, - workflow: workflow, - adaptor: "@openfn/language-common@latest" + insert(:adaptor, + name: name, + icon_square_ext: "png", + icon_square_sha256: square_sha, + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil ) - insert(:job, workflow: workflow, adaptor: "@openfn/language-common@1.0.0") - - ref = push(socket, "request_project_adaptors", %{}) - - assert_reply ref, :ok, %{project_adaptors: project_adaptors} - - # Should only appear once in project_adaptors - common_adaptors = - Enum.filter(project_adaptors, &(&1.name == "@openfn/language-common")) + ref = push(socket, "request_adaptors", %{}) + assert_reply ref, :ok, %{adaptors: adaptors} - assert length(common_adaptors) <= 1 + record = Enum.find(adaptors, &(&1.name == name)) + assert record, "expected legacy registry to include #{name}" + assert is_binary(record.icon_urls.square) + assert record.icon_urls.rectangle == nil end test "returns correctly structured project credentials", %{ @@ -4631,6 +4656,59 @@ defmodule LightningWeb.WorkflowChannelTest do end end + describe "PubSub subscription and adaptors broadcasting" do + test "forwards adaptors_updated envelope from client topic to socket", %{ + socket: _socket + } do + payload = %{adaptors: [%{name: "a"}]} + + Phoenix.PubSub.broadcast( + Lightning.PubSub, + Lightning.Adaptors.Supervisor.client_topic(Lightning.Adaptors), + %{event: "adaptors_updated", payload: payload} + ) + + assert_push "adaptors_updated", %{adaptors: [%{name: "a"}]} + end + + test "credentials_updated forwarder still pushes after adaptors clause added", + %{workflow: workflow} do + rendered_credentials = %{ + project_credentials: [], + keychain_credentials: [] + } + + Phoenix.PubSub.broadcast( + Lightning.PubSub, + "workflow:collaborate:#{workflow.id}", + %{event: "credentials_updated", payload: rendered_credentials} + ) + + assert_push "credentials_updated", %{ + project_credentials: [], + keychain_credentials: [] + } + end + + test "does not push adaptors_updated for unrelated events on client topic", + %{socket: socket} do + Process.flag(:trap_exit, true) + Process.unlink(socket.channel_pid) + ref = Process.monitor(socket.channel_pid) + + capture_log(fn -> + Phoenix.PubSub.broadcast( + Lightning.PubSub, + Lightning.Adaptors.Supervisor.client_topic(Lightning.Adaptors), + %{event: "something_else", payload: %{}} + ) + + refute_push "adaptors_updated", _, 50 + assert_receive {:DOWN, ^ref, :process, _, _}, 200 + end) + end + end + describe "request_history" do test "returns work orders with runs for workflow", %{ socket: socket, diff --git a/test/lightning_web/controllers/adaptor_controller_test.exs b/test/lightning_web/controllers/adaptor_controller_test.exs new file mode 100644 index 00000000000..cf80fd6d141 --- /dev/null +++ b/test/lightning_web/controllers/adaptor_controller_test.exs @@ -0,0 +1,136 @@ +defmodule LightningWeb.AdaptorControllerTest do + use LightningWeb.ConnCase, async: true + + import Lightning.Factories + + alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias LightningWeb.AdaptorIconURL + + describe "GET /adaptors/catalogue" do + setup %{conn: conn} do + %{conn: log_in_user(conn, insert(:user))} + end + + test "returns every adaptor with name, latest_version, versions, icon_urls, and repository", + %{conn: conn} do + square_sha = :crypto.hash(:sha256, "square") + + {:ok, _adaptor} = + AdaptorsRepo.upsert_adaptor(%{ + name: "@openfn/language-http", + source: :npm, + latest_version: "2.0.0", + repository: "https://github.com/openfn/language-http", + icon_square_ext: "png", + icon_square_sha256: square_sha, + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil, + versions: [version_record("1.0.0"), version_record("2.0.0")] + }) + + conn = get(conn, ~p"/adaptors/catalogue") + + assert %{"data" => [entry]} = json_response(conn, 200) + + expected_square_url = + AdaptorIconURL.build( + "@openfn/language-http", + %{icon_square_ext: "png", icon_square_sha256: square_sha}, + :square + ) + + assert entry["name"] == "@openfn/language-http" + assert entry["latest_version"] == "2.0.0" + assert entry["repository"] == "https://github.com/openfn/language-http" + assert Enum.sort(entry["versions"]) == ["1.0.0", "2.0.0"] + assert entry["icon_urls"]["square"] == expected_square_url + assert entry["icon_urls"]["rectangle"] == nil + end + + test "a repeat request with a matching If-None-Match returns 304 with no body", + %{ + conn: conn + } do + {:ok, _adaptor} = + AdaptorsRepo.upsert_adaptor(%{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + versions: [version_record("1.0.0")] + }) + + first = get(conn, ~p"/adaptors/catalogue") + [etag] = get_resp_header(first, "etag") + + second = + conn + |> put_req_header("if-none-match", etag) + |> get(~p"/adaptors/catalogue") + + assert response(second, 304) == "" + end + + test "removing a version from an adaptor that doesn't hold the current max changes the ETag", + %{ + conn: conn + } do + {:ok, _b} = + AdaptorsRepo.upsert_adaptor(%{ + name: "@openfn/language-b", + source: :npm, + latest_version: "1.0.0", + versions: [version_record("1.0.0")] + }) + + {:ok, _a} = + AdaptorsRepo.upsert_adaptor(%{ + name: "@openfn/language-a", + source: :npm, + latest_version: "1.0.0", + versions: [version_record("1.0.0")] + }) + + first = get(conn, ~p"/adaptors/catalogue") + [first_etag] = get_resp_header(first, "etag") + + {:ok, _b} = + AdaptorsRepo.upsert_adaptor(%{ + name: "@openfn/language-b", + source: :npm, + latest_version: "1.0.0", + versions: [] + }) + + second = get(conn, ~p"/adaptors/catalogue") + [second_etag] = get_resp_header(second, "etag") + + assert first_etag != second_etag + + third = + conn + |> put_req_header("if-none-match", first_etag) + |> get(~p"/adaptors/catalogue") + + assert response(third, 200) + end + end + + test "an unauthenticated request is rejected with a JSON 401", %{conn: conn} do + conn = get(conn, ~p"/adaptors/catalogue") + + assert json_response(conn, 401) == %{"error" => "Unauthorized"} + end + + defp version_record(version) do + %{ + version: version, + integrity: "sha512-#{version}", + tarball_url: "https://example.com/x/-/x-#{version}.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + end +end diff --git a/test/lightning_web/controllers/adaptor_icon_controller_test.exs b/test/lightning_web/controllers/adaptor_icon_controller_test.exs new file mode 100644 index 00000000000..79c3cb46ff5 --- /dev/null +++ b/test/lightning_web/controllers/adaptor_icon_controller_test.exs @@ -0,0 +1,564 @@ +defmodule LightningWeb.AdaptorIconControllerTest do + # async: false — all tests share the Lightning.Adaptors supervisor name. + use LightningWeb.ConnCase, async: false + + import Mox + + alias Lightning.Adaptors.IconCache + alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + alias LightningWeb.AdaptorIconController + alias LightningWeb.AdaptorIconURL + + setup :verify_on_exit! + + # The production `Lightning.Adaptors.Supervisor` is started in + # `application.ex` under the name `Lightning.Adaptors` and — in test — + # uses `Lightning.Adaptors.StrategyMock` (see `config/test.exs`). No + # per-test supervisor start is needed. + + defp sha8_from_bytes(bytes) do + :crypto.hash(:sha256, bytes) + |> binary_part(0, 4) + |> Base.encode16(case: :lower) + end + + defp source, do: AdaptorsSupervisor.source(Lightning.Adaptors) + + defp unique_adaptor_name do + "@openfn/language-test-#{System.unique_integer([:positive])}" + end + + defp insert_adaptor(name, overrides \\ %{}) do + attrs = + Map.merge( + %{ + name: name, + source: :npm, + latest_version: "1.0.0", + deprecated: false + }, + overrides + ) + + {:ok, _adaptor} = AdaptorsRepo.upsert_adaptor(attrs) + end + + defp write_icon(name, shape, ext, bytes) do + {:ok, _sha} = IconCache.write!(source(), name, shape, ext, bytes) + :ok + end + + describe "show/2 — match + warm disk" do + test "returns 200 with immutable Cache-Control", %{conn: conn} do + name = unique_adaptor_name() + bytes = "png icon bytes" + sha256 = :crypto.hash(:sha256, bytes) + sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "png", + icon_square_sha256: sha256 + }) + + write_icon(name, :square, "png", bytes) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => sha8, + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 200 + + assert get_resp_header(result, "cache-control") == [ + "public, max-age=31536000, immutable" + ] + + assert result.resp_body == bytes + end + + test "serves correct Content-Type for png", %{conn: conn} do + name = unique_adaptor_name() + bytes = "png bytes" + sha256 = :crypto.hash(:sha256, bytes) + sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "png", + icon_square_sha256: sha256 + }) + + write_icon(name, :square, "png", bytes) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => sha8, + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 200 + [ct] = get_resp_header(result, "content-type") + assert ct =~ "image/png" + end + + test "serves correct Content-Type for svg", %{conn: conn} do + name = unique_adaptor_name() + bytes = "" + sha256 = :crypto.hash(:sha256, bytes) + sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "svg", + icon_square_sha256: sha256 + }) + + write_icon(name, :square, "svg", bytes) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => sha8, + "ext" => "svg" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 200 + [ct] = get_resp_header(result, "content-type") + assert ct =~ "image/svg+xml" + end + + test "sha8 is case-insensitive on input", %{conn: conn} do + name = unique_adaptor_name() + bytes = "case test bytes" + sha256 = :crypto.hash(:sha256, bytes) + sha8_lower = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + sha8_upper = String.upcase(sha8_lower) + + insert_adaptor(name, %{ + icon_square_ext: "png", + icon_square_sha256: sha256 + }) + + write_icon(name, :square, "png", bytes) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => sha8_upper, + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 200 + end + + test "works for rectangle shape", %{conn: conn} do + name = unique_adaptor_name() + bytes = "rect bytes" + sha256 = :crypto.hash(:sha256, bytes) + sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_rectangle_ext: "png", + icon_rectangle_sha256: sha256 + }) + + write_icon(name, :rectangle, "png", bytes) + + params = %{ + "name" => name, + "shape" => "rectangle", + "sha8" => sha8, + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 200 + assert result.resp_body == bytes + end + end + + describe "show/2 — match + cold disk" do + test "strategy is called, bytes written to disk, 200 returned", %{conn: conn} do + name = unique_adaptor_name() + bytes = "cold icon bytes" + sha256 = :crypto.hash(:sha256, bytes) + sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "png", + icon_square_sha256: sha256 + }) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_icon, + 1, + fn ^name, :square -> {:ok, %{data: bytes, ext: "png"}} end + ) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => sha8, + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 200 + + assert get_resp_header(result, "cache-control") == [ + "public, max-age=31536000, immutable" + ] + + assert result.resp_body == bytes + end + end + + describe "show/2 — stale sha8" do + test "redirects 302 with no-store and current Location", %{conn: conn} do + name = unique_adaptor_name() + bytes = "current icon bytes" + sha256 = :crypto.hash(:sha256, bytes) + current_sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "png", + icon_square_sha256: sha256 + }) + + stale_sha8 = "00000000" + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => stale_sha8, + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 302 + assert get_resp_header(result, "cache-control") == ["no-store"] + + encoded_name = URI.encode(name, &URI.char_unreserved?/1) + + assert get_resp_header(result, "location") == [ + "/adaptors/icons/#{encoded_name}/square-#{current_sha8}.png" + ] + end + + test "Location URL is lowercase even when sha8 input was uppercase", %{ + conn: conn + } do + name = unique_adaptor_name() + bytes = "case url bytes" + sha256 = :crypto.hash(:sha256, bytes) + current_sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "png", + icon_square_sha256: sha256 + }) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => "FFFFFFFF", + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 302 + [location] = get_resp_header(result, "location") + + # sha8 segment is lowercase; percent-encoded chars use uppercase hex per RFC 3986 + assert location =~ "square-#{current_sha8}.png" + end + + test "Location matches what AdaptorIconURL.build/3 would emit", %{conn: conn} do + name = unique_adaptor_name() + bytes = "channel sync bytes" + sha256 = :crypto.hash(:sha256, bytes) + + insert_adaptor(name, %{ + icon_square_ext: "png", + icon_square_sha256: sha256 + }) + + meta = %{ + icon_square_ext: "png", + icon_square_sha256: sha256, + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil + } + + expected_url = AdaptorIconURL.build(name, meta, :square) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => "00000000", + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 302 + assert get_resp_header(result, "location") == [expected_url] + end + end + + describe "show/2 — 404" do + test "adaptor not in DB", %{conn: conn} do + params = %{ + "name" => "nonexistent-adaptor-#{System.unique_integer([:positive])}", + "shape" => "square", + "sha8" => "aabbccdd", + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 404 + end + + test "ext mismatch — DB has png, URL says svg", %{conn: conn} do + name = unique_adaptor_name() + sha256 = :crypto.hash(:sha256, "some bytes") + sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "png", + icon_square_sha256: sha256 + }) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => sha8, + "ext" => "svg" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 404 + end + + test "ext mismatch — DB has svg, URL says png", %{conn: conn} do + name = unique_adaptor_name() + sha256 = :crypto.hash(:sha256, "") + sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "svg", + icon_square_sha256: sha256 + }) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => sha8, + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 404 + end + + test "stored ext is nil (no icon for shape)", %{conn: conn} do + name = unique_adaptor_name() + insert_adaptor(name) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => "aabbccdd", + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 404 + end + + test "bad shape value — not square or rectangle", %{conn: conn} do + params = %{ + "name" => unique_adaptor_name(), + "shape" => "circle", + "sha8" => "aabbccdd", + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 404 + end + + test "missing params — fallback clause", %{conn: conn} do + result = AdaptorIconController.show(conn, %{"name" => "something"}) + + assert result.status == 404 + end + + test "stale sha but stored ext is nil (icon removed upstream) — no redirect", + %{ + conn: conn + } do + # Adaptor row exists but has no icon for the square shape. + # Even though there's a stale sha8 in the URL, there's no canonical + # URL to redirect to — 404 instead of 302. + name = unique_adaptor_name() + insert_adaptor(name) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => "00000000", + "ext" => "png" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 404 + end + end + + # The tests above call the controller directly; these confirm the route + # and pipeline wire up to it too. + describe "GET /adaptors/icons/... (full router pipeline)" do + test "200 on sha match", %{conn: conn} do + name = unique_adaptor_name() + bytes = "router pipeline bytes" + sha256 = :crypto.hash(:sha256, bytes) + sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "png", + icon_square_sha256: sha256 + }) + + write_icon(name, :square, "png", bytes) + + encoded = URI.encode(name, &URI.char_unreserved?/1) + conn = get(conn, "/adaptors/icons/#{encoded}/square-#{sha8}.png") + + assert conn.status == 200 + assert conn.resp_body == bytes + end + + test "404 on unknown adaptor", %{conn: conn} do + conn = get(conn, "/adaptors/icons/nope/square-aabbccdd.png") + + assert conn.status == 404 + end + end + + describe "AdaptorIconURL.build/3" do + test "returns nil when ext is nil" do + meta = %{ + icon_square_ext: nil, + icon_square_sha256: :crypto.hash(:sha256, "x"), + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil + } + + assert AdaptorIconURL.build("@openfn/language-http", meta, :square) == nil + end + + test "returns nil when sha256 is nil" do + meta = %{ + icon_square_ext: "png", + icon_square_sha256: nil, + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil + } + + assert AdaptorIconURL.build("@openfn/language-http", meta, :square) == nil + end + + test "URL-encodes slashes and @ in adaptor name" do + sha256 = :crypto.hash(:sha256, "bytes") + + meta = %{ + icon_square_ext: "png", + icon_square_sha256: sha256, + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil + } + + url = AdaptorIconURL.build("@openfn/language-http", meta, :square) + + refute is_nil(url) + assert url =~ "%40openfn%2Flanguage-http" + end + + test "sha8 in URL is always 8 lowercase hex chars" do + sha256 = :crypto.hash(:sha256, "bytes") + expected_sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + meta = %{ + icon_square_ext: "png", + icon_square_sha256: sha256, + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil + } + + url = AdaptorIconURL.build("@openfn/language-http", meta, :square) + + # sha8 is lowercase; percent-encoded chars use uppercase hex per RFC 3986 + assert url =~ "square-#{expected_sha8}.png" + assert expected_sha8 == String.downcase(expected_sha8) + end + + test "builds distinct URLs for square and rectangle shapes" do + sq_sha = :crypto.hash(:sha256, "square bytes") + rect_sha = :crypto.hash(:sha256, "rectangle bytes") + + meta = %{ + icon_square_ext: "png", + icon_square_sha256: sq_sha, + icon_rectangle_ext: "svg", + icon_rectangle_sha256: rect_sha + } + + sq_url = AdaptorIconURL.build("@openfn/language-http", meta, :square) + rect_url = AdaptorIconURL.build("@openfn/language-http", meta, :rectangle) + + assert sq_url =~ "square-" + assert rect_url =~ "rectangle-" + assert sq_url =~ ".png" + assert rect_url =~ ".svg" + refute sq_url == rect_url + end + + test "sha8 matches what show/2 computes from the same sha256" do + bytes = "verify sha8 computation" + sha256 = :crypto.hash(:sha256, bytes) + + meta = %{ + icon_square_ext: "png", + icon_square_sha256: sha256, + icon_rectangle_ext: nil, + icon_rectangle_sha256: nil + } + + url = AdaptorIconURL.build("name", meta, :square) + + assert sha8_from_bytes(bytes) in String.split(url, ["-", "."]) + end + end +end diff --git a/test/lightning_web/live/credential_live_test.exs b/test/lightning_web/live/credential_live_test.exs index b5237cba088..cc89c3730d0 100644 --- a/test/lightning_web/live/credential_live_test.exs +++ b/test/lightning_web/live/credential_live_test.exs @@ -43,6 +43,13 @@ defmodule LightningWeb.CredentialLiveTest do setup :register_and_log_in_user setup :create_project_for_current_user + # `Credentials.get_schema/1` reads through `Lightning.Adaptors.Repo`; + # seed schemas there so it doesn't fall through to the Strategy mock. + setup do + Lightning.AdaptorTestHelpers.seed_all_credential_schemas() + :ok + end + defp get_decoded_state(url) when is_nil(url) do [ "test", diff --git a/test/lightning_web/live/maintenance_live/index_test.exs b/test/lightning_web/live/maintenance_live/index_test.exs new file mode 100644 index 00000000000..68c31bac8a3 --- /dev/null +++ b/test/lightning_web/live/maintenance_live/index_test.exs @@ -0,0 +1,79 @@ +defmodule LightningWeb.MaintenanceLive.IndexTest do + # async: false because the icon-refresh test stubs Lightning.Adaptors.StrategyMock + # globally, which the singleton Scheduler GenServer (not in the test pid's + # caller chain) needs to see. + use LightningWeb.ConnCase, async: false + + import Mox + import Phoenix.LiveViewTest + + setup :set_mox_global + + describe "Index as a regular user" do + setup :register_and_log_in_user + + test "cannot access the maintenance page", %{conn: conn} do + {:ok, _live, html} = + live(conn, ~p"/settings/maintenance", on_error: :raise) + |> follow_redirect(conn, "/projects") + + assert html =~ "Sorry, you don't have access to that." + end + end + + describe "Index as a superuser" do + setup :register_and_log_in_superuser + + test "renders the Refresh Adaptor Registry card", %{conn: conn} do + {:ok, _live, html} = + live(conn, ~p"/settings/maintenance", on_error: :raise) + + assert html =~ "Maintenance" + assert html =~ "Refresh Adaptor Registry" + assert html =~ "Re-fetch the list of available adaptors" + assert html =~ "Run" + end + + test "clicking Run flashes that the refresh was queued", %{conn: conn} do + {:ok, live, _html} = + live(conn, ~p"/settings/maintenance", on_error: :raise) + + live + |> element("#refresh-adaptors-button") + |> render_click() + + assert has_element?( + live, + "p[role=alert][phx-value-key=info]", + "Adaptor refresh queued." + ) + end + + test "renders the Refresh Adaptor Icons card", %{conn: conn} do + {:ok, live, html} = + live(conn, ~p"/settings/maintenance", on_error: :raise) + + assert html =~ "Refresh Adaptor Icons" + assert has_element?(live, "#refresh-icons-button") + end + + test "clicking the icons button reports the refresh result", %{conn: conn} do + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + {:ok, live, _html} = + live(conn, ~p"/settings/maintenance", on_error: :raise) + + live + |> element("#refresh-icons-button") + |> render_click() + + assert has_element?( + live, + "p[role=alert][phx-value-key=info]", + "Icon refresh complete" + ) + end + end +end diff --git a/test/lightning_web/live/project_live_test.exs b/test/lightning_web/live/project_live_test.exs index 6e8460cb0ce..35a90def6db 100644 --- a/test/lightning_web/live/project_live_test.exs +++ b/test/lightning_web/live/project_live_test.exs @@ -880,6 +880,14 @@ defmodule LightningWeb.ProjectLiveTest do setup :register_and_log_in_user setup :create_project_for_current_user + setup do + # Credential creation flows render the JsonSchemaBodyComponent, which + # calls `Credentials.get_schema/1` and so reads through + # `Lightning.Adaptors.Repo`. + Lightning.AdaptorTestHelpers.seed_credential_schema("http") + :ok + end + test "access project settings page", %{conn: conn, project: project} do {:ok, _view, html} = live(conn, ~p"/projects/#{project}/settings", on_error: :raise) diff --git a/test/lightning_web/live/workflow_live/collaborate_test.exs b/test/lightning_web/live/workflow_live/collaborate_test.exs index 28cfa1cae8d..4ea41f8a096 100644 --- a/test/lightning_web/live/workflow_live/collaborate_test.exs +++ b/test/lightning_web/live/workflow_live/collaborate_test.exs @@ -1028,6 +1028,14 @@ defmodule LightningWeb.WorkflowLive.CollaborateTest do end describe "credential modal interactions" do + # `Credentials.get_schema/1` reads through `Lightning.Adaptors.Repo`; + # seed the `http` fixture so the JsonSchemaBodyComponent renders + # without raising. + setup do + Lightning.AdaptorTestHelpers.seed_credential_schema("http") + :ok + end + test "opens credential modal with schema via handle_event", %{conn: conn} do user = insert(:user) diff --git a/test/mix/tasks/lightning.refresh_adaptors_test.exs b/test/mix/tasks/lightning.refresh_adaptors_test.exs new file mode 100644 index 00000000000..ae46ebc60a2 --- /dev/null +++ b/test/mix/tasks/lightning.refresh_adaptors_test.exs @@ -0,0 +1,85 @@ +defmodule Mix.Tasks.Lightning.RefreshAdaptorsTest do + use ExUnit.Case, async: false + use Mimic + + setup_all do + Mimic.copy(Lightning.Adaptors) + :ok + end + + setup do + Mix.shell(Mix.Shell.Process) + on_exit(fn -> Mix.shell(Mix.Shell.IO) end) + :ok + end + + describe "bare invocation" do + test "calls refresh_now/0 and exits 0 on :ok" do + stub(Lightning.Adaptors, :refresh_now, fn -> :ok end) + Mix.Tasks.Lightning.RefreshAdaptors.run([]) + assert_received {:mix_shell, :info, [_]} + end + + test "exits 2 on other error" do + stub(Lightning.Adaptors, :refresh_now, fn -> {:error, :network_down} end) + + assert catch_exit(Mix.Tasks.Lightning.RefreshAdaptors.run([])) == + {:shutdown, 2} + + assert_received {:mix_shell, :error, [_]} + end + end + + describe "--name flag" do + test "dispatches to refresh_package/1 with the exact package string" do + pkg = "@openfn/language-http" + stub(Lightning.Adaptors, :refresh_package, fn ^pkg -> :ok end) + Mix.Tasks.Lightning.RefreshAdaptors.run(["--name", pkg]) + assert_received {:mix_shell, :info, [_]} + end + + test "exits 1 on {:error, :not_found}" do + stub(Lightning.Adaptors, :refresh_package, fn _pkg -> + {:error, :not_found} + end) + + assert catch_exit( + Mix.Tasks.Lightning.RefreshAdaptors.run([ + "--name", + "@openfn/language-http" + ]) + ) == {:shutdown, 1} + + assert_received {:mix_shell, :error, [_]} + end + + test "exits 2 on other error" do + stub(Lightning.Adaptors, :refresh_package, fn _pkg -> + {:error, :timeout} + end) + + assert catch_exit( + Mix.Tasks.Lightning.RefreshAdaptors.run([ + "--name", + "@openfn/language-http" + ]) + ) == {:shutdown, 2} + + assert_received {:mix_shell, :error, [_]} + end + end + + describe "rejected flags" do + test "raises on unknown --strategy flag" do + assert_raise OptionParser.ParseError, fn -> + Mix.Tasks.Lightning.RefreshAdaptors.run(["--strategy", "local"]) + end + end + + test "raises on unknown --source flag" do + assert_raise OptionParser.ParseError, fn -> + Mix.Tasks.Lightning.RefreshAdaptors.run(["--source", "local"]) + end + end + end +end diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex new file mode 100644 index 00000000000..63bfd6a7f3b --- /dev/null +++ b/test/support/adaptor_test_helpers.ex @@ -0,0 +1,253 @@ +defmodule Lightning.AdaptorTestHelpers do + @moduledoc """ + Seeds `Lightning.Adaptors.Repo` and clears the global + `Lightning.Adaptors.Supervisor` Cachex. + + The production `Lightning.Adaptors` supervisor starts with the + application and is shared across the test suite: its Cachex persists + across the `Ecto.Adapters.SQL.Sandbox` boundary, so tests that seed + rows via the `:adaptor` factory (`insert(:adaptor, attrs)`) must + clear the cache to make them visible to facade reads. + + For tests that run their own isolated supervisor instead of the + production one, see `test/lightning/adaptors_test.exs` and + `test/lightning/adaptors/store_test.exs`. + """ + + import Lightning.Factories + + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + @doc """ + Clear the production `Lightning.Adaptors` Cachex so subsequent reads + fall back through the DB. + """ + @spec clear_global_adaptors_cache() :: :ok + def clear_global_adaptors_cache do + cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) + Cachex.clear(cache) + :ok + end + + @doc """ + Insert one `Adaptors.Repo.Adaptor` row using the factory and clear + the global Cachex so it's immediately visible to facade reads. + + `attrs` is forwarded to the `:adaptor` factory verbatim. + """ + @spec seed_adaptor(keyword() | map()) :: Lightning.Adaptors.Repo.Adaptor.t() + def seed_adaptor(attrs \\ []) do + row = insert(:adaptor, attrs) + clear_global_adaptors_cache() + row + end + + @doc """ + Seed a credential schema row keyed by short name (e.g. `"postgresql"`), + reading the JSON body from `test/fixtures/schemas/.json`. + + `Credentials.get_schema/1` reads schemas from the adaptor registry, + not directly from disk, so tests exercising it must seed them here. + """ + @spec seed_credential_schema(String.t()) :: + Lightning.Adaptors.Repo.Adaptor.t() + def seed_credential_schema(short_name) when is_binary(short_name) do + # Keep the raw JSON binary (not a decoded map) so + # `Lightning.Credentials.Schema.new/2` can decode it downstream with + # `Jason.decode!(_, objects: :ordered_objects)` and preserve field order. + schema_body = + Path.join(["test", "fixtures", "schemas", "#{short_name}.json"]) + |> File.read!() + + row = + insert(:adaptor, name: short_name, source: :npm, schema_data: schema_body) + + # Cachex's fallback runs in the Courier process — it can't see the + # test-owned sandbox connection. Pre-populate the cache so reads + # never need to fall through to a DB lookup from the Courier. + cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) + source = AdaptorsSupervisor.source(Lightning.Adaptors) + Cachex.put(cache, {:schema, short_name, source}, {:ok, schema_body}) + + row + end + + @doc """ + Seed every credential schema present in `test/fixtures/schemas/`. + + Use from a `setup` block in tests that exercise multiple credential + types (e.g. `LightningWeb.CredentialLiveTest`). + """ + @spec seed_all_credential_schemas() :: :ok + def seed_all_credential_schemas do + Path.wildcard("test/fixtures/schemas/*.json") + |> Enum.each(fn path -> + # Skip empty fixture files — some (e.g. `asana.json`, + # `primero.json`) are intentional empty placeholders. + if File.stat!(path).size > 0 do + short_name = path |> Path.basename(".json") + seed_credential_schema(short_name) + end + end) + + :ok + end + + @doc """ + Seed an `@openfn/*` adaptor package with a concrete `latest_version` + so `Lightning.Adaptors.PackageName.to_wire/1` resolves `@latest` + correctly. + """ + @spec seed_adaptor_package(String.t(), String.t() | [String.t()]) :: + Lightning.Adaptors.Repo.Adaptor.t() + def seed_adaptor_package(name, versions) + when is_binary(name) and is_list(versions) do + # The first version in the list is treated as latest. + [latest | _] = versions + + {:ok, row} = + Lightning.Adaptors.Repo.upsert_adaptor(%{ + name: name, + source: :npm, + latest_version: latest, + description: nil, + homepage: nil, + repository: nil, + license: nil, + deprecated: false, + schema_data: nil, + schema_sha256: nil, + versions: + Enum.map(versions, fn v -> + %{ + version: v, + integrity: "sha512-#{v}", + tarball_url: "https://example.com/x-#{v}.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + end) + }) + + row + end + + def seed_adaptor_package(name, latest_version) + when is_binary(name) and is_binary(latest_version) do + seed_adaptor_package(name, [latest_version]) + end + + @doc """ + Build a record matching `t:Lightning.Adaptors.Strategy.adaptor_record/0` + for use in `Mox.stub`/`Mox.expect` setups. + """ + @spec build_strategy_adaptor_record(String.t(), String.t()) :: map() + def build_strategy_adaptor_record(name, latest_version) do + %{ + name: name, + source: :npm, + latest_version: latest_version, + description: nil, + homepage: nil, + repository: nil, + license: nil, + deprecated: false, + schema_data: nil, + schema_sha256: nil, + versions: [ + %{ + version: latest_version, + integrity: "sha512-#{latest_version}", + tarball_url: "https://example.com/x-#{latest_version}.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + ] + } + end + + @doc """ + Pre-populate the production `Lightning.Adaptors` supervisor's Cachex + with a packages map, so async tests get seeded data without the + Cachex Courier process falling through to a DB query it can't see + (the sandboxed connection is invisible to it). + + Returns the supervisor source atom for convenience. + """ + @spec warm_packages_cache([map()]) :: :npm | :local + def warm_packages_cache(metas) when is_list(metas) do + cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) + source = AdaptorsSupervisor.source(Lightning.Adaptors) + + Cachex.put(cache, {:packages, source}, {:ok, metas}) + + source + end + + @doc """ + Bulk-seed the common `@openfn/*` packages, with the versions + expected by tests across the suite. + """ + @spec seed_common_packages() :: :ok + def seed_common_packages do + packages = [ + {"@openfn/language-common", ["1.6.2", "1.2.22", "1.1.0"]}, + {"@openfn/language-http", ["3.1.12", "2.0.0", "1.0.0"]}, + {"@openfn/language-postgresql", ["3.2.0", "2.0.0", "1.0.0"]}, + {"@openfn/language-dhis2", ["3.0.4", "2.0.0", "1.0.0"]}, + {"@openfn/language-salesforce", ["3.0.0", "2.0.0", "1.0.0"]}, + {"@openfn/language-godata", ["2.0.0", "1.0.0"]}, + {"@openfn/language-googlesheets", ["2.0.0", "1.0.0"]} + ] + + Enum.each(packages, fn {name, versions} -> + seed_adaptor_package(name, versions) + end) + + # Warm Cachex for the production supervisor so reads from any + # process (including the LiveView's caller chain) see the seeded + # data without falling through to the Cachex Courier's + # sandbox-blind DB query. + cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) + source = AdaptorsSupervisor.source(Lightning.Adaptors) + + metas = + Enum.map(packages, fn {name, [latest | _]} -> + %{ + name: name, + latest_version: latest, + description: nil, + deprecated: false, + icon_square_ext: nil, + icon_rectangle_ext: nil, + icon_square_sha256: nil, + icon_rectangle_sha256: nil + } + end) + + Cachex.put(cache, {:packages, source}, {:ok, metas}) + + Enum.each(packages, fn {name, versions} -> + version_metas = + Enum.map(versions, fn v -> + %{ + version: v, + integrity: "sha512-#{v}", + size_bytes: 1024, + published_at: nil, + deprecated: false + } + end) + + Cachex.put(cache, {:versions, name, source}, {:ok, version_metas}) + end) + + :ok + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index c482a2e4b9a..9f0c8398d3c 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -73,14 +73,11 @@ Application.put_env(:lightning, Lightning.Extensions, ) # Pin the `Lightning.Adaptors.IconCache` on-disk path to a per-OS-PID -# directory and wipe it at startup so: -# 1. Each `mix test` invocation begins with an empty icon cache — -# `System.unique_integer/1` resets per-VM and recycles, so without -# this, leftover files from a prior run can mask a Mox expectation -# by short-circuiting `IconCache.cached?/4`. -# 2. Concurrent `mix test` invocations (different tmux panes, parallel -# CI shards) use distinct directories and never collide — each BEAM -# has its own OS PID. +# directory and wipe it at startup. Without the wipe, leftover files +# from a prior run can mask a Mox expectation by short-circuiting +# `IconCache.cached?/4`, since `System.unique_integer/1` resets per-VM +# and recycles. Keying by OS PID also keeps concurrent `mix test` runs +# (parallel CI shards, separate tmux panes) from colliding. icon_dir = Path.join([ System.tmp_dir!(), diff --git a/tooling/adaptor_cache/README.md b/tooling/adaptor_cache/README.md new file mode 100644 index 00000000000..d65eb29d57d --- /dev/null +++ b/tooling/adaptor_cache/README.md @@ -0,0 +1,136 @@ +# Adaptor Cache + +> A local caching reverse proxy sitting in front of the three upstreams +> `Lightning.Adaptors.*` reads from. + +Every `Lightning.Adaptors.Scheduler` refresh tick makes one npm `/-/v1/search` +call, then one packument call and one jsDelivr schema call per changed package, +plus up to four `raw.githubusercontent.com` calls per package for icons (two +shapes x the png-then-svg fallback). Iterating on the subsystem means running +that loop over and over against the real internet. This proxy caches all of it +to disk so the second and every later run is local, and the whole thing works +offline (on a plane, on bad wifi, wherever). + +Driven by `bin/adaptor_cache` from the repo root; see that script's `--help` for +the full command list. + +## What's in this folder + +| File | Purpose | +| -------------------- | -------------------------------------------------------------------------------------------- | +| `docker-compose.yml` | Runs the `nginx` container, bound to `127.0.0.1` only | +| `nginx.conf` | The three `/npm/`, `/jsdelivr/`, `/github/` proxy locations and the persistent on-disk cache | + +## Prerequisites + +- Docker + Docker Compose +- **Bring the cache up at least once while online.** nginx resolves + `registry.npmjs.org`, `cdn.jsdelivr.net` and `raw.githubusercontent.com` at + container startup, not per-request. Starting it offline for the first time + fails with `host not found in upstream` — do the first `bin/adaptor_cache up` + with a real connection, after that it's fine offline. + +## Environment variables + +Point Lightning at the cache by exporting these: + +```sh +export ADAPTOR_REGISTRY_URL=http://localhost:4874/npm +export ADAPTOR_JSDELIVR_URL=http://localhost:4874/jsdelivr +export ADAPTOR_GITHUB_URL=http://localhost:4874/github +``` + +`bin/adaptor_cache up` prints these for you with the right port baked in, so you +don't have to remember them. + +- `ADAPTOR_CACHE_PORT` — host port to bind (default: `4874`). Set it before any + `bin/adaptor_cache` command if `4874` is taken, and update the three exports + above to match. + +## Usage + +```sh +bin/adaptor_cache up # start the proxy and print the export lines +bin/adaptor_cache down # stop the proxy, keeping the cache on disk +bin/adaptor_cache status # show container state and reachability +bin/adaptor_cache purge # stop the proxy AND drop the cache volume +bin/adaptor_cache logs # tail the access log (cache=HIT / cache=MISS) +bin/adaptor_cache check # probe all three prefixes, prove MISS then HIT +bin/adaptor_cache --help # full usage +``` + +With the cache up and the three vars exported, run +`mix lightning.refresh_adaptors` as usual. The first run populates the cache; +every run after that should be fast and work with no network at all. + +### Reading `bin/adaptor_cache logs` + +Each line is one proxied request: + +``` +2026-08-25T10:00:00+00:00 status=200 cache=HIT GET /npm/-/v1/search?text=@openfn&size=250 +``` + +- `cache=HIT` — served entirely from disk, no upstream request made. +- `cache=MISS` — not in the cache (or expired), fetched from the real upstream + and stored. +- `cache=EXPIRED` / `cache=REVALIDATED` — the entry had aged out, so nginx did + go upstream. REVALIDATED means it sent a conditional request, got a 304, and + reused what was on disk. +- `cache=STALE` / `cache=UPDATING` — nginx served what it had on disk because + the upstream was unreachable, or because another request was already + refetching. This is what you see offline. + +On a warm cache, MISS should only appear for packages the cache has never seen. + +## How the URL mapping works + +The proxy has one location block per upstream, and Lightning's `NPM` strategy +already builds full paths under each — the proxy just rewrites the host: + +| Lightning request | Through the proxy | Prefix | Real upstream | +| -------------------------------------- | ------------------------------------------------------------------------------------------ | ------------ | --------------------------------------- | +| npm search / packument (`registry.ex`) | `http://localhost:4874/npm/-/v1/search?...` | `/npm/` | `https://registry.npmjs.org/...` | +| jsDelivr schema fetch (`schema.ex`) | `http://localhost:4874/jsdelivr/npm/@openfn/language-http@2.1.0/configuration-schema.json` | `/jsdelivr/` | `https://cdn.jsdelivr.net/...` | +| GitHub icon fetch (`github.ex`) | `http://localhost:4874/github/OpenFn/adaptors/main/packages/http/assets/square.png` | `/github/` | `https://raw.githubusercontent.com/...` | + +Setting `ADAPTOR_REGISTRY_URL`, `ADAPTOR_JSDELIVR_URL` and `ADAPTOR_GITHUB_URL` +to the proxy's `/npm`, `/jsdelivr` and `/github` base URLs is all that's needed +— the strategy code appends the same paths it always did, just against a +different host. + +## Caveats + +- **The legacy `Lightning.AdaptorRegistry` and `mix lightning.install_schemas` + bypass this entirely.** Both have hardcoded upstream URLs and don't read the + `ADAPTOR_*` env vars, so they'll always hit the real internet regardless of + whether the cache is up. +- **Redirects bypass the cache.** Tesla's `FollowRedirects` middleware requests + the absolute `Location` URL, which points at the real upstream even when the + redirect is same-host, so anything that 30x's is fetched live. +- **Never set this as your global npm registry in `~/.npmrc`.** The `/npm/` + prefix is a transparent GET proxy of registry.npmjs.org, so npm would mostly + work, badly: this cache ignores Cache-Control and holds 200s for seven days, + so `npm install` resolves against a week-stale packument, and npm records the + registry it fetched from in `package-lock.json`'s `resolved` URLs, giving you + a lockfile that only installs on a machine running this container. + +## Troubleshooting + +**`host not found in upstream` on startup.** You brought the container up +offline for the first time. Get online and run `bin/adaptor_cache up` once so +nginx can resolve the three upstream hostnames, then it's fine offline after +that. + +**`bin/adaptor_cache check` fails on one prefix.** Run `bin/adaptor_cache logs` +and look for the failing request — a `cache=MISS` on the _second_ identical +request usually means the upstream is refusing the request outright (check +status code) rather than a caching problem. + +**Port already in use.** Set `ADAPTOR_CACHE_PORT` to something else before +`bin/adaptor_cache up`, and update the three `ADAPTOR_*_URL` exports to match +the new port. + +**Stale or wrong data cached.** `bin/adaptor_cache purge` drops the on-disk +cache volume entirely (unlike `down`, which keeps it); `up` again to rebuild +from empty. diff --git a/tooling/adaptor_cache/docker-compose.yml b/tooling/adaptor_cache/docker-compose.yml new file mode 100644 index 00000000000..c5ea519c754 --- /dev/null +++ b/tooling/adaptor_cache/docker-compose.yml @@ -0,0 +1,25 @@ +# Caching reverse proxy for the Lightning.Adaptors.* upstreams. +# +# Driven by bin/adaptor_cache; see README.md in this directory for the why and +# the env vars to export. +services: + nginx: + image: nginx:1.27.4-alpine + container_name: adaptor-cache + ports: + - '127.0.0.1:${ADAPTOR_CACHE_PORT:-4874}:80' + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + # Named volume, so `docker compose down` keeps the cache; only + # `bin/adaptor_cache purge` drops it. + - adaptor-cache-data:/var/cache/nginx/adaptors + restart: unless-stopped + healthcheck: + test: ['CMD', 'wget', '-q', '-O', '-', 'http://127.0.0.1/_healthz'] + interval: 5s + timeout: 3s + retries: 12 + start_period: 2s + +volumes: + adaptor-cache-data: {} diff --git a/tooling/adaptor_cache/nginx.conf b/tooling/adaptor_cache/nginx.conf new file mode 100644 index 00000000000..b38d46210f7 --- /dev/null +++ b/tooling/adaptor_cache/nginx.conf @@ -0,0 +1,151 @@ +# Caching reverse proxy for the three Lightning.Adaptors.* upstreams. +# +# Point Lightning at it with: +# export ADAPTOR_REGISTRY_URL=http://localhost:4874/npm +# export ADAPTOR_JSDELIVR_URL=http://localhost:4874/jsdelivr +# export ADAPTOR_GITHUB_URL=http://localhost:4874/github +# +# Workers run as root deliberately. The cache lives on a Docker named volume +# mounted at /var/cache/nginx/adaptors, which Docker creates root-owned; the +# image's default `nginx` worker user could not write into it and every request +# would log a permission error and bypass the cache. This is a local dev tool +# bound to 127.0.0.1, so root workers are an acceptable trade for not needing +# an entrypoint chown. +user root; +worker_processes 1; +error_log /dev/stderr warn; +pid /var/run/nginx.pid; + +events { + worker_connections 256; +} + +http { + default_type application/octet-stream; + + # $upstream_cache_status says whether a request was answered from disk (HIT) + # or went out to the upstream (MISS, EXPIRED, REVALIDATED). + # `bin/adaptor_cache check` parses it. + log_format cache '$time_iso8601 status=$status cache=$upstream_cache_status ' + '$request_method $request_uri'; + access_log /dev/stdout cache; + + sendfile on; + tcp_nopush on; + keepalive_timeout 65; + + # Generous on purpose: nothing should be evicted mid-session, and the whole + # @openfn/language-* corpus is a few hundred MB at worst. + proxy_cache_path /var/cache/nginx/adaptors + levels=1:2 + keys_zone=adaptors:64m + max_size=20g + inactive=30d + use_temp_path=off; + + proxy_http_version 1.1; + + # All three upstreams are SNI-dependent CDNs. Without proxy_ssl_server_name + # the TLS handshake goes out with no server name and the upstream serves a + # default cert or refuses outright. proxy_ssl_name defaults to $proxy_host, + # which is the right value here. + proxy_ssl_server_name on; + + proxy_cache adaptors; + + # Cache aggressively regardless of upstream freshness headers. npm sends + # short max-age values and Set-Cookie; jsDelivr varies on Accept-Encoding. + # This is a dev cache, so serving a stale body is fine. + proxy_ignore_headers Cache-Control Expires Set-Cookie Vary; + + proxy_cache_valid 200 301 302 7d; + # 404s are frequent and expected: NPM.GitHub tries .png before + # .svg, so every SVG-only adaptor produces a 404 on the png attempt. + # Cache them so the fan-out is cheap, but briefly, so a newly-added icon + # shows up the same day. + proxy_cache_valid 404 10m; + proxy_cache_valid any 1m; + + # One request per cache key reaches the upstream; the rest wait on it. + # Without this, NPM.GitHub's icon fan-out opens as many upstream connections + # as its max concurrency allows on a cold cache. + proxy_cache_lock on; + proxy_cache_lock_timeout 30s; + + # Offline: serve whatever is on disk when the upstream cannot be reached. + proxy_cache_use_stale error timeout invalid_header updating + http_429 http_500 http_502 http_503 http_504; + proxy_cache_background_update on; + proxy_cache_revalidate on; + + # `always` matters for the 404s: add_header's default status list covers 2xx + # and 3xx but not 404, and cached 404s are routine here because of the + # png-then-svg icon fallback. + add_header X-Cache-Status $upstream_cache_status always; + add_header X-Adaptor-Cache-Upstream $proxy_host always; + + proxy_connect_timeout 10s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + + server { + listen 80; + server_name _; + + location = /_healthz { + access_log off; + return 200 "ok\n"; + } + + # nginx does not merge proxy_set_header across configuration levels: the + # moment a location declares one, every proxy_set_header from an enclosing + # level is dropped. So each location below repeats the same three headers + # rather than inheriting them, and Accept-Encoding must not be hoisted. + # + # Accept-Encoding is pinned to identity because Lightning's Tesla clients + # have no decompress middleware and NPM.Schema takes sha256 over the raw + # response bytes. If a gzip-capable client (curl --compressed) populated + # the cache, replaying that entry to Lightning would give it the wrong + # bytes and the wrong schema_sha256. Identity keeps every cached body + # byte-identical for every client. + + # GET /npm/-/v1/search?text=@openfn&size=250 + # GET /npm/@openfn/language-http + location /npm/ { + proxy_set_header Host registry.npmjs.org; + proxy_set_header Connection close; + proxy_set_header Accept-Encoding ""; + # A literal hostname (not a variable) so nginx resolves it once at + # startup from the container's resolver. Using a variable would require + # a `resolver` directive and runtime DNS. + proxy_pass https://registry.npmjs.org/; + } + + # GET /jsdelivr/npm/@/configuration-schema.json + location /jsdelivr/ { + proxy_set_header Host cdn.jsdelivr.net; + proxy_set_header Connection close; + proxy_set_header Accept-Encoding ""; + proxy_pass https://cdn.jsdelivr.net/; + } + + # GET /github/OpenFn/adaptors//packages//assets/. + # + # The conditional-GET path still works through the cache. NPM.GitHub sends + # if-none-match from the etag it stored last time and treats a 304 as + # :not_modified. nginx caches the upstream 200 along with its ETag, and on + # a hit its not-modified filter compares the client's If-None-Match against + # that ETag and downgrades the 200 to a 304 itself. There is no upstream + # hop, and Lightning's etag bookkeeping sees what GitHub would have sent. + location /github/ { + proxy_set_header Host raw.githubusercontent.com; + proxy_set_header Connection close; + proxy_set_header Accept-Encoding ""; + proxy_pass https://raw.githubusercontent.com/; + } + + location / { + return 404 "adaptor_cache: use /npm/, /jsdelivr/ or /github/\n"; + } + } +} From 7c0c719b50d5158e03b32cd01a28df7e5152b0a2 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Fri, 28 Aug 2026 19:37:45 +0200 Subject: [PATCH 03/37] Cut over to the Lightning.Adaptors facade, delete the old registry - Local strategy supports multiple root directories - Lightning.Adaptors gets a public facade, closing Job's direct escape hatch - ADAPTORS_* env vars wired into bootstrap with back-compat, NPM defaults no longer duplicated - Old AdaptorRegistry deleted - Pre-existing lint/dialyzer failures on the branch fixed --- .env.example | 46 +- DEPLOYMENT.md | 1 - RUNNINGLOCAL.md | 6 +- assets/js/workflow-diagram/useAdaptorIcons.ts | 41 -- .../components/AdaptorSelectionModal.test.tsx | 5 - .../components/ConfigureAdaptorModal.test.tsx | 5 - .../components/Header.keyboard.test.tsx | 5 - .../components/Header.test.tsx | 5 - .../components/inspector/JobForm.test.tsx | 5 - bin/adaptor_cache | 6 +- config/test.exs | 3 - lib/lightning/adaptor_registry.ex | 471 +----------------- lib/lightning/adaptor_service.ex | 55 +- lib/lightning/adaptors.ex | 166 +++++- lib/lightning/adaptors/local.ex | 94 +++- lib/lightning/adaptors/npm.ex | 10 +- lib/lightning/adaptors/package_name.ex | 32 +- lib/lightning/adaptors/repo.ex | 23 +- lib/lightning/adaptors/scheduler.ex | 43 +- lib/lightning/ai_assistant/ai_assistant.ex | 4 +- lib/lightning/application.ex | 5 - lib/lightning/config/bootstrap.ex | 163 ++++-- lib/lightning/release.ex | 22 + lib/lightning/workflows/job.ex | 27 +- .../channels/run_with_options.ex | 4 +- .../tasks/download_adaptor_registry_cache.ex | 40 +- lib/mix/tasks/seed_adaptors_from_file.ex | 64 +++ test/lightning/adaptor_registry_test.exs | 374 -------------- test/lightning/adaptor_service_test.exs | 121 ----- test/lightning/adaptors/local_test.exs | 130 ++++- test/lightning/adaptors/repo_test.exs | 30 -- test/lightning/adaptors_test.exs | 100 ++++ .../ai_assistant/ai_assistant_test.exs | 10 +- .../ai_assistant/unsaved_job_test.exs | 2 +- test/lightning/config/bootstrap_test.exs | 251 +++++++++- .../download_adaptor_registry_test.exs | 138 +++-- test/lightning/jobs_test.exs | 2 + test/lightning/metadata_service_test.exs | 21 + test/lightning/projects/provisioner_test.exs | 2 + test/lightning/setup_utils_test.exs | 9 + test/lightning/workflows/job_test.exs | 32 +- .../workflow_channel_broadcast_test.exs | 2 + .../tasks/seed_adaptors_from_file_test.exs | 149 ++++++ test/support/adaptor_test_helpers.ex | 28 +- test/support/fixtures/jobs_fixtures.ex | 10 +- tooling/adaptor_cache/README.md | 22 +- tooling/adaptor_cache/nginx.conf | 6 +- 47 files changed, 1450 insertions(+), 1340 deletions(-) delete mode 100644 assets/js/workflow-diagram/useAdaptorIcons.ts create mode 100644 lib/mix/tasks/seed_adaptors_from_file.ex delete mode 100644 test/lightning/adaptor_registry_test.exs delete mode 100644 test/lightning/adaptor_service_test.exs create mode 100644 test/mix/tasks/seed_adaptors_from_file_test.exs diff --git a/.env.example b/.env.example index 32883a4d40b..a48e63a3bfa 100644 --- a/.env.example +++ b/.env.example @@ -243,10 +243,12 @@ # data in your instance) you can set the following environment variable to "yes" # IS_RESETTABLE_DEMO=no -# This file to which the registry should be read from. In case the file doesnt -# exist, Lightning will attempt to fetch the file and write it to the same location. -# For this reason, you have to make sure that the directory exists and it is writable -# ADAPTORS_REGISTRY_JSON_PATH=/path/to/adaptor_registry_cache.json +# To boot from a static adaptor catalogue snapshot instead of reaching npm, +# seed it before starting the app. In dev/CI (Mix available): +# mix lightning.seed_adaptors_from_file --path /path/to/snapshot.json +# In a release (no Mix), from the app directory instead: +# bin/lightning eval 'Lightning.Release.seed_adaptors("/path/to/snapshot.json")' +# See `mix help lightning.seed_adaptors_from_file`. # # Enable local adaptors mode. OPENFN_ADAPTORS_REPO takes one repo path, or a # comma-separated list to merge several. See RUNNINGLOCAL.md for the details. @@ -254,18 +256,36 @@ # OPENFN_ADAPTORS_REPO=/path/to/repo/ # OPENFN_ADAPTORS_REPO=/path/to/private,/path/to/canonical # +# The new Lightning.Adaptors subsystem. ADAPTORS_STRATEGY picks which +# strategy serves adaptors: npm (default) or local. LOCAL_ADAPTORS above is a +# deprecated back-compat alias for ADAPTORS_STRATEGY=local, and +# OPENFN_ADAPTORS_REPO above is a deprecated back-compat alias for +# ADAPTORS_LOCAL_REPO below (both log a boot warning). +# ADAPTORS_STRATEGY=local +# ADAPTORS_LOCAL_REPO=/path/to/repo/ +# ADAPTORS_LOCAL_REPO=/path/to/private,/path/to/canonical +# +# Directory the adaptor icon cache is written to. Defaults to a subdirectory +# under the system temp dir. +# ADAPTORS_ICONS_PATH=/path/to/icon/cache +# # Lightning.Adaptors.NPM upstream URLs. Leave these unset in production: the -# defaults in lib/lightning/config/bootstrap.ex are the real npm, jsDelivr and -# raw.githubusercontent endpoints. Point them at the local caching reverse -# proxy while iterating on the adaptors subsystem, so refresh ticks are served -# from disk and work offline. Start it with `bin/adaptor_cache up`; see -# tooling/adaptor_cache/README.md. -# ADAPTOR_REGISTRY_URL=http://localhost:4874/npm -# ADAPTOR_JSDELIVR_URL=http://localhost:4874/jsdelivr -# ADAPTOR_GITHUB_URL=http://localhost:4874/github +# real npm, jsDelivr and raw.githubusercontent endpoints are the defaults +# baked into the strategy sub-modules themselves (@default_registry_url in +# npm/registry.ex, @default_jsdelivr_url in npm/schema.ex, +# @default_github_url/@default_github_ref in npm/github.ex). Point them at +# the local caching reverse proxy while iterating on the adaptors subsystem, +# so refresh ticks are served from disk and work offline. Start it with +# `bin/adaptor_cache up`; see tooling/adaptor_cache/README.md. +# ADAPTORS_NPM_REGISTRY_URL=http://localhost:4874/npm +# ADAPTORS_NPM_JSDELIVR_URL=http://localhost:4874/jsdelivr +# ADAPTORS_NPM_GITHUB_URL=http://localhost:4874/github # # The git ref under OpenFn/adaptors that icons are read from. -# ADAPTOR_GITHUB_REF=main +# ADAPTORS_NPM_GITHUB_REF=main +# +# HTTP receive timeout (ms) for registry/schema/icon fetches. Defaults to 30s. +# ADAPTORS_NPM_HTTP_TIMEOUT=30000 # ============================================================================== # <><><> WEBHOOK RETRY SETTINGS <><><> diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 7890db6547f..b6047daa1b6 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -222,7 +222,6 @@ For SMTP, the following environment variables are required: | `PRIMARY_ENCRYPTION_KEY` | A base64 encoded 32 character long string.
See [Encryption](#encryption). | | `QUEUE_RESULT_RETENTION_PERIOD_MINUTES` | The number of minutes to keep completed (successful) `ObanJobs` in the queue (not to be confused with runs and/or history) | | `SCHEMAS_PATH` | Path to the credential schemas that provide forms for different adaptors | -| `ADAPTORS_REGISTRY_JSON_PATH` | Path to adaptor registry file. When provided, the app will attempt to read from it then later fallback to the internet | | `SECRET_KEY_BASE` | A secret key used as a base to generate secrets for encrypting and signing data. | | `SENTRY_DSN` | If using Sentry for error monitoring, your DSN | | `URL_HOST` | The host used for writing URLs (e.g., `demo.openfn.org`) | diff --git a/RUNNINGLOCAL.md b/RUNNINGLOCAL.md index 2b86c1aabb0..ae27966897d 100644 --- a/RUNNINGLOCAL.md +++ b/RUNNINGLOCAL.md @@ -226,9 +226,9 @@ bin/adaptor_cache check # prove all three upstreams cache correctly Then point Lightning at it: ```sh -export ADAPTOR_REGISTRY_URL=http://localhost:4874/npm -export ADAPTOR_JSDELIVR_URL=http://localhost:4874/jsdelivr -export ADAPTOR_GITHUB_URL=http://localhost:4874/github +export ADAPTORS_NPM_REGISTRY_URL=http://localhost:4874/npm +export ADAPTORS_NPM_JSDELIVR_URL=http://localhost:4874/jsdelivr +export ADAPTORS_NPM_GITHUB_URL=http://localhost:4874/github ``` See `tooling/adaptor_cache/README.md` for the full command list, how to read the diff --git a/assets/js/workflow-diagram/useAdaptorIcons.ts b/assets/js/workflow-diagram/useAdaptorIcons.ts deleted file mode 100644 index 2c9d97e399a..00000000000 --- a/assets/js/workflow-diagram/useAdaptorIcons.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { useState, useEffect } from 'react'; - -export type AdaptorIconData = { - [adaptor: string]: - | { - rectangle: string; - square: string; - } - | undefined; -}; - -// This is a shared promise to load the adaptor data -let deffered: Promise | undefined; - -const useAdaptorIcons = (): AdaptorIconData | null => { - const [data, setData] = useState(null); - - useEffect(() => { - if (!deffered) { - // The first request to adaptor data will initiate the fetch - // and read the data - deffered = fetch('/images/adaptors/adaptor_icons.json') - .then(response => response.json() as Promise) - .catch(err => { - console.error('Error fetching Adaptor Icons manifest:', err); - return {} as AdaptorIconData; - }); - } - - // Subsequent calls will chain the fetch promise and instantly resolve once - // the data is down - void deffered.then(d => { - setData(d); - return d; - }); - }, []); - - return data; -}; - -export default useAdaptorIcons; diff --git a/assets/test/collaborative-editor/components/AdaptorSelectionModal.test.tsx b/assets/test/collaborative-editor/components/AdaptorSelectionModal.test.tsx index c312176f321..34db252b9ab 100644 --- a/assets/test/collaborative-editor/components/AdaptorSelectionModal.test.tsx +++ b/assets/test/collaborative-editor/components/AdaptorSelectionModal.test.tsx @@ -12,11 +12,6 @@ import { AdaptorSelectionModal } from '../../../js/collaborative-editor/componen import { StoreContext } from '../../../js/collaborative-editor/contexts/StoreProvider'; import type { Adaptor } from '../../../js/collaborative-editor/types/adaptor'; -// Mock useAdaptorIcons to avoid fetching icon manifest -vi.mock('#/workflow-diagram/useAdaptorIcons', () => ({ - default: () => null, -})); - // Mock adaptor data const mockProjectAdaptors: Adaptor[] = [ { diff --git a/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx b/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx index c6cd02b7623..37614f3be7b 100644 --- a/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx +++ b/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx @@ -29,11 +29,6 @@ import type { ProjectCredential, } from '../../../js/collaborative-editor/types/credential'; -// Mock useAdaptorIcons to avoid fetching icon manifest -vi.mock('#/workflow-diagram/useAdaptorIcons', () => ({ - default: () => null, -})); - // Mock adaptor data const mockProjectAdaptors: Adaptor[] = [ { diff --git a/assets/test/collaborative-editor/components/Header.keyboard.test.tsx b/assets/test/collaborative-editor/components/Header.keyboard.test.tsx index 13b8bec5799..0b53a680cd0 100644 --- a/assets/test/collaborative-editor/components/Header.keyboard.test.tsx +++ b/assets/test/collaborative-editor/components/Header.keyboard.test.tsx @@ -44,11 +44,6 @@ vi.mock('../../../js/react/lib/use-url-state', () => ({ useURLState: () => getURLStateMockValue(urlState), })); -// Mock useAdaptorIcons to prevent async fetch warnings -vi.mock('../../../js/workflow-diagram/useAdaptorIcons', () => ({ - default: () => ({}), -})); - // Mock Tooltip to prevent Radix UI timer-based updates vi.mock('../../../js/collaborative-editor/components/Tooltip', () => ({ Tooltip: ({ children }: { children: React.ReactNode }) => <>{children}, diff --git a/assets/test/collaborative-editor/components/Header.test.tsx b/assets/test/collaborative-editor/components/Header.test.tsx index 2826f9c3d2f..be46f977d3a 100644 --- a/assets/test/collaborative-editor/components/Header.test.tsx +++ b/assets/test/collaborative-editor/components/Header.test.tsx @@ -44,11 +44,6 @@ vi.mock('../../../js/react/lib/use-url-state', () => ({ useURLState: () => getURLStateMockValue(urlState), })); -// Mock useAdaptorIcons to prevent async fetch warnings -vi.mock('../../../js/workflow-diagram/useAdaptorIcons', () => ({ - default: () => ({}), -})); - let storeCleanup: (() => void) | null = null; afterEach(() => { diff --git a/assets/test/collaborative-editor/components/inspector/JobForm.test.tsx b/assets/test/collaborative-editor/components/inspector/JobForm.test.tsx index 6c279bf1135..7f43a6e9d12 100644 --- a/assets/test/collaborative-editor/components/inspector/JobForm.test.tsx +++ b/assets/test/collaborative-editor/components/inspector/JobForm.test.tsx @@ -40,11 +40,6 @@ import { } from '../../__helpers__/channelMocks'; import { createWorkflowYDoc } from '../../__helpers__/workflowFactory'; -// Mock useAdaptorIcons to avoid fetching icon manifest -vi.mock('#/workflow-diagram/useAdaptorIcons', () => ({ - default: () => null, -})); - /** * Helper to create and connect a workflow store with Y.Doc */ diff --git a/bin/adaptor_cache b/bin/adaptor_cache index e6aec611332..53f20ff4158 100755 --- a/bin/adaptor_cache +++ b/bin/adaptor_cache @@ -97,9 +97,9 @@ print_exports() { Point Lightning at the cache by exporting these (or uncommenting them in .env): - export ADAPTOR_REGISTRY_URL=${BASE_URL}/npm - export ADAPTOR_JSDELIVR_URL=${BASE_URL}/jsdelivr - export ADAPTOR_GITHUB_URL=${BASE_URL}/github + export ADAPTORS_NPM_REGISTRY_URL=${BASE_URL}/npm + export ADAPTORS_NPM_JSDELIVR_URL=${BASE_URL}/jsdelivr + export ADAPTORS_NPM_GITHUB_URL=${BASE_URL}/github Then: mix lightning.refresh_adaptors Watch it work: bin/adaptor_cache logs diff --git a/config/test.exs b/config/test.exs index 6a05ec32858..1057be2db8a 100644 --- a/config/test.exs +++ b/config/test.exs @@ -98,9 +98,6 @@ config :lightning, :workers, # In test we don't send emails. config :lightning, Lightning.Mailer, adapter: Swoosh.Adapters.Test -config :lightning, Lightning.AdaptorRegistry, - use_cache: "test/fixtures/adaptor_registry_cache.json" - # Adaptors.Supervisor config for test boot. # # - `:strategy` — the production `Lightning.Adaptors.Supervisor` mounted in diff --git a/lib/lightning/adaptor_registry.ex b/lib/lightning/adaptor_registry.ex index 87c010eb240..ce3b06ab513 100644 --- a/lib/lightning/adaptor_registry.ex +++ b/lib/lightning/adaptor_registry.ex @@ -1,474 +1,15 @@ defmodule Lightning.AdaptorRegistry do @moduledoc """ - Registry process to query and maintain a list of adaptors available for - writing jobs. - - Currently it queries NPM for all modules in the `@openfn` organization and - filters out modules that are known not to be adaptors. - - **Usage** - - ``` - # Starting the process - AdaptorRegistry.start_link() - # Getting a list of all adaptors - Lightning.AdaptorRegistry.AdaptorRegistry.all() - ``` - - **Caching** - - By default the results are cached to disk, and will be reused every start. - - In order to disable or configure caching pass see: `start_link/1`. - - The process uses `:continue` to return before the adaptors have been queried. - This does mean that the first call to the process will be delayed until - the `handle_continue/2` has finished. - - **Timeouts** - - There is a 'general' timeout of 30s, this is used for GenServer calls like - `all/1` and also internally when the modules are being queried. NPM can - be extremely fast to respond if the package is cached on their side, but - can take a couple of seconds if not cached. - """ - - use GenServer - - require Logger - - @excluded_adaptors [ - "@openfn/language-devtools", - "@openfn/language-template", - "@openfn/language-fhir-jembi", - "@openfn/language-collections" - ] - @timeout 30_000 - - # Captures the package name and optional version. Anchored with \A…\z (NOT - # ^…$, since $ matches before a trailing \n). Accepts scoped (@scope/name) and - # unscoped names with an optional @version (semver, prerelease, or the tokens - # `latest` / `local`); excludes newlines and shell metacharacters. Used to - # validate the :adaptor field (via `adaptor_format/0`, where the capture groups - # are ignored) and to split a package string in `resolve_package_name/1`. - @adaptor_format ~r{\A(@?[\w.-]+(?:/[\w.-]+)?)(?:@([\w.-]+))?\z} - - defmodule Npm do - @moduledoc """ - NPM API functions - """ - - def client do - Tesla.client([ - {Tesla.Middleware.BaseUrl, "https://registry.npmjs.org"}, - Tesla.Middleware.JSON - ]) - end - - @doc """ - Retrieve all packages for a given user or organization. Return empty list if - application cannot connect to NPM. (E.g., because it's started offline.) - """ - @spec user_packages(user :: String.t()) :: [map()] - def user_packages(user) do - Tesla.get(client(), "/-/user/#{user}/package") - |> case do - {:error, :nxdomain} -> - Logger.info("Unable to connect to NPM; no adaptors fetched.") - [] - - {:ok, resp} -> - Map.get(resp, :body) - end - end - - @doc """ - Retrieve all details for an NPM package - """ - @spec package_detail(package_name :: String.t()) :: map() - def package_detail(package_name) do - Tesla.get!(client(), "/#{package_name}").body - end - end - - @impl GenServer - def init(opts) do - {:ok, [], {:continue, opts}} - end - - @impl GenServer - def handle_continue(opts, _state) do - adaptors = - case Enum.into(opts, %{}) do - %{local_adaptors_repos: repo_paths} - when is_list(repo_paths) and repo_paths != [] -> - read_adaptors_from_local_repos(repo_paths) - - %{use_cache: use_cache} - when use_cache === true or is_binary(use_cache) -> - cache_path = - if is_binary(use_cache) do - use_cache - else - Path.join([ - System.tmp_dir!(), - "lightning", - "adaptor_registry_cache.json" - ]) - end - - read_from_cache(cache_path) || write_to_cache(cache_path, fetch()) - - _other -> - fetch() - end - - {:noreply, adaptors} - end - - # false positive, it's a file from init - # sobelow_skip ["Traversal.FileModule"] - defp write_to_cache(path, adaptors) when is_binary(path) do - Logger.debug("Writing Adapter Registry to #{path}") - cache_file = File.open!(path, [:write]) - IO.binwrite(cache_file, Jason.encode_to_iodata!(adaptors)) - File.close(cache_file) - - adaptors - end - - # false positive, it's a file from init - # sobelow_skip ["Traversal.FileModule"] - defp read_from_cache(path) when is_binary(path) do - File.read(path) - |> case do - {:ok, file} -> - Logger.debug("Found Adapter Registry from #{path}") - Jason.decode!(file, keys: :atoms!) - - {:error, _} -> - nil - end - end - - @doc """ - Starts the AdaptorRegistry - - **Options** - - - `:use_cache` (defaults to false) - stores the last set of results on disk - and uses the cached file for every subsequent start. - It can either be a boolean, or a string - the latter being a file path - to set where the cache file is located. - - `:local_adaptors_repos` - an ordered list of paths to local adaptor - monorepos (each containing a `packages/` subdirectory). When set, the - registry skips NPM and lists adaptors from these directories instead. - Earlier paths win on dirname collisions; shadowed entries are summarised - in a single warning log. - - `:name` (defaults to AdaptorRegistry) - the name of the process, useful - for testing and/or running multiple versions of the registry - """ - @spec start_link( - opts :: [ - use_cache: boolean() | binary(), - local_adaptors_repos: [binary()], - name: term() - ] - ) :: - {:error, any} | {:ok, pid} - def start_link(opts \\ [use_cache: true]) do - Logger.info("Starting AdaptorRegistry") - {name, opts} = Keyword.pop(opts, :name, __MODULE__) - GenServer.start_link(__MODULE__, opts, name: name) - end - - @impl GenServer - def handle_call(:all, _from, state) do - {:reply, state, state} - end - - @impl GenServer - def handle_call({:versions_for, module_name}, _from, state) do - versions = - state - |> Enum.find(fn %{name: name} -> name == module_name end) - |> case do - nil -> nil - %{versions: versions} -> versions - end - - {:reply, versions, state} - end - - @impl GenServer - def handle_call({:latest_for, module_name}, _from, state) do - latest = - state - |> Enum.find(fn %{name: name} -> name == module_name end) - |> case do - nil -> nil - %{latest: latest} -> latest - end - - {:reply, latest, state} - end - - @impl GenServer - def handle_call({:exists?, module_name}, _from, state) do - {:reply, Enum.any?(state, fn %{name: name} -> name == module_name end), - state} - end - - @doc """ - Get the current in-process list of adaptors. - This call will wait behind the `:continue` message when the process starts - up, so it may take a while the first time it is called (and the list hasn't - been fetched yet). - """ - @spec all(server :: GenServer.server()) :: list() - def all(server \\ __MODULE__) do - GenServer.call(server, :all, @timeout) - end - - @doc """ - Get a list of versions for a given module. - """ - @spec versions_for(server :: GenServer.server(), module_name :: String.t()) :: - list() | nil - def versions_for(server \\ __MODULE__, module_name) do - GenServer.call(server, {:versions_for, module_name}, @timeout) - end - - @doc """ - Get a latest version for a given module. - """ - @spec latest_for(server :: GenServer.server(), module_name :: String.t()) :: - list() | nil - def latest_for(server \\ __MODULE__, module_name) do - GenServer.call(server, {:latest_for, module_name}, @timeout) - end - - @doc """ - Returns true if the given package name is present in the registry. - - Name-only membership check (version is ignored). - """ - @spec exists?(server :: GenServer.server(), module_name :: String.t() | nil) :: - boolean() - def exists?(server \\ __MODULE__, module_name) - def exists?(_server, nil), do: false - - def exists?(server, module_name) when is_binary(module_name) do - GenServer.call(server, {:exists?, module_name}, @timeout) - end - - @doc """ - Fetch a list of packages for the @openfn organisation - """ - @spec fetch() :: [map()] - def fetch do - start = DateTime.utc_now() - Logger.debug("Fetching adaptors from NPM.") - - result = - Npm.user_packages("openfn") - |> Enum.map(fn {name, _} -> name end) - |> Enum.filter(fn name -> - Regex.match?(~r/@openfn\/language-\w+/, name) - end) - |> Enum.reject(fn name -> - name in @excluded_adaptors - end) - |> Task.async_stream( - &fetch_npm_details/1, - ordered: false, - max_concurrency: 10, - timeout: @timeout - ) - |> Stream.map(fn {:ok, detail} -> detail end) - |> Enum.to_list() - - diff = DateTime.utc_now() |> DateTime.diff(start, :millisecond) - Logger.debug(fn -> "Finished fetching adaptors in #{diff}ms." end) - - result - end - - defp fetch_npm_details(package_name) do - details = Npm.package_detail(package_name) - - %{ - name: details["name"], - repo: details["repository"]["url"], - latest: details["dist-tags"]["latest"], - versions: - Enum.reject(details["versions"], fn {_version, detail} -> - detail["deprecated"] - end) - |> Enum.map(fn {version, _detail} -> - %{version: version} - end) - } - end - - defp read_adaptors_from_local_repos(repo_paths) when is_list(repo_paths) do - Logger.debug("Using local adaptors repos at #{inspect(repo_paths)}") - - repo_paths - |> Enum.flat_map(&adaptors_in_repo/1) - |> dedupe_first_wins() - end - - defp adaptors_in_repo(repo_path) do - packages_path = Path.join(repo_path, "packages") - - case File.ls(packages_path) do - {:ok, entries} -> - Enum.map(entries, fn package -> - %{ - name: "@openfn/language-" <> package, - repo: "file://" <> Path.join([repo_path, "packages", package]), - latest: "local", - versions: [] - } - end) - - {:error, reason} -> - Logger.error( - "Skipping local adaptors repo #{inspect(repo_path)}: " <> - "cannot list #{inspect(packages_path)} (#{:file.format_error(reason)})" - ) - - [] - end - end - - # First-occurrence wins: when two roots ship a package with the same - # `@openfn/language-` name, the entry from the earlier root is kept. - # Listing your private repo before the canonical one therefore lets you - # override individual adaptors locally without forking the whole canonical - # tree. Shadowed entries are summarised in a single warning so the override - # case (the intended use of ordering) does not flood logs with one line per - # package. - defp dedupe_first_wins(adaptors) do - {kept_reversed, winners, shadowed} = - Enum.reduce(adaptors, {[], %{}, []}, fn adaptor, - {kept, winners, shadowed} -> - if Map.has_key?(winners, adaptor.name) do - {kept, winners, [adaptor | shadowed]} - else - {[adaptor | kept], Map.put(winners, adaptor.name, adaptor), shadowed} - end - end) - - log_shadowed(Enum.reverse(shadowed), winners) - Enum.reverse(kept_reversed) - end - - defp log_shadowed([], _winners), do: :ok - - defp log_shadowed(shadowed, winners) do - details = - shadowed - |> Enum.group_by(& &1.name) - |> Enum.map_join("; ", fn {name, losers} -> - winner = Map.fetch!(winners, name) - loser_repos = Enum.map_join(losers, ", ", & &1.repo) - "#{name} (using #{winner.repo}, shadowed #{loser_repos})" - end) - - Logger.warning( - "AdaptorRegistry: #{length(shadowed)} adaptor(s) shadowed by earlier " <> - "local-adaptors repo entries: #{details}" - ) - end - - @doc """ - The regular expression describing a valid adaptor package string: an npm - package name (scoped or unscoped) with an optional `@version`. - - The single definition of a valid adaptor name — used here to split a package - string, by `Lightning.AdaptorService` when resolving an install, and by - `Lightning.Workflows.Job` to validate the `:adaptor` field. + Holds `local_adaptors_enabled?/0`, still read by + `mix lightning.install_schemas` to decide whether to read credential + schemas from a local adaptors repo instead of npm. """ - @spec adaptor_format() :: Regex.t() - def adaptor_format, do: @adaptor_format @doc """ - Destructures an NPM style package name into module name and version. - - **Example** - - iex> resolve_package_name("@openfn/language-salesforce@1.2.3") - { "@openfn/language-salesforce", "1.2.3" } - iex> resolve_package_name("@openfn/language-salesforce") - { "@openfn/language-salesforce", nil } - + Whether `Lightning.Config.adaptor_registry/0` has at least one local + adaptors repo configured (`LOCAL_ADAPTORS`/`OPENFN_ADAPTORS_REPO`). """ - @spec resolve_package_name(package_name :: nil) :: {nil, nil} - def resolve_package_name(package_name) when is_nil(package_name), - do: {nil, nil} - - @spec resolve_package_name(package_name :: String.t()) :: - {binary | nil, binary | nil} - def resolve_package_name(package_name) when is_binary(package_name) do - @adaptor_format - |> Regex.run(package_name) - |> case do - [_, name, version] -> - {name, version} - - [_, name] -> - {name, nil} - - _ -> - {nil, nil} - end - |> then(fn - {name, version} when is_binary(name) -> - if local_adaptors_enabled?() do - {name, "local"} - else - {name, version} - end - - other -> - other - end) - end - - @doc """ - Same as `resolve_package_name/1` except will throw an exception if a package - name cannot be matched. - """ - @spec resolve_package_name!(package_name :: String.t()) :: - {binary, binary | nil} - def resolve_package_name!(package_name) when is_binary(package_name) do - {package_name, version} = resolve_package_name(package_name) - - if is_nil(package_name) do - raise ArgumentError, "Only npm style package names are currently supported" - end - - {package_name, version} - end - - def resolve_adaptor(adaptor) do - case resolve_package_name(adaptor) do - {nil, nil} -> - "" - - {adaptor_name, "local"} -> - "#{adaptor_name}@local" - - {adaptor_name, "latest"} -> - "#{adaptor_name}@#{latest_for(adaptor_name)}" - - _ -> - adaptor - end - end - + @spec local_adaptors_enabled?() :: boolean() def local_adaptors_enabled? do case Lightning.Config.adaptor_registry()[:local_adaptors_repos] do [_ | _] -> true diff --git a/lib/lightning/adaptor_service.ex b/lib/lightning/adaptor_service.ex index 1ab57d381c2..e3c0905489b 100644 --- a/lib/lightning/adaptor_service.ex +++ b/lib/lightning/adaptor_service.ex @@ -10,9 +10,6 @@ defmodule Lightning.AdaptorService do The service requires at least `:adaptors_path`, which is used to both query which adaptors are installed and when to install new adaptors. - Another optional setting is: `:repo`, which must point at a module that will be - used to do the querying and installing. - ## Installing Adaptors Using the `install/2` function an adaptor can be installed, which will also @@ -22,6 +19,10 @@ defmodule Lightning.AdaptorService do elsewhere such as delaying or rejecting processing until the adaptor becomes available. + Every install is gated on the adaptor catalogue (`Lightning.Adaptors`): + `install/2` refuses to run `npm install` for a package name the catalogue + doesn't recognise, including when the catalogue is empty. + ## Looking up adaptors The module leans on Elixir's built-in `Version` module to provide version @@ -54,11 +55,11 @@ defmodule Lightning.AdaptorService do """ use Agent - alias Lightning.AdaptorRegistry + alias Lightning.Adaptors require Logger - defmodule Adaptor do + defmodule InstalledAdaptor do @moduledoc false @type install_status :: :present | :installing @@ -88,7 +89,7 @@ defmodule Lightning.AdaptorService do This function is called when the service starts up in order to query which adaptors are already installed. """ - @callback list_local(path :: String.t()) :: list(Adaptor.t()) + @callback list_local(path :: String.t()) :: list(InstalledAdaptor.t()) def list_local(path, _depth \\ 4) when is_binary(path) do System.cmd("npm", ~w[list --global --json --long --prefix #{path}], env: [] @@ -103,7 +104,7 @@ defmodule Lightning.AdaptorService do local_name |> String.starts_with?("@openfn") end) |> Enum.map(fn {local_name, details} -> - %Adaptor{ + %InstalledAdaptor{ name: details["name"], version: details["version"], path: details["path"], @@ -193,10 +194,9 @@ defmodule Lightning.AdaptorService do @type t :: %__MODULE__{ name: GenServer.server(), - adaptors: [Adaptor.t()], + adaptors: [InstalledAdaptor.t()], adaptors_path: binary(), - repo: module(), - adaptor_registry: GenServer.server() + repo: module() } @enforce_keys [:adaptors_path] @@ -204,8 +204,7 @@ defmodule Lightning.AdaptorService do [ :name, adaptors: [], - repo: Repo, - adaptor_registry: Lightning.AdaptorRegistry + repo: Repo ] def find_adaptor(%{adaptors: adaptors}, fun) when is_function(fun) do @@ -235,12 +234,13 @@ defmodule Lightning.AdaptorService do Agent.get(agent, fn state -> state.adaptors end) end - @spec find_adaptor(Agent.agent(), package :: String.t()) :: Adaptor.t() | nil + @spec find_adaptor(Agent.agent(), package :: String.t()) :: + InstalledAdaptor.t() | nil def find_adaptor(agent, package) when is_binary(package) do find_adaptor(agent, resolve_package_name(package)) end - @spec find_adaptor(Agent.agent(), package_spec()) :: Adaptor.t() | nil + @spec find_adaptor(Agent.agent(), package_spec()) :: InstalledAdaptor.t() | nil def find_adaptor(agent, {package_name, version}) do requirement = version_to_requirement(version) @@ -284,7 +284,7 @@ defmodule Lightning.AdaptorService do end @spec install(Agent.agent(), binary()) :: - {:ok, Adaptor.t()} + {:ok, InstalledAdaptor.t()} | {:error, :adaptor_not_permitted} | {:error, {Collectable.t(), exit_status :: non_neg_integer}} def install(agent, package) when is_binary(package) do @@ -292,13 +292,11 @@ defmodule Lightning.AdaptorService do end @spec install(Agent.agent(), package_spec()) :: - {:ok, Adaptor.t()} + {:ok, InstalledAdaptor.t()} | {:error, :adaptor_not_permitted} | {:error, {Collectable.t(), exit_status :: non_neg_integer}} def install(agent, {package_name, _version} = package_spec) do - registry = Agent.get(agent, fn state -> state.adaptor_registry end) - - if AdaptorRegistry.exists?(registry, package_name) do + if known?(package_name) do agent |> find_adaptor(package_spec) |> case do @@ -314,11 +312,15 @@ defmodule Lightning.AdaptorService do end end + defp known?(nil), do: false + + defp known?(name), do: Adaptors.get_adaptor(name) != nil + @spec install!(Agent.agent(), package_spec()) :: - {:ok, Adaptor.t()} + {:ok, InstalledAdaptor.t()} | {:error, {Collectable.t(), exit_status :: non_neg_integer}} defp install!(agent, {package_name, version} = package_spec) do - new_adaptor = %Adaptor{ + new_adaptor = %InstalledAdaptor{ name: package_name, version: version, status: :installing @@ -350,15 +352,8 @@ defmodule Lightning.AdaptorService do end end - def resolve_package_name(package_name) when is_binary(package_name) do - AdaptorRegistry.adaptor_format() - |> Regex.run(package_name) - |> case do - [_, name, version] -> {name, version} - [_, name] -> {name, nil} - _ -> {nil, nil} - end - end + def resolve_package_name(package_name) when is_binary(package_name), + do: Adaptors.parse_spec(package_name) @doc """ Turns a package name and version into a string for NPM. diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index 959b7e1cbb0..530b8b2d214 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -1,25 +1,69 @@ defmodule Lightning.Adaptors do @moduledoc """ - Public facade for all adaptor metadata. + Public interface to the adaptor catalogue. - Delegates reads to `Lightning.Adaptors.Store`, refresh calls to - `Lightning.Adaptors.Scheduler`, and version resolution to - `Lightning.Adaptors.Repo`. No logic lives here. + ## Catalogue reads + + * `packages/0,1` - every adaptor for the active source + * `versions/2` - published versions of one adaptor + * `get_adaptor/1` - one adaptor as a `Lightning.Adaptors.Package`, + or `nil` + * `catalogue/0` and `catalogue_stamp/0` - the full catalogue and its + ETag basis + * `schema/1,2` and `icon/2,3` - per-adaptor assets + + ## Adaptor specs + + An adaptor spec is the `"name@version"` string a job stores, where the + version may be a semver, `latest`, `local`, or absent. + + * `parse_spec/1` - split a spec into `{name, version}` + * `valid_format?/1` - does a string match the strict spec format + * `resolve_version/2` - turn `latest`/`local` into a concrete version + * `to_wire/1` - render a spec for the worker's install step + + ## Refreshing + + * `refresh_now/0,1`, `refresh_package/1,2`, `refresh_icons/0,1` + * `seed_from_file/2` - populate the catalogue from a JSON snapshot, + used by `mix lightning.seed_adaptors_from_file` and + `Lightning.Release` Most functions come in a dual-arity shape: the zero-/single-arg form passes the compile-time default supervisor name `@sup`; the extra-arity form accepts an explicit supervisor name for test isolation. - `resolve_version/2`, `catalogue/0`, and `catalogue_stamp/0` are - exceptions — they read the global Repo directly, not a running - supervisor process, so there is nothing to swap for test isolation. + `get_adaptor/1`, `resolve_version/2`, `catalogue/0`, and + `catalogue_stamp/0` are exceptions — none of them go through `Store`'s + cache process. `get_adaptor/1` and `resolve_version/2` read the active + source from `Config.current_source/0` (a process-independent + `Application.get_env` read), so there's nothing to swap. `catalogue/0` + and `catalogue_stamp/0` read it from `AdaptorsSupervisor.source/1` + instead — a boot-time snapshot owned by a running supervisor — so + those two are only correct for `@sup`, the default supervisor. """ alias Lightning.Adaptors.Config + alias Lightning.Adaptors.PackageName alias Lightning.Adaptors.Repo alias Lightning.Adaptors.Scheduler alias Lightning.Adaptors.Store alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + defmodule Package do + @moduledoc """ + One catalogue adaptor, as seen by callers outside + `Lightning.Adaptors`. Not wire-serialised and not an `Ecto.Schema`. + """ + + @type t :: %__MODULE__{ + name: String.t(), + source: :npm | :local, + latest_version: String.t() | nil + } + + defstruct [:name, :source, :latest_version] + end + @sup Lightning.Adaptors @type package_meta :: Store.package_meta() @@ -31,9 +75,6 @@ defmodule Lightning.Adaptors do @spec packages(atom()) :: {:ok, [package_meta()]} | {:error, :timeout | term()} def packages(sup), do: Store.packages(sup) - @spec versions(String.t()) :: {:ok, [version_meta()]} | {:error, term()} - def versions(pkg), do: versions(@sup, pkg) - @spec versions(atom(), String.t()) :: {:ok, [version_meta()]} | {:error, term()} def versions(sup, pkg), do: Store.versions(sup, pkg) @@ -66,6 +107,57 @@ defmodule Lightning.Adaptors do @spec catalogue_stamp() :: {DateTime.t() | nil, non_neg_integer()} def catalogue_stamp, do: Repo.catalogue_stamp(AdaptorsSupervisor.source(@sup)) + @doc """ + One adaptor from the active source's catalogue, by bare package name. + + Takes a name, not a `name@version` spec — use `parse_spec/1` first if + you have a spec. Returns `nil` when the catalogue has no such adaptor, + including when it is empty. + """ + @spec get_adaptor(String.t()) :: Package.t() | nil + def get_adaptor(name) when is_binary(name) do + case Repo.get_adaptor(name, Config.current_source()) do + nil -> + nil + + adaptor -> + %Package{ + name: adaptor.name, + source: adaptor.source, + latest_version: adaptor.latest_version + } + end + end + + @doc """ + Split an adaptor spec into `{name, version}`, with `version` `nil` when + the spec carries none. Returns `{nil, nil}` for a string that isn't a + well-formed spec. + """ + @spec parse_spec(String.t()) :: {String.t() | nil, String.t() | nil} + def parse_spec(spec) when is_binary(spec) do + case Regex.run(PackageName.strict_format(), spec) do + [_, name, version] -> {name, version} + [_, name] -> {name, nil} + _ -> {nil, nil} + end + end + + @doc """ + Whether a string is a well-formed adaptor spec: a package name plus an + optional `@version`, with no newlines or shell metacharacters. + """ + @spec valid_format?(String.t()) :: boolean() + def valid_format?(spec) when is_binary(spec), + do: Regex.match?(PackageName.strict_format(), spec) + + @doc """ + Render an adaptor spec for the worker's install step, resolving + `latest` to a concrete version and preserving `local`. + """ + @spec to_wire(String.t() | nil) :: String.t() + defdelegate to_wire(spec), to: PackageName + @spec resolve_version(String.t(), String.t()) :: {:ok, String.t()} | {:error, :not_found} def resolve_version(name, requested) when requested in ["latest", "local"] do @@ -112,4 +204,58 @@ defmodule Lightning.Adaptors do @doc false def icon_meta(sup, name), do: Store.icon_meta(sup, name) + + @doc """ + Populate the adaptor catalogue from a JSON snapshot file, without + reaching npm. + + The file is a JSON array of adaptor records in the shape + `Lightning.Adaptors.Repo.upsert_adaptor/1` accepts — the same shape + `mix lightning.download_adaptor_registry_cache` writes. + + `opts`: + + * `:source` - `:npm` (default) or `:local`. + * `:replace` - when `true`, deletes every existing row for that + source before seeding, so the file becomes the source's entire + contents rather than a merge. The delete and every upsert run in + one transaction, so a bad record aborts the whole seed rather + than leaving the source partially replaced. + """ + @spec seed_from_file(Path.t(), keyword()) :: {:ok, non_neg_integer()} + def seed_from_file(path, opts \\ []) do + source = Keyword.get(opts, :source, :npm) + replace? = Keyword.get(opts, :replace, false) + + records = + path + |> File.read!() + |> Jason.decode!() + |> Enum.map(&normalize_snapshot_record(&1, source)) + + {:ok, _} = + Lightning.Repo.transaction(fn -> + if replace?, do: Repo.delete_all_for_source(source) + Enum.each(records, &Repo.upsert_adaptor/1) + end) + + {:ok, length(records)} + end + + # Top-level record keys and per-version keys map onto known schema + # fields, so they can be turned into existing atoms. `dependencies` and + # `peer_dependencies` values are left with string keys — that's the + # shape the `:map` columns already store. + defp normalize_snapshot_record(record, source) when is_map(record) do + record + |> atomize_known_keys() + |> Map.put(:source, source) + |> Map.update(:versions, [], fn versions -> + Enum.map(versions, &atomize_known_keys/1) + end) + end + + defp atomize_known_keys(map) do + Map.new(map, fn {k, v} -> {String.to_existing_atom(k), v} end) + end end diff --git a/lib/lightning/adaptors/local.ex b/lib/lightning/adaptors/local.ex index c47a942af6f..26b7bb0668d 100644 --- a/lib/lightning/adaptors/local.ex +++ b/lib/lightning/adaptors/local.ex @@ -5,20 +5,28 @@ defmodule Lightning.Adaptors.Local do Serves adaptor metadata, schemas, and icons from an on-disk OpenFn adaptors monorepo checkout. Gated by `LOCAL_ADAPTORS=true` and `OPENFN_ADAPTORS_REPO=/path/to/adaptors` at the runtime-config layer; - this module only reads the resolved path via - `Lightning.Adaptors.Config.strategy_opts(__MODULE__)[:path]`. + this module only reads the resolved paths via + `Lightning.Adaptors.Config.strategy_opts(__MODULE__)[:paths]`, an + ordered list of root directories. Each callback walks the filesystem afresh — caching is the Store's responsibility. The module is stateless; no GenServer, no ETS. ## Layout - Walks `$path/packages/*/`, reads each subdirectory's `package.json` - for the authoritative `name` and `version`. Directories with missing - or unparseable `package.json` are skipped with `Logger.warning` so a - malformed entry never crashes boot. Multiple directories sharing the - same `name` are collapsed into one record: `latest_version` is the - highest semver and `versions` lists every on-disk path. + Walks `packages/*/` under each configured root, reads each + subdirectory's `package.json` for the authoritative `name` and + `version`. Directories with missing or unparseable `package.json` are + skipped with `Logger.warning` so a malformed entry never crashes + boot. Multiple directories sharing the same `name` within one root + are collapsed into one record: `latest_version` is the highest + semver and `versions` lists every on-disk path in that root. + + When the same `name` appears under more than one configured root, + the earliest root in the list wins outright: its version directories + are used and the other root's are ignored entirely, not merged in, + and a single `Logger.warning` names every shadowed package once per + scan. `source: :local` is **not** set here — the Store stamps it before upsert. No network calls anywhere in this module. @@ -96,29 +104,69 @@ defmodule Lightning.Adaptors.Local do end defp discover do - case Config.strategy_opts(__MODULE__)[:path] do - nil -> + case Config.strategy_opts(__MODULE__)[:paths] do + paths when paths in [nil, []] or not is_list(paths) -> Logger.warning( - "Lightning.Adaptors.Local: :path is not configured " <> - "(set OPENFN_ADAPTORS_REPO or :lightning, Lightning.Adaptors.Local, path:)" + "Lightning.Adaptors.Local: :paths is not configured " <> + "(set OPENFN_ADAPTORS_REPO or :lightning, Lightning.Adaptors.Local, paths:)" ) {:error, :no_repo_path} - path -> - records = - path - |> Path.join("packages") - |> Path.join("*") - |> Path.wildcard() - |> Enum.filter(&File.dir?/1) - |> Enum.flat_map(&read_package_dir/1) - |> group_by_name() - - {:ok, records} + paths -> + {:ok, discover_paths(paths)} end end + # First occurrence wins: each root is grouped (highest semver) on its own + # first, then a name found in more than one root keeps only its earliest + # root's record. Matches install_schemas.ex:171-172's precedence for the + # same rule. + defp discover_paths(paths) do + paths + |> Enum.flat_map(&scan_root/1) + |> warn_shadowed() + |> Enum.uniq_by(& &1.name) + end + + defp scan_root(path) do + packages_dir = Path.join(path, "packages") + + if File.dir?(packages_dir) do + packages_dir + |> Path.join("*") + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + |> Enum.flat_map(&read_package_dir/1) + |> group_by_name() + else + Logger.warning( + "Lightning.Adaptors.Local: skipping root #{inspect(path)}: " <> + "#{inspect(packages_dir)} does not exist" + ) + + [] + end + end + + defp warn_shadowed(records) do + shadowed = + records + |> Enum.frequencies_by(& &1.name) + |> Enum.filter(fn {_name, count} -> count > 1 end) + |> Enum.map(fn {name, _count} -> name end) + + if shadowed != [] do + Logger.warning( + "Lightning.Adaptors.Local: shadowed duplicate package name(s) " <> + "across configured paths, first occurrence wins: " <> + Enum.join(shadowed, ", ") + ) + end + + records + end + defp read_package_dir(dir) do pkg_json_path = Path.join(dir, "package.json") diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index a65adeccbdf..156b35715c9 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -80,12 +80,12 @@ defmodule Lightning.Adaptors.NPM do # Strategy boundary: re-encode the decoded schema map to a JSON binary # so the row is persisted as text and `Jason.decode!(_, - # objects: :ordered_objects)` re-engages downstream. NPM's upstream - # Schema sub-module already decoded into a regular map, so field order - # is whatever map iteration yields — the round-trip preserves it for - # the Local strategy (raw binary in) and is a no-op for NPM data. + # objects: :ordered_objects)` re-engages downstream. `Schema.schema/2` + # always decodes via `Jason.decode/1`, so `data` is a map (or nil) here, + # never a raw binary — the Local strategy's own raw-binary schema text + # takes a separate path (`Local.read_schema/1`) and never reaches this + # function. defp encode_schema(nil), do: nil - defp encode_schema(data) when is_binary(data), do: data defp encode_schema(data) when is_map(data), do: Jason.encode!(data) @impl Lightning.Adaptors.Strategy diff --git a/lib/lightning/adaptors/package_name.ex b/lib/lightning/adaptors/package_name.ex index a629fa74c9d..746bdc2091e 100644 --- a/lib/lightning/adaptors/package_name.ex +++ b/lib/lightning/adaptors/package_name.ex @@ -3,12 +3,15 @@ defmodule Lightning.Adaptors.PackageName do NPM-style package-name parsing and worker wire-shape recomposition for the `Lightning.Adaptors.*` subsystem. - This module is the single source of truth for the legacy - `AdaptorRegistry.resolve_adaptor/1` and `resolve_package_name/1` - contracts, ported to read through the `Lightning.Adaptors` facade. + This module is the single source of truth for adaptor package name + parsing and wire recomposition, read through the `Lightning.Adaptors` + facade. - `parse/1` splits `"name@version"` strings; `to_wire/1` resolves the - `latest` literal through `Lightning.Adaptors.resolve_version/2`, + `parse/1` splits `"name@version"` strings using the same strict, + anchored format `strict_format/0` validates against, so a spec that + reaches `to_wire/1` after passing changeset validation is guaranteed to + parse to the same name — never a truncated or re-derived one. `to_wire/1` + resolves the `latest` literal through `Lightning.Adaptors.resolve_version/2`, preserves `"name@local"` as a literal regardless of source, and emits `"name@local"` under a `:local` strategy source. """ @@ -16,16 +19,29 @@ defmodule Lightning.Adaptors.PackageName do alias Lightning.Adaptors alias Lightning.Adaptors.Config - @package_name_regex ~r/(@?[\/\d\n\w-]+)(?:@([\d\.\w-]+))?$/ + # Anchored with \A…\z (NOT ^…$, since $ matches before a trailing \n). + # Accepts scoped (@scope/name) and unscoped names with an optional + # @version (semver, prerelease, or the tokens `latest` / `local`); + # excludes newlines and shell metacharacters. + @strict_format ~r{\A(@?[\w.-]+(?:/[\w.-]+)?)(?:@([\w.-]+))?\z} + + @doc """ + The strict, anchored package-name format: name plus optional `@version`, + rejecting embedded newlines and shell metacharacters. Read through + `Lightning.Adaptors.valid_format?/1` and + `Lightning.Adaptors.parse_spec/1`. + """ + @spec strict_format() :: Regex.t() + def strict_format, do: @strict_format @spec parse(nil) :: {nil, nil} def parse(nil), do: {nil, nil} @spec parse(String.t()) :: {String.t() | nil, String.t() | nil} def parse(package_name) when is_binary(package_name) do - case Regex.run(@package_name_regex, package_name) do + case Regex.run(@strict_format, package_name) do [_, name, version] -> {name, version} - [_, _name] -> {package_name, nil} + [_, name] -> {name, nil} _ -> {nil, nil} end end diff --git a/lib/lightning/adaptors/repo.ex b/lib/lightning/adaptors/repo.ex index e53a40a8cd5..b15d6da7c8f 100644 --- a/lib/lightning/adaptors/repo.ex +++ b/lib/lightning/adaptors/repo.ex @@ -171,6 +171,15 @@ defmodule Lightning.Adaptors.Repo do end end + @doc """ + Delete every row for `source`. + """ + @spec delete_all_for_source(source()) :: :ok + def delete_all_for_source(source) do + Lightning.Repo.delete_all(from a in Adaptor, where: a.source == ^source) + :ok + end + @doc """ Advance `checked_at` for a known `(name, source)` row without loading it. No-op when no row matches. @@ -190,20 +199,6 @@ defmodule Lightning.Adaptors.Repo do :ok end - @doc """ - The `limit` rows for a given `source` whose `checked_at` is oldest - first. Backs the Scheduler's per-tick work list. - """ - @spec stalest(pos_integer(), source()) :: [Adaptor.t()] - def stalest(limit, source) when is_integer(limit) and limit > 0 do - Lightning.Repo.all( - from a in Adaptor, - where: a.source == ^source, - order_by: [asc: a.checked_at], - limit: ^limit - ) - end - @doc """ Lean list of source-scoped adaptors that are missing at least one icon shape. Returns only the fields the Scheduler needs to decide whether to diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 3f275119765..9fe26e53f41 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -436,7 +436,6 @@ defmodule Lightning.Adaptors.Scheduler do defp accumulate_icon_change(acc, shape, row, package_icons, state) do sha_key = :"icon_#{shape}_sha256" - ext_key = :"icon_#{shape}_ext" etag_key = :"icon_#{shape}_etag" case Map.get(package_icons, shape) do @@ -447,11 +446,7 @@ defmodule Lightning.Adaptors.Scheduler do # that differs from what we have. nil never clobbers. maybe_accumulate_etag(acc, etag_key, row, Map.get(entry, :etag)) else - accumulate_fetched_icon(acc, shape, row, entry, ext, sha, bytes, state, - sha_key: sha_key, - ext_key: ext_key, - etag_key: etag_key - ) + accumulate_fetched_icon(acc, shape, row, entry, ext, sha, bytes, state) end :not_modified -> @@ -463,27 +458,25 @@ defmodule Lightning.Adaptors.Scheduler do end end - defp accumulate_fetched_icon(acc, shape, row, entry, ext, sha, bytes, state, - sha_key: sha_key, - ext_key: ext_key, - etag_key: etag_key - ) do - try do - {:ok, ^sha} = IconCache.write!(state.source, row.name, shape, ext, bytes) + defp accumulate_fetched_icon(acc, shape, row, entry, ext, sha, bytes, state) do + sha_key = :"icon_#{shape}_sha256" + ext_key = :"icon_#{shape}_ext" + etag_key = :"icon_#{shape}_etag" - acc - |> Map.put(ext_key, ext) - |> Map.put(sha_key, sha) - |> maybe_accumulate_etag(etag_key, row, Map.get(entry, :etag)) - rescue - e -> - Logger.warning( - "Scheduler: IconCache.write!(#{row.name}, #{shape}) failed: " <> - Exception.message(e) - ) + {:ok, ^sha} = IconCache.write!(state.source, row.name, shape, ext, bytes) - acc - end + acc + |> Map.put(ext_key, ext) + |> Map.put(sha_key, sha) + |> maybe_accumulate_etag(etag_key, row, Map.get(entry, :etag)) + rescue + e -> + Logger.warning( + "Scheduler: IconCache.write!(#{row.name}, #{shape}) failed: " <> + Exception.message(e) + ) + + acc end # nil → preserve existing etag on the row (do not clobber). diff --git a/lib/lightning/ai_assistant/ai_assistant.ex b/lib/lightning/ai_assistant/ai_assistant.ex index e84214afe5d..536b81c0fa9 100644 --- a/lib/lightning/ai_assistant/ai_assistant.ex +++ b/lib/lightning/ai_assistant/ai_assistant.ex @@ -545,7 +545,7 @@ defmodule Lightning.AiAssistant do ## Returns An updated `ChatSession` struct with `:expression` and `:adaptor` fields populated. - The adaptor is resolved through `Lightning.AdaptorRegistry`. + The adaptor is resolved through `Lightning.Adaptors.to_wire/1`. """ @spec put_expression_and_adaptor(ChatSession.t(), String.t(), String.t()) :: ChatSession.t() @@ -553,7 +553,7 @@ defmodule Lightning.AiAssistant do %{ session | expression: expression, - adaptor: Lightning.Adaptors.PackageName.to_wire(adaptor) + adaptor: Lightning.Adaptors.to_wire(adaptor) } end diff --git a/lib/lightning/application.ex b/lib/lightning/application.ex index 4d9a19c3cbd..d84aae759ef 100644 --- a/lib/lightning/application.ex +++ b/lib/lightning/application.ex @@ -39,10 +39,6 @@ defmodule Lightning.Application do # formatter: Logger.Formatter.new() # }) - adaptor_registry_childspec = - {Lightning.AdaptorRegistry, - Application.get_env(:lightning, Lightning.AdaptorRegistry, [])} - adaptor_service_childspec = {Lightning.AdaptorService, [name: :adaptor_service] @@ -168,7 +164,6 @@ defmodule Lightning.Application do LightningWeb.Endpoint, Lightning.Workflows.Presence, LightningWeb.WorkerPresence, - adaptor_registry_childspec, adaptor_service_childspec, {Lightning.Adaptors.Supervisor, name: Lightning.Adaptors}, {Lightning.TaskWorker, name: :cli_task_worker}, diff --git a/lib/lightning/config/bootstrap.ex b/lib/lightning/config/bootstrap.ex index f947a8a4ef4..bc2d64753d8 100644 --- a/lib/lightning/config/bootstrap.ex +++ b/lib/lightning/config/bootstrap.ex @@ -31,6 +31,8 @@ defmodule Lightning.Config.Bootstrap do alias Lightning.Config.Utils + require Logger + def source_envs do {:ok, _} = source([ @@ -235,18 +237,7 @@ defmodule Lightning.Config.Bootstrap do # Comma-separated to match the ws-worker parser, so the picker view and # @local resolution agree on the same repo list. See RUNNINGLOCAL.md. local_adaptors_repos = - env!("OPENFN_ADAPTORS_REPO", :string, nil) - |> case do - nil -> - [] - - value when is_binary(value) -> - value - |> String.split(",", trim: true) - |> Enum.map(&String.trim/1) - |> Enum.reject(&(&1 == "")) - |> Enum.map(&Path.expand/1) - end + parse_repo_list(env!("OPENFN_ADAPTORS_REPO", :string, nil)) use_local_adaptors_repos? = env!("LOCAL_ADAPTORS", &Utils.ensure_boolean/1, false) @@ -258,16 +249,15 @@ defmodule Lightning.Config.Bootstrap do end end) + # local_adaptors_repos also feeds the new Lightning.Adaptors.Local + # strategy below (dual-write). install_schemas.ex:177 still reads this + # exact key, so it is not moved, only copied. config :lightning, Lightning.AdaptorRegistry, - use_cache: - env!( - "ADAPTORS_REGISTRY_JSON_PATH", - :string, - Utils.get_env([:lightning, Lightning.AdaptorRegistry, :use_cache]) - ), local_adaptors_repos: if(use_local_adaptors_repos?, do: local_adaptors_repos, else: []) + configure_adaptors_strategy(local_adaptors_repos, use_local_adaptors_repos?) + # Upstreams for the NPM strategy. Each key reaches exactly one sub-module # through Lightning.Adaptors.Config.strategy_opts/1: registry_url is the # npm search and packument endpoint (NPM.Registry), jsdelivr_url serves @@ -277,18 +267,18 @@ defmodule Lightning.Config.Bootstrap do # # Point them at `bin/adaptor_cache` to serve all three from a local disk # cache while working on adaptors. - config :lightning, Lightning.Adaptors.NPM, - registry_url: - env!("ADAPTOR_REGISTRY_URL", :string, "https://registry.npmjs.org"), - jsdelivr_url: - env!("ADAPTOR_JSDELIVR_URL", :string, "https://cdn.jsdelivr.net"), - github_url: - env!( - "ADAPTOR_GITHUB_URL", - :string, - "https://raw.githubusercontent.com" - ), - github_ref: env!("ADAPTOR_GITHUB_REF", :string, "main") + # Defaults live in the strategy sub-modules' own @default_* attributes; + # bootstrap only maps an env var onto an override when one is set. + config :lightning, + Lightning.Adaptors.NPM, + [ + registry_url: env!("ADAPTORS_NPM_REGISTRY_URL", :string, nil), + jsdelivr_url: env!("ADAPTORS_NPM_JSDELIVR_URL", :string, nil), + github_url: env!("ADAPTORS_NPM_GITHUB_URL", :string, nil), + github_ref: env!("ADAPTORS_NPM_GITHUB_REF", :string, nil), + http_timeout: env!("ADAPTORS_NPM_HTTP_TIMEOUT", :integer?, nil) + ] + |> Enum.reject(fn {_key, value} -> is_nil(value) end) config :lightning, schemas_path: @@ -1022,6 +1012,119 @@ defmodule Lightning.Config.Bootstrap do ] end + defp configure_adaptors_strategy( + local_adaptors_repos, + use_local_adaptors_repos? + ) do + adaptors_strategy_value = + case env!("ADAPTORS_STRATEGY", :string, nil) do + nil -> nil + value -> String.trim(value) + end + + adaptors_strategy = + case adaptors_strategy_value do + blank when blank in [nil, ""] -> + local_adaptors_back_compat_strategy(use_local_adaptors_repos?) + + "npm" -> + Lightning.Adaptors.NPM + + "local" -> + Lightning.Adaptors.Local + + unknown -> + raise """ + Unknown ADAPTORS_STRATEGY: #{unknown} + + Currently supported strategies are: + + - npm (default) + - local + """ + end + + local_strategy_paths = + resolve_local_strategy_paths(local_adaptors_repos, adaptors_strategy) + + if adaptors_strategy == Lightning.Adaptors.Local and + local_strategy_paths == [] do + raise """ + ADAPTORS_STRATEGY is set to local, but neither ADAPTORS_LOCAL_REPO nor the deprecated OPENFN_ADAPTORS_REPO is set. + """ + end + + config :lightning, + Lightning.Adaptors, + [ + # config/test.exs pins :strategy to StrategyMock so the + # application-level supervisor never hits the network during the + # test suite. config/runtime.exs deep-merges this config over + # test.exs on every boot (including :test), so writing a real + # strategy here unconditionally would silently replace the mock. + strategy: + if(config_env() == :test, do: nil, else: adaptors_strategy), + # An operator-supplied path is expanded once at boot, unlike + # Config.icon_path/0's {:tmp, ...} default, which is deliberately + # resolved at call time (see its doc) so a compiled release + # doesn't bake in a build-time tmp path. An explicit override has + # no such concern. + icon_path: + env!("ADAPTORS_ICONS_PATH", :string, nil) |> expand_or_nil() + ] + |> Enum.reject(fn {_key, value} -> is_nil(value) end) + + config :lightning, Lightning.Adaptors.Local, paths: local_strategy_paths + end + + defp local_adaptors_back_compat_strategy(use_local_adaptors_repos?) do + if use_local_adaptors_repos? do + Logger.warning( + "LOCAL_ADAPTORS is deprecated, use ADAPTORS_STRATEGY=local instead." + ) + + Lightning.Adaptors.Local + else + Lightning.Adaptors.NPM + end + end + + # ADAPTORS_LOCAL_REPO wins outright when set. When unset, fall back to + # the (ungated) OPENFN_ADAPTORS_REPO parse above, warning only when the + # new subsystem is actually running the Local strategy — an operator + # who still needs OPENFN_ADAPTORS_REPO for the old registry while + # running the new subsystem on npm shouldn't be warned about a var they + # legitimately need. + defp resolve_local_strategy_paths(local_adaptors_repos, adaptors_strategy) do + case env!("ADAPTORS_LOCAL_REPO", :string, nil) |> parse_repo_list() do + [] -> + if local_adaptors_repos != [] and + adaptors_strategy == Lightning.Adaptors.Local do + Logger.warning( + "OPENFN_ADAPTORS_REPO is deprecated, use ADAPTORS_LOCAL_REPO instead." + ) + end + + local_adaptors_repos + + paths -> + paths + end + end + + defp parse_repo_list(nil), do: [] + + defp parse_repo_list(value) when is_binary(value) do + value + |> String.split(",", trim: true) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + |> Enum.map(&Path.expand/1) + end + + defp expand_or_nil(nil), do: nil + defp expand_or_nil(path) when is_binary(path), do: Path.expand(path) + defp get_env(app) do Process.get({Config, :config}) |> Keyword.get(app) diff --git a/lib/lightning/release.ex b/lib/lightning/release.ex index 1429e369a39..185761afbe2 100644 --- a/lib/lightning/release.ex +++ b/lib/lightning/release.ex @@ -36,6 +36,28 @@ defmodule Lightning.Release do end end + @doc """ + Populate the adaptor catalogue from a JSON snapshot file, without + reaching npm. The release-safe path for `Lightning.Adaptors.seed_from_file/2` + — there is no Mix in a release, so `mix lightning.seed_adaptors_from_file` + cannot run there; this is what `bin/lightning eval` calls instead. + + ## Usage + + bin/lightning eval 'Lightning.Release.seed_adaptors("/path/to/snapshot.json")' + bin/lightning eval 'Lightning.Release.seed_adaptors("/path/to/snapshot.json", replace: true)' + """ + def seed_adaptors(path, opts \\ []) do + load_app() + + {:ok, result, _apps} = + Ecto.Migrator.with_repo(@repo, fn _repo -> + Lightning.Adaptors.seed_from_file(path, opts) + end) + + result + end + def rollback(repo, version) do load_app() diff --git a/lib/lightning/workflows/job.ex b/lib/lightning/workflows/job.ex index f003296bec3..00b9ec5bd31 100644 --- a/lib/lightning/workflows/job.ex +++ b/lib/lightning/workflows/job.ex @@ -17,7 +17,7 @@ defmodule Lightning.Workflows.Job do """ use Lightning.Schema - alias Lightning.AdaptorRegistry + alias Lightning.Adaptors alias Lightning.Credentials.Credential alias Lightning.Credentials.KeychainCredential alias Lightning.Credentials.Scoping @@ -140,9 +140,11 @@ defmodule Lightning.Workflows.Job do defp validate_adaptor(changeset) do changeset = - validate_format(changeset, :adaptor, AdaptorRegistry.adaptor_format(), - message: "adaptor has invalid format" - ) + validate_change(changeset, :adaptor, fn :adaptor, adaptor -> + if Adaptors.valid_format?(adaptor), + do: [], + else: [adaptor: "adaptor has invalid format"] + end) if changeset.valid? do validate_known_adaptor(changeset) @@ -151,25 +153,20 @@ defmodule Lightning.Workflows.Job do end end - # Rejects an adaptor the registry doesn't know about, so an unknown package - # cannot be persisted on a job. + # `job.adaptor` reaches the worker's install step unfiltered, so an + # adaptor missing from the catalogue is rejected here whatever its name + # — an empty catalogue permits nothing. defp validate_known_adaptor(changeset) do validate_change(changeset, :adaptor, fn :adaptor, adaptor -> - if adaptor_known?(adaptor) do + with {name, _version} when is_binary(name) <- Adaptors.parse_spec(adaptor), + %Adaptors.Package{} <- Adaptors.get_adaptor(name) do [] else - [adaptor: "is not a recognised adaptor"] + _ -> [adaptor: "is not a recognised adaptor"] end end) end - defp adaptor_known?(adaptor) do - case AdaptorRegistry.resolve_package_name(adaptor) do - {name, _version} when is_binary(name) -> AdaptorRegistry.exists?(name) - _ -> false - end - end - defp validate_keychain_credential_project_membership(changeset) do keychain_credential_id = get_field(changeset, :keychain_credential_id) workflow_id = get_field(changeset, :workflow_id) diff --git a/lib/lightning_web/channels/run_with_options.ex b/lib/lightning_web/channels/run_with_options.ex index 15c2e4cf4e0..5f13975c15e 100644 --- a/lib/lightning_web/channels/run_with_options.ex +++ b/lib/lightning_web/channels/run_with_options.ex @@ -1,7 +1,7 @@ defmodule LightningWeb.RunWithOptions do @moduledoc false - alias Lightning.Adaptors.PackageName + alias Lightning.Adaptors alias Lightning.Run alias Lightning.Workflows.Snapshot.Edge alias Lightning.Workflows.Snapshot.Job @@ -41,7 +41,7 @@ defmodule LightningWeb.RunWithOptions do def render(%Job{} = job) do %{ "id" => job.id, - "adaptor" => PackageName.to_wire(job.adaptor), + "adaptor" => Adaptors.to_wire(job.adaptor), "credential_id" => get_credential_id(job), "body" => job.body, "name" => job.name diff --git a/lib/mix/tasks/download_adaptor_registry_cache.ex b/lib/mix/tasks/download_adaptor_registry_cache.ex index 1b992d6209a..2ba175149ea 100644 --- a/lib/mix/tasks/download_adaptor_registry_cache.ex +++ b/lib/mix/tasks/download_adaptor_registry_cache.ex @@ -1,34 +1,60 @@ defmodule Mix.Tasks.Lightning.DownloadAdaptorRegistryCache do - @shortdoc "Downloads the adaptor registry json cache" + @shortdoc "Downloads an adaptor catalogue snapshot for offline seeding" @moduledoc """ - Downloads the adaptor registry json cache + Fetches every `@openfn/language-*` adaptor from npm via + `Lightning.Adaptors.NPM` and writes the full records to a JSON file, in + the shape `Lightning.Adaptors.Repo.upsert_adaptor/1` accepts. + + The file this writes is what `mix lightning.seed_adaptors_from_file` + reads. + Use --path to specify the location """ use Mix.Task - alias Lightning.AdaptorRegistry + alias Lightning.Adaptors.NPM + alias Lightning.Adaptors.NPM.Registry def run(args) do Application.ensure_started(:telemetry) Finch.start_link(name: Lightning.Finch) - case AdaptorRegistry.fetch() do - [] -> + case Registry.list_adaptors() do + {:ok, []} -> Mix.shell().error( "No adaptors found! Check that you have internet connection" ) - adaptors -> + {:ok, listing} -> + adaptors = + listing + |> Task.async_stream(&fetch_full_record/1, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.map(fn {:ok, record} -> record end) + |> Enum.reject(&is_nil/1) + path = parse_path(args) cache_file = File.open!(path, [:write]) IO.binwrite(cache_file, Jason.encode_to_iodata!(adaptors)) File.close(cache_file) Mix.shell().info( - "AdaptorRegistry downloaded successfully. File stored at: #{path}" + "Adaptor catalogue downloaded successfully. File stored at: #{path}" ) + + {:error, reason} -> + Mix.shell().error("Unable to fetch adaptor listing: #{inspect(reason)}") + end + end + + defp fetch_full_record(%{name: name}) do + case NPM.fetch_adaptor(name) do + {:ok, record} -> Map.put(record, :source, :npm) + {:error, _reason} -> nil end end diff --git a/lib/mix/tasks/seed_adaptors_from_file.ex b/lib/mix/tasks/seed_adaptors_from_file.ex new file mode 100644 index 00000000000..e6c4e80ccf0 --- /dev/null +++ b/lib/mix/tasks/seed_adaptors_from_file.ex @@ -0,0 +1,64 @@ +defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFile do + @shortdoc "Seed the adaptor catalogue from a local JSON snapshot" + + @moduledoc """ + Populate the `adaptors` table from a JSON file, without reaching npm. + + The file is a JSON array of adaptor records in the shape + `Lightning.Adaptors.Repo.upsert_adaptor/1` accepts — the same shape + `mix lightning.download_adaptor_registry_cache` writes. + + ## Usage + + mix lightning.seed_adaptors_from_file --path snapshot.json + mix lightning.seed_adaptors_from_file --path snapshot.json --source local + mix lightning.seed_adaptors_from_file --path snapshot.json --replace + + `--source` defaults to `npm`. `--replace` deletes every existing row for + that source first, so the file becomes the source's entire contents + rather than a merge. + + ## In a release + + There is no Mix (or this task) in a release image. Seed from a + snapshot before starting the app by running the equivalent through + `bin/lightning eval` instead: + + bin/lightning eval 'Lightning.Release.seed_adaptors("/path/to/snapshot.json", replace: true)' + """ + + use Mix.Task + + alias Lightning.Adaptors + + @impl Mix.Task + def run(argv) do + Mix.Task.run("app.start") + + {opts, _args} = + OptionParser.parse!(argv, + strict: [path: :string, source: :string, replace: :boolean] + ) + + path = + opts[:path] || + raise "Usage: mix lightning.seed_adaptors_from_file --path " + + source = parse_source(opts[:source]) + + {:ok, count} = + Adaptors.seed_from_file(path, + source: source, + replace: opts[:replace] || false + ) + + Mix.shell().info("Seeded #{count} adaptor(s) from #{path}.") + end + + defp parse_source(nil), do: :npm + defp parse_source("npm"), do: :npm + defp parse_source("local"), do: :local + + defp parse_source(other), + do: raise("Unknown --source: #{other} (expected npm or local)") +end diff --git a/test/lightning/adaptor_registry_test.exs b/test/lightning/adaptor_registry_test.exs deleted file mode 100644 index 4de10b8bc88..00000000000 --- a/test/lightning/adaptor_registry_test.exs +++ /dev/null @@ -1,374 +0,0 @@ -defmodule Lightning.AdaptorRegistryTest do - use Lightning.DataCase, async: false - - import Mox - import Tesla.Test - - setup :set_mox_from_context - setup :verify_on_exit! - - alias Lightning.AdaptorRegistry - - describe "start_link/1" do - test "uses cache from a specific location" do - file_path = - Briefly.create!(extname: ".json") - |> tap(fn path -> - File.write!(path, ~S""" - [{ - "latest": "3.0.5", - "name": "@openfn/language-dhis2", - "repo": "git+https://github.com/openfn/language-dhis2.git", - "versions": [] - }] - """) - end) - - start_supervised!( - {AdaptorRegistry, [name: :test_adaptor_registry, use_cache: file_path]} - ) - - results = AdaptorRegistry.all(:test_adaptor_registry) - assert length(results) == 1 - end - - test "retrieves a list of adaptors when caching is disabled" do - default_npm_response = - File.read!("test/fixtures/language-common-npm.json") |> Jason.decode!() - - expect_tesla_call( - times: 7, - returns: fn env, [] -> - case env.url do - "https://registry.npmjs.org/-/user/openfn/package" -> - {:ok, - json( - %Tesla.Env{status: 200}, - File.read!("test/fixtures/openfn-packages-npm.json") - |> Jason.decode!() - )} - - "https://registry.npmjs.org/@openfn/" <> _adaptor -> - {:ok, json(%Tesla.Env{status: 200}, default_npm_response)} - end - end - ) - - expected_adaptors = [ - "@openfn/language-asana", - "@openfn/language-common", - "@openfn/language-commcare", - "@openfn/language-dhis2", - "@openfn/language-http", - "@openfn/language-salesforce" - ] - - start_supervised!( - {AdaptorRegistry, [name: :test_adaptor_registry, use_cache: false]} - ) - - results = AdaptorRegistry.all(:test_adaptor_registry) - - assert_received_tesla_call(env, []) - - assert_tesla_env(env, %Tesla.Env{ - method: :get, - url: "https://registry.npmjs.org/-/user/openfn/package" - }) - - 1..length(expected_adaptors) - |> Enum.each(fn _ -> - assert_received_tesla_call(env, []) - - assert %Tesla.Env{ - method: :get, - url: "https://registry.npmjs.org/" <> adaptor - } = env - - assert adaptor in expected_adaptors - end) - - assert length(results) == 6 - - versions = [ - %{version: "1.1.0"}, - %{version: "1.1.1"}, - %{version: "1.2.0"}, - %{version: "1.2.1"}, - %{version: "1.2.2"}, - %{version: "1.2.4"}, - %{version: "1.2.5"}, - %{version: "1.2.6"}, - %{version: "1.2.7"}, - %{version: "1.2.8"}, - %{version: "1.4.0"}, - %{version: "1.4.1"}, - %{version: "1.4.2"}, - %{version: "1.5.0"}, - %{version: "1.6.0"}, - %{version: "1.6.1"}, - %{version: "1.6.2"} - ] - - assert %{ - name: "@openfn/language-common", - repo: "git+https://github.com/OpenFn/language-common.git", - latest: "1.6.2", - versions: versions - } in results - - assert AdaptorRegistry.versions_for( - :test_adaptor_registry, - "@openfn/language-common" - ) == - versions - - assert AdaptorRegistry.versions_for( - :test_adaptor_registry, - "@openfn/language-foobar" - ) == - nil - end - - @tag :tmp_dir - test "lists directory names from a single-element local_adaptors_repos list", - %{tmp_dir: tmp_dir, test: test} do - expected_adaptors = ["foo", "bar", "baz"] - - Enum.each(expected_adaptors, fn adaptor -> - [tmp_dir, "packages", adaptor] |> Path.join() |> File.mkdir_p!() - end) - - start_supervised!( - {AdaptorRegistry, [name: test, local_adaptors_repos: [tmp_dir]]} - ) - - results = AdaptorRegistry.all(test) - - for adaptor <- expected_adaptors do - expected_result = %{ - name: "@openfn/language-#{adaptor}", - repo: "file://" <> Path.join([tmp_dir, "packages", adaptor]), - latest: "local", - versions: [] - } - - assert expected_result in results - end - end - - @tag :tmp_dir - test "merges adaptors from multiple local_adaptors_repos", %{ - tmp_dir: tmp_dir, - test: test - } do - repo_a = Path.join(tmp_dir, "a") - repo_b = Path.join(tmp_dir, "b") - [repo_a, "packages", "alpha"] |> Path.join() |> File.mkdir_p!() - [repo_b, "packages", "beta"] |> Path.join() |> File.mkdir_p!() - - start_supervised!( - {AdaptorRegistry, [name: test, local_adaptors_repos: [repo_a, repo_b]]} - ) - - names = AdaptorRegistry.all(test) |> Enum.map(& &1.name) |> Enum.sort() - - assert names == ["@openfn/language-alpha", "@openfn/language-beta"] - end - - @tag :tmp_dir - test "first repo wins on collision and emits a warning", %{ - tmp_dir: tmp_dir, - test: test - } do - repo_a = Path.join(tmp_dir, "a") - repo_b = Path.join(tmp_dir, "b") - [repo_a, "packages", "http"] |> Path.join() |> File.mkdir_p!() - [repo_b, "packages", "http"] |> Path.join() |> File.mkdir_p!() - - log = - ExUnit.CaptureLog.capture_log(fn -> - start_supervised!( - {AdaptorRegistry, - [name: test, local_adaptors_repos: [repo_a, repo_b]]} - ) - - # force the GenServer to finish handle_continue - AdaptorRegistry.all(test) - end) - - results = AdaptorRegistry.all(test) - assert length(results) == 1 - - assert hd(results).repo == - "file://" <> Path.join([repo_a, "packages", "http"]) - - assert log =~ "@openfn/language-http" - assert log =~ "shadowed" - - assert log =~ "using" - assert log =~ "file://" <> Path.join([repo_a, "packages", "http"]) - assert log =~ "file://" <> Path.join([repo_b, "packages", "http"]) - end - - @tag :tmp_dir - test "soft-fails when a repo path is missing or unreadable", %{ - tmp_dir: tmp_dir, - test: test - } do - good_repo = Path.join(tmp_dir, "good") - missing_repo = Path.join(tmp_dir, "does-not-exist") - [good_repo, "packages", "alpha"] |> Path.join() |> File.mkdir_p!() - - log = - ExUnit.CaptureLog.capture_log(fn -> - start_supervised!( - {AdaptorRegistry, - [name: test, local_adaptors_repos: [missing_repo, good_repo]]} - ) - - AdaptorRegistry.all(test) - end) - - names = AdaptorRegistry.all(test) |> Enum.map(& &1.name) - assert names == ["@openfn/language-alpha"] - assert log =~ "Skipping local adaptors repo" - assert log =~ missing_repo - end - end - - describe "local_adaptors_enabled?/0" do - test "returns true when a non-empty plural list is configured" do - Mox.stub(Lightning.MockConfig, :adaptor_registry, fn -> - [local_adaptors_repos: ["/some/path"]] - end) - - assert AdaptorRegistry.local_adaptors_enabled?() - end - - test "returns false when the list is empty" do - Mox.stub(Lightning.MockConfig, :adaptor_registry, fn -> - [local_adaptors_repos: []] - end) - - refute AdaptorRegistry.local_adaptors_enabled?() - end - - test "returns false when the key is absent" do - Mox.stub(Lightning.MockConfig, :adaptor_registry, fn -> [] end) - - refute AdaptorRegistry.local_adaptors_enabled?() - end - end - - describe "exists?/2" do - setup do - file_path = - Briefly.create!(extname: ".json") - |> tap(fn path -> - File.write!(path, ~S""" - [ - { - "latest": "1.6.2", - "name": "@openfn/language-common", - "repo": "git+https://github.com/openfn/language-common.git", - "versions": [] - }, - { - "latest": "3.0.5", - "name": "@openfn/language-dhis2", - "repo": "git+https://github.com/openfn/language-dhis2.git", - "versions": [] - } - ] - """) - end) - - registry = - start_supervised!( - {AdaptorRegistry, [name: :exists_registry, use_cache: file_path]} - ) - - {:ok, registry: registry} - end - - test "returns true for a package present in the registry" do - assert AdaptorRegistry.exists?(:exists_registry, "@openfn/language-common") - assert AdaptorRegistry.exists?(:exists_registry, "@openfn/language-dhis2") - end - - test "returns false for a package absent from the registry" do - refute AdaptorRegistry.exists?(:exists_registry, "@openfn/language-http") - end - - test "returns false for nil" do - refute AdaptorRegistry.exists?(:exists_registry, nil) - end - end - - describe "exists?/2 in local mode" do - @tag :tmp_dir - test "returns true for a locally-listed adaptor", %{ - tmp_dir: tmp_dir, - test: test - } do - adaptor_name = "locally-listed" - [tmp_dir, "packages", adaptor_name] |> Path.join() |> File.mkdir_p!() - - start_supervised!( - {AdaptorRegistry, [name: test, local_adaptors_repos: [tmp_dir]]} - ) - - assert AdaptorRegistry.exists?(test, "@openfn/language-#{adaptor_name}") - refute AdaptorRegistry.exists?(test, "@openfn/language-common") - end - end - - describe "resolve_package_name/1" do - test "it can split an NPM style package name" do - assert AdaptorRegistry.resolve_package_name("@openfn/language-foo@1.2.3") == - {"@openfn/language-foo", "1.2.3"} - - assert AdaptorRegistry.resolve_package_name( - "@openfn/language-foo@1.2.3-pre" - ) == - {"@openfn/language-foo", "1.2.3-pre"} - - assert AdaptorRegistry.resolve_package_name("@openfn/language-foo") == - {"@openfn/language-foo", nil} - - assert AdaptorRegistry.resolve_package_name("") == - {nil, nil} - end - - test "returns {nil, nil} for strings that aren't clean npm package names" do - for bad <- [ - "@openfn/x\npwd\nb@1.0.0", - "@openfn/language-http@1.0.0; touch /tmp/x", - "@openfn/language-common@latest and stuff", - " @openfn/language-http", - "$(whoami)" - ] do - assert AdaptorRegistry.resolve_package_name(bad) == {nil, nil}, - "expected #{inspect(bad)} to be rejected" - end - end - - @tag :tmp_dir - test "returns local as the version when local_adaptors_repos config is set", - %{tmp_dir: tmp_dir} do - Mox.stub(Lightning.MockConfig, :adaptor_registry, fn -> - [local_adaptors_repos: [tmp_dir]] - end) - - assert AdaptorRegistry.resolve_package_name("@openfn/language-foo@1.2.3") == - {"@openfn/language-foo", "local"} - - assert AdaptorRegistry.resolve_package_name("@openfn/language-foo") == - {"@openfn/language-foo", "local"} - - assert AdaptorRegistry.resolve_package_name("") == - {nil, nil} - end - end -end diff --git a/test/lightning/adaptor_service_test.exs b/test/lightning/adaptor_service_test.exs deleted file mode 100644 index e845dbfffe1..00000000000 --- a/test/lightning/adaptor_service_test.exs +++ /dev/null @@ -1,121 +0,0 @@ -defmodule Lightning.AdaptorServiceTest do - use Lightning.DataCase, async: false - - import ExUnit.CaptureLog - - alias Lightning.AdaptorRegistry - alias Lightning.AdaptorService - alias Lightning.AdaptorService.Adaptor - - @permitted "@openfn/language-adaptor-service-test" - - defmodule StubRepo do - @moduledoc false - alias Lightning.AdaptorService.Adaptor - - @present [ - %Adaptor{ - name: "@openfn/language-adaptor-service-test", - version: "1.0.0", - path: "/fake/path", - local_name: "@openfn/language-adaptor-service-test", - status: :present - } - ] - - def list_local(_path), do: @present - def list_local(_path, _depth), do: @present - - def install(_aliased_name, _dir), do: {"", 0} - end - - describe "Repo.install/2" do - @tag :tmp_dir - test "is not vulnerable to shell injection", %{tmp_dir: dir} do - marker = Path.join(dir, "pwned") - - Lightning.AdaptorService.Repo.install( - ["bogus-#{System.unique_integer([:positive])} > #{marker}"], - dir - ) - - refute File.exists?(marker) - end - end - - describe "AdaptorService.install/2 allowlist" do - setup do - cache = - Briefly.create!(extname: ".json") - |> tap(fn path -> - File.write!( - path, - Jason.encode!([ - %{ - name: @permitted, - latest: "1.0.0", - repo: "git+https://example.com/test.git", - versions: [] - } - ]) - ) - end) - - start_supervised!( - {AdaptorRegistry, name: :test_asvc_registry, use_cache: cache} - ) - - start_supervised!( - {AdaptorService, - name: :test_adaptor_service, - adaptors_path: "/tmp/fake", - repo: StubRepo, - adaptor_registry: :test_asvc_registry} - ) - - :ok - end - - test "refuses a non-permitted adaptor" do - log = - capture_log(fn -> - assert AdaptorService.install( - :test_adaptor_service, - "@openfn/language-http@1.0.0" - ) == - {:error, :adaptor_not_permitted} - end) - - assert log =~ - "Refusing to install non-permitted adaptor: \"@openfn/language-http\"" - end - - test "permits an adaptor present in the registry and already on disk" do - assert {:ok, %Adaptor{name: @permitted}} = - AdaptorService.install(:test_adaptor_service, @permitted) - end - end - - describe "resolve_package_name/1" do - test "splits a well-formed package string" do - assert AdaptorService.resolve_package_name("@openfn/language-http@1.2.3") == - {"@openfn/language-http", "1.2.3"} - - assert AdaptorService.resolve_package_name("@openfn/language-http") == - {"@openfn/language-http", nil} - end - - test "returns {nil, nil} for malformed / injection-shaped strings, not raising" do - for bad <- [ - "@openfn/x\npwd\nb@1.0.0", - "@openfn/language-http@1.0.0; touch /tmp/x", - "@openfn/language-common@latest and stuff", - "$(whoami)", - "" - ] do - assert AdaptorService.resolve_package_name(bad) == {nil, nil}, - "expected #{inspect(bad)} to be rejected" - end - end - end -end diff --git a/test/lightning/adaptors/local_test.exs b/test/lightning/adaptors/local_test.exs index 1fc69bb4e91..8164a1403ec 100644 --- a/test/lightning/adaptors/local_test.exs +++ b/test/lightning/adaptors/local_test.exs @@ -6,16 +6,10 @@ defmodule Lightning.Adaptors.LocalTest do alias Lightning.Adaptors.Local setup do - root = - Path.join( - System.tmp_dir!(), - "lightning_adaptors_local_test_#{System.unique_integer([:positive])}" - ) - - File.mkdir_p!(Path.join(root, "packages")) + root = new_root!() original = Application.get_env(:lightning, Local, :__unset__) - Application.put_env(:lightning, Local, path: root) + Application.put_env(:lightning, Local, paths: [root]) on_exit(fn -> case original do @@ -58,8 +52,12 @@ defmodule Lightning.Adaptors.LocalTest do write_package!(root, "http-2", "@openfn/language-http", "2.3.4") write_package!(root, "http-3", "@openfn/language-http", "2.3.1") + {result, log} = with_log(fn -> Local.list_adaptors() end) + assert {:ok, [%{name: "@openfn/language-http", latest_version: "2.3.4"}]} = - Local.list_adaptors() + result + + refute log =~ "shadowed" end test "skips a directory with a missing package.json and logs a warning", @@ -108,6 +106,109 @@ defmodule Lightning.Adaptors.LocalTest do assert Local.list_adaptors() == {:error, :no_repo_path} end) =~ "not configured" end + + test "returns {:error, :no_repo_path} and logs a warning when :paths is not a list" do + Application.put_env(:lightning, Local, paths: "/mnt/a,/mnt/b") + + assert capture_log(fn -> + assert Local.list_adaptors() == {:error, :no_repo_path} + end) =~ "not configured" + end + end + + describe "multi-directory support" do + test "lists adaptors found across every configured path", %{root: root} do + root2 = new_root!() + on_exit(fn -> File.rm_rf!(root2) end) + Application.put_env(:lightning, Local, paths: [root, root2]) + + write_package!(root, "http", "@openfn/language-http", "1.0.0") + + write_package!( + root2, + "sf", + "@openfn/language-salesforce", + "2.0.0" + ) + + {:ok, listing} = Local.list_adaptors() + + assert Enum.sort_by(listing, & &1.name) == [ + %{name: "@openfn/language-http", latest_version: "1.0.0"}, + %{name: "@openfn/language-salesforce", latest_version: "2.0.0"} + ] + end + + test "resolves a package that only exists in the second configured path", + %{root: root} do + root2 = new_root!() + on_exit(fn -> File.rm_rf!(root2) end) + Application.put_env(:lightning, Local, paths: [root, root2]) + + write_package!(root2, "sf", "@openfn/language-salesforce", "3.2.1") + + assert {:ok, + %{name: "@openfn/language-salesforce", latest_version: "3.2.1"}} = + Local.fetch_adaptor("@openfn/language-salesforce") + end + + test "the first configured path wins on a name collision and only its versions are kept", + %{root: root} do + root2 = new_root!() + on_exit(fn -> File.rm_rf!(root2) end) + Application.put_env(:lightning, Local, paths: [root, root2]) + + write_package!(root, "http", "@openfn/language-http", "1.0.0") + write_package!(root2, "http", "@openfn/language-http", "9.9.9") + + {result, log} = + with_log(fn -> Local.fetch_adaptor("@openfn/language-http") end) + + assert {:ok, record} = result + assert record.latest_version == "1.0.0" + assert [%{version: "1.0.0"}] = record.versions + + assert log =~ "shadowed" + assert log =~ "@openfn/language-http" + end + + test "logs a single warning naming every shadowed package, not one per collision", + %{root: root} do + root2 = new_root!() + on_exit(fn -> File.rm_rf!(root2) end) + Application.put_env(:lightning, Local, paths: [root, root2]) + + write_package!(root, "http", "@openfn/language-http", "1.0.0") + write_package!(root2, "http-dup", "@openfn/language-http", "1.0.1") + write_package!(root, "sf", "@openfn/language-salesforce", "1.0.0") + write_package!(root2, "sf-dup", "@openfn/language-salesforce", "1.0.1") + + {_result, log} = with_log(fn -> Local.list_adaptors() end) + + assert log =~ "@openfn/language-http" + assert log =~ "@openfn/language-salesforce" + + assert length(String.split(log, "shadowed duplicate")) == 2 + end + + test "logs a warning and skips a root whose packages dir is missing, still returning the other roots", + %{root: root} do + missing_root = + Path.join( + System.tmp_dir!(), + "lightning_adaptors_local_test_missing_#{System.unique_integer([:positive])}" + ) + + Application.put_env(:lightning, Local, paths: [root, missing_root]) + + write_package!(root, "http", "@openfn/language-http", "1.0.0") + + {result, log} = with_log(fn -> Local.list_adaptors() end) + + assert {:ok, [%{name: "@openfn/language-http"}]} = result + assert log =~ "skipping" + assert log =~ missing_root + end end describe "fetch_adaptor/1" do @@ -345,6 +446,17 @@ defmodule Lightning.Adaptors.LocalTest do write_package_raw!(root, dir_name, %{"name" => name, "version" => version}) end + defp new_root! do + root = + Path.join( + System.tmp_dir!(), + "lightning_adaptors_local_test_#{System.unique_integer([:positive])}" + ) + + File.mkdir_p!(Path.join(root, "packages")) + root + end + defp write_package_raw!(root, dir_name, package_json) do dir = Path.join([root, "packages", dir_name]) File.mkdir_p!(dir) diff --git a/test/lightning/adaptors/repo_test.exs b/test/lightning/adaptors/repo_test.exs index c56c9ec8162..ee41cf514db 100644 --- a/test/lightning/adaptors/repo_test.exs +++ b/test/lightning/adaptors/repo_test.exs @@ -202,36 +202,6 @@ defmodule Lightning.Adaptors.RepoTest do end end - describe "stalest/2 (§12.2)" do - test "orders by :checked_at ascending" do - base = DateTime.utc_now() - - seed_adaptor(name: "@openfn/a", checked_at: DateTime.add(base, -300)) - seed_adaptor(name: "@openfn/b", checked_at: DateTime.add(base, -100)) - seed_adaptor(name: "@openfn/c", checked_at: DateTime.add(base, -200)) - - assert AdaptorRepo.stalest(10, :npm) |> Enum.map(& &1.name) == - ["@openfn/a", "@openfn/c", "@openfn/b"] - end - - test "honours the limit" do - base = DateTime.utc_now() - seed_adaptor(name: "@openfn/a", checked_at: DateTime.add(base, -300)) - seed_adaptor(name: "@openfn/b", checked_at: DateTime.add(base, -200)) - seed_adaptor(name: "@openfn/c", checked_at: DateTime.add(base, -100)) - - assert length(AdaptorRepo.stalest(2, :npm)) == 2 - end - - test "filters by source" do - seed_adaptor(name: "@openfn/a", source: :npm) - seed_adaptor(name: "@openfn/a", source: :local) - - assert [%Adaptor{source: :npm}] = AdaptorRepo.stalest(10, :npm) - assert [%Adaptor{source: :local}] = AdaptorRepo.stalest(10, :local) - end - end - describe "max_checked_at/1" do test "returns the largest :checked_at for the given source" do base = DateTime.utc_now() diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index 7d52228b327..737a73d26a1 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -200,6 +200,106 @@ defmodule Lightning.AdaptorsTest do end end + describe "get_adaptor/1" do + test "returns a Package for an adaptor in the active source" do + {:ok, _} = + AdaptorsRepo.upsert_adaptor(adaptor_record(latest_version: "4.1.0")) + + assert %Adaptors.Package{ + name: "@openfn/language-http", + source: :npm, + latest_version: "4.1.0" + } = Adaptors.get_adaptor("@openfn/language-http") + end + + test "returns nil for an adaptor absent from the catalogue" do + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + + assert Adaptors.get_adaptor("@openfn/never-existed") == nil + end + + test "returns nil when the catalogue is empty" do + assert Adaptors.get_adaptor("@openfn/language-http") == nil + end + + test "returns nil for a row under a different source than the active one" do + {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record(source: :local)) + + assert Adaptors.get_adaptor("@openfn/language-http") == nil + end + end + + describe "to_wire/1" do + test "delegates to PackageName.to_wire/1" do + {:ok, _} = + AdaptorsRepo.upsert_adaptor(adaptor_record(latest_version: "2.0.0")) + + assert Adaptors.to_wire("@openfn/language-http@latest") == + "@openfn/language-http@2.0.0" + + assert Adaptors.to_wire("@openfn/language-http@1.0.0") == + "@openfn/language-http@1.0.0" + + assert Adaptors.to_wire(nil) == "" + end + end + + describe "parse_spec/1" do + test "splits a spec carrying a version" do + assert Adaptors.parse_spec("@openfn/language-http@1.2.3") == + {"@openfn/language-http", "1.2.3"} + + assert Adaptors.parse_spec("@openfn/language-http@latest") == + {"@openfn/language-http", "latest"} + + assert Adaptors.parse_spec("common@1.0.0") == {"common", "1.0.0"} + end + + test "returns a nil version for a spec without one" do + assert Adaptors.parse_spec("@openfn/language-http") == + {"@openfn/language-http", nil} + end + + test "returns {nil, nil} for a string that isn't a well-formed spec" do + assert Adaptors.parse_spec("@openfn/language-http; rm -rf /") == + {nil, nil} + + assert Adaptors.parse_spec("@openfn/x\npwd\nb@1.0.0") == {nil, nil} + assert Adaptors.parse_spec("") == {nil, nil} + end + end + + describe "valid_format?/1" do + test "true for well-formed specs" do + [ + "@openfn/language-http", + "@openfn/language-http@1.2.3", + "@openfn/language-http@1.2.3-pre", + "@openfn/language-http@latest", + "@openfn/language-http@local", + "common", + "common@1.0.0" + ] + |> Enum.each(fn spec -> + assert Adaptors.valid_format?(spec), "expected #{inspect(spec)} to pass" + end) + end + + test "false for malformed / injection-shaped strings" do + [ + "@openfn/x\npwd\nb@1.0.0", + "@openfn/language-http@7.3.2; touch /tmp/x", + "@openfn/language-common@latest and stuff", + "@openfn/a/b/c@1.0.0", + "" + ] + |> Enum.each(fn spec -> + refute Adaptors.valid_format?(spec), + "expected #{inspect(spec)} to be rejected" + end) + end + end + describe "refresh_now/1" do test "delegates to Scheduler.refresh_now via global_scheduler_name/1", %{ sup: sup diff --git a/test/lightning/ai_assistant/ai_assistant_test.exs b/test/lightning/ai_assistant/ai_assistant_test.exs index 3d759dd0ab1..99816ed8c5a 100644 --- a/test/lightning/ai_assistant/ai_assistant_test.exs +++ b/test/lightning/ai_assistant/ai_assistant_test.exs @@ -451,7 +451,7 @@ defmodule Lightning.AiAssistantTest do assert session.expression == job_1.body assert session.adaptor == - Lightning.Adaptors.PackageName.to_wire(job_1.adaptor) + Lightning.Adaptors.to_wire(job_1.adaptor) assert length(session.messages) == 1 message = hd(session.messages) @@ -979,7 +979,7 @@ defmodule Lightning.AiAssistantTest do assert updated_session.expression == expression assert updated_session.adaptor == - Lightning.Adaptors.PackageName.to_wire(adaptor) + Lightning.Adaptors.to_wire(adaptor) end end @@ -1103,7 +1103,7 @@ defmodule Lightning.AiAssistantTest do assert enriched.expression == job.body assert enriched.adaptor == - Lightning.Adaptors.PackageName.to_wire(job.adaptor) + Lightning.Adaptors.to_wire(job.adaptor) end test "adds run logs when follow_run_id is in meta", %{ @@ -1269,9 +1269,7 @@ defmodule Lightning.AiAssistantTest do assert enriched.expression == "console.log('test');" assert enriched.adaptor == - Lightning.Adaptors.PackageName.to_wire( - "@openfn/language-http@latest" - ) + Lightning.Adaptors.to_wire("@openfn/language-http@latest") end test "fetches logs when follow_run_id is added mid-session", %{ diff --git a/test/lightning/ai_assistant/unsaved_job_test.exs b/test/lightning/ai_assistant/unsaved_job_test.exs index 85948f688f2..17d5ef94c89 100644 --- a/test/lightning/ai_assistant/unsaved_job_test.exs +++ b/test/lightning/ai_assistant/unsaved_job_test.exs @@ -68,7 +68,7 @@ defmodule Lightning.AiAssistant.UnsavedJobTest do enriched_session = AiAssistant.enrich_session_with_job_context(session) assert enriched_session.expression == "fn(state => state);" - # AdaptorRegistry.resolve_adaptor returns versioned adaptor + # PackageName.to_wire resolves "latest" to a versioned adaptor assert String.starts_with?( enriched_session.adaptor, "@openfn/language-http" diff --git a/test/lightning/config/bootstrap_test.exs b/test/lightning/config/bootstrap_test.exs index f97fa992aff..f259dd9de98 100644 --- a/test/lightning/config/bootstrap_test.exs +++ b/test/lightning/config/bootstrap_test.exs @@ -4,6 +4,7 @@ defmodule Lightning.Config.BootstrapTest do alias Lightning.Config.Bootstrap import Mox + import ExUnit.CaptureLog setup :verify_on_exit! @opts_key {Config, :opts} @@ -603,25 +604,26 @@ defmodule Lightning.Config.BootstrapTest do end describe "adaptors NPM upstream URLs" do - test "default to the production upstreams when nothing is set" do + test "no keys are forced when nothing is set, so sub-modules' own @default_* wins" do Dotenvy.source([%{}]) Bootstrap.configure() npm = get_env(:lightning, Lightning.Adaptors.NPM) - assert npm[:registry_url] == "https://registry.npmjs.org" - assert npm[:jsdelivr_url] == "https://cdn.jsdelivr.net" - assert npm[:github_url] == "https://raw.githubusercontent.com" - assert npm[:github_ref] == "main" + refute Keyword.has_key?(npm, :registry_url) + refute Keyword.has_key?(npm, :jsdelivr_url) + refute Keyword.has_key?(npm, :github_url) + refute Keyword.has_key?(npm, :github_ref) + refute Keyword.has_key?(npm, :http_timeout) end - test "are overridden by the ADAPTOR_* env vars" do + test "are overridden by the ADAPTORS_NPM_* env vars" do Dotenvy.source([ %{ - "ADAPTOR_REGISTRY_URL" => "http://localhost:4874/npm", - "ADAPTOR_JSDELIVR_URL" => "http://localhost:4874/jsdelivr", - "ADAPTOR_GITHUB_URL" => "http://localhost:4874/github", - "ADAPTOR_GITHUB_REF" => "some-feature-branch" + "ADAPTORS_NPM_REGISTRY_URL" => "http://localhost:4874/npm", + "ADAPTORS_NPM_JSDELIVR_URL" => "http://localhost:4874/jsdelivr", + "ADAPTORS_NPM_GITHUB_URL" => "http://localhost:4874/github", + "ADAPTORS_NPM_GITHUB_REF" => "some-feature-branch" } ]) @@ -634,6 +636,235 @@ defmodule Lightning.Config.BootstrapTest do assert npm[:github_url] == "http://localhost:4874/github" assert npm[:github_ref] == "some-feature-branch" end + + test "ADAPTORS_NPM_HTTP_TIMEOUT sets http_timeout when present" do + Dotenvy.source([%{"ADAPTORS_NPM_HTTP_TIMEOUT" => "5000"}]) + + Bootstrap.configure() + + npm = get_env(:lightning, Lightning.Adaptors.NPM) + + assert npm[:http_timeout] == 5000 + end + + test "ADAPTORS_NPM_HTTP_TIMEOUT does not force a 0ms timeout when set but empty" do + Dotenvy.source([%{"ADAPTORS_NPM_HTTP_TIMEOUT" => ""}]) + + Bootstrap.configure() + + npm = get_env(:lightning, Lightning.Adaptors.NPM) + + refute Keyword.has_key?(npm, :http_timeout) + end + end + + describe "adaptors strategy" do + test "defaults to the npm strategy when nothing is set" do + Dotenvy.source([%{}]) + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors)[:strategy] == + Lightning.Adaptors.NPM + end + + test "ADAPTORS_STRATEGY=npm explicitly selects the npm strategy" do + Dotenvy.source([%{"ADAPTORS_STRATEGY" => "npm"}]) + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors)[:strategy] == + Lightning.Adaptors.NPM + end + + test "ADAPTORS_STRATEGY=local selects the local strategy" do + Dotenvy.source([ + %{"ADAPTORS_STRATEGY" => "local", "ADAPTORS_LOCAL_REPO" => "/path"} + ]) + + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors)[:strategy] == + Lightning.Adaptors.Local + end + + test "ADAPTORS_STRATEGY is trimmed of surrounding whitespace" do + Dotenvy.source([ + %{"ADAPTORS_STRATEGY" => " local ", "ADAPTORS_LOCAL_REPO" => "/path"} + ]) + + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors)[:strategy] == + Lightning.Adaptors.Local + end + + test "ADAPTORS_STRATEGY=local with no repo path configured raises" do + assert_raise RuntimeError, + ~r/ADAPTORS_STRATEGY is set to local, but neither ADAPTORS_LOCAL_REPO nor/, + fn -> + Dotenvy.source([%{"ADAPTORS_STRATEGY" => "local"}]) + + Bootstrap.configure() + end + end + + test "does not write :strategy into the real application env in :test, so config/test.exs's mock survives" do + Process.put({Config, :opts}, {:test, ""}) + + Dotenvy.source([%{}]) + Bootstrap.configure() + + refute Keyword.has_key?( + get_env(:lightning, Lightning.Adaptors), + :strategy + ) + end + + test "raises when ADAPTORS_STRATEGY is not npm or local" do + assert_raise RuntimeError, ~r/ADAPTORS_STRATEGY/, fn -> + Dotenvy.source([%{"ADAPTORS_STRATEGY" => "bogus"}]) + Bootstrap.configure() + end + end + + test "LOCAL_ADAPTORS=true with ADAPTORS_STRATEGY unset back-compats to the local strategy and warns" do + log = + capture_log(fn -> + Dotenvy.source([ + %{"LOCAL_ADAPTORS" => "true", "OPENFN_ADAPTORS_REPO" => "/path"} + ]) + + Bootstrap.configure() + end) + + assert get_env(:lightning, Lightning.Adaptors)[:strategy] == + Lightning.Adaptors.Local + + assert log =~ "ADAPTORS_STRATEGY=local" + end + + test "ADAPTORS_STRATEGY explicitly set wins over the LOCAL_ADAPTORS back-compat" do + log = + capture_log(fn -> + Dotenvy.source([ + %{ + "LOCAL_ADAPTORS" => "true", + "OPENFN_ADAPTORS_REPO" => "/path", + "ADAPTORS_STRATEGY" => "npm" + } + ]) + + Bootstrap.configure() + end) + + assert get_env(:lightning, Lightning.Adaptors)[:strategy] == + Lightning.Adaptors.NPM + + refute log =~ "ADAPTORS_STRATEGY=local" + end + end + + describe "adaptors icons path" do + test "is unset when ADAPTORS_ICONS_PATH is not set" do + Dotenvy.source([%{}]) + Bootstrap.configure() + + # A present-but-nil :icon_path would defeat + # Lightning.Adaptors.Config's own default, so the key must be absent + # entirely, not just nil. + refute Keyword.has_key?( + get_env(:lightning, Lightning.Adaptors), + :icon_path + ) + end + + test "ADAPTORS_ICONS_PATH sets and expands the icon path" do + Dotenvy.source([%{"ADAPTORS_ICONS_PATH" => "./tmp/icons"}]) + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors)[:icon_path] == + Path.expand("./tmp/icons") + end + end + + describe "adaptors local strategy repo paths" do + test "defaults to an empty list when nothing is set" do + Dotenvy.source([%{}]) + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == [] + end + + test "ADAPTORS_LOCAL_REPO parses a comma-separated list" do + Dotenvy.source([%{"ADAPTORS_LOCAL_REPO" => "/a,/b"}]) + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == [ + "/a", + "/b" + ] + end + + test "OPENFN_ADAPTORS_REPO back-compats to the local strategy paths and warns when ADAPTORS_STRATEGY=local" do + log = + capture_log(fn -> + Dotenvy.source([ + %{ + "OPENFN_ADAPTORS_REPO" => "/path", + "ADAPTORS_STRATEGY" => "local" + } + ]) + + Bootstrap.configure() + end) + + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == ["/path"] + assert log =~ "ADAPTORS_LOCAL_REPO" + end + + test "a blank ADAPTORS_LOCAL_REPO falls back to OPENFN_ADAPTORS_REPO instead of discarding it" do + log = + capture_log(fn -> + Dotenvy.source([ + %{ + "OPENFN_ADAPTORS_REPO" => "/path", + "ADAPTORS_LOCAL_REPO" => " , ", + "ADAPTORS_STRATEGY" => "local" + } + ]) + + Bootstrap.configure() + end) + + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == ["/path"] + assert log =~ "ADAPTORS_LOCAL_REPO" + end + + test "ADAPTORS_LOCAL_REPO takes precedence over OPENFN_ADAPTORS_REPO" do + Dotenvy.source([ + %{ + "OPENFN_ADAPTORS_REPO" => "/old", + "ADAPTORS_LOCAL_REPO" => "/new" + } + ]) + + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == ["/new"] + end + + test "LOCAL_ADAPTORS=true and OPENFN_ADAPTORS_REPO dual-write both the old registry and the new Local strategy" do + Dotenvy.source([ + %{"LOCAL_ADAPTORS" => "true", "OPENFN_ADAPTORS_REPO" => "/path"} + ]) + + Bootstrap.configure() + + assert get_env(:lightning, Lightning.AdaptorRegistry)[ + :local_adaptors_repos + ] == ["/path"] + + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == ["/path"] + end end describe "per_workflow_claim_limit" do diff --git a/test/lightning/download_adaptor_registry_test.exs b/test/lightning/download_adaptor_registry_test.exs index 9224b9feb81..bfebd7a7f87 100644 --- a/test/lightning/download_adaptor_registry_test.exs +++ b/test/lightning/download_adaptor_registry_test.exs @@ -2,29 +2,55 @@ defmodule Lightning.DownloadAdaptorRegistryCacheTest do use ExUnit.Case, async: false import ExUnit.CaptureIO - import Mox - import Tesla.Test - - setup :set_mox_from_context - setup :verify_on_exit! alias Mix.Tasks.Lightning.DownloadAdaptorRegistryCache + @package "@openfn/language-http" + @latest_version "2.1.0" + + # Bypass servers for the npm registry and jsDelivr, installed onto the + # NPM strategy's own Application key. Matches the setup in + # test/lightning/adaptors/npm_test.exs. + setup do + registry = Bypass.open() + jsdelivr = Bypass.open() + + Application.put_env(:lightning, Lightning.Adaptors.NPM, + registry_url: "http://localhost:#{registry.port}", + jsdelivr_url: "http://localhost:#{jsdelivr.port}", + http_timeout: 1_000 + ) + + prev_adapter = Application.get_env(:tesla, :adapter) + + Application.put_env( + :tesla, + :adapter, + {Tesla.Adapter.Finch, name: Lightning.Finch} + ) + + on_exit(fn -> + Application.delete_env(:lightning, Lightning.Adaptors.NPM) + + if prev_adapter do + Application.put_env(:tesla, :adapter, prev_adapter) + else + Application.delete_env(:tesla, :adapter) + end + end) + + %{registry: registry, jsdelivr: jsdelivr} + end + describe "download_adaptor_registry_cache mix task" do @describetag :tmp_dir - test "does not write file when no adaptors are found", %{tmp_dir: tmp_dir} do - expect_tesla_call( - times: 1, - returns: fn env, [] -> - case env.url do - "https://registry.npmjs.org/-/user/openfn/package" -> - {:ok, json(%Tesla.Env{status: 200}, [])} - - "https://registry.npmjs.org/@openfn/language-asana" -> - {:ok, json(%Tesla.Env{status: 200}, [])} - end - end - ) + test "does not write file when no adaptors are found", %{ + tmp_dir: tmp_dir, + registry: registry + } do + Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> + json_resp(conn, 200, %{"objects" => []}) + end) file_path = Path.join([tmp_dir, "cache.json"]) refute File.exists?(file_path) @@ -36,26 +62,28 @@ defmodule Lightning.DownloadAdaptorRegistryCacheTest do refute File.exists?(file_path) end - test "writes to specified file", %{tmp_dir: tmp_dir} do - language_common_response = - File.read!("test/fixtures/language-common-npm.json") |> Jason.decode!() - - expect_tesla_call( - times: 7, - returns: fn env, [] -> - case env.url do - "https://registry.npmjs.org/-/user/openfn/package" -> - {:ok, - json( - %Tesla.Env{status: 200}, - File.read!("test/fixtures/openfn-packages-npm.json") - |> Jason.decode!() - )} - - "https://registry.npmjs.org/@openfn/" <> _adaptor -> - {:ok, json(%Tesla.Env{status: 200}, language_common_response)} - end - end + test "writes full adaptor records to the specified file", %{ + tmp_dir: tmp_dir, + registry: registry, + jsdelivr: jsdelivr + } do + Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> + json_resp(conn, 200, %{ + "objects" => [ + %{"package" => %{"name" => @package, "version" => @latest_version}} + ] + }) + end) + + Bypass.expect(registry, "GET", "/" <> @package, fn conn -> + json_resp(conn, 200, build_packument()) + end) + + Bypass.expect( + jsdelivr, + "GET", + "/npm/#{@package}@#{@latest_version}/configuration-schema.json", + fn conn -> Plug.Conn.resp(conn, 200, "{}") end ) file_path = Path.join([tmp_dir, "cache.json"]) @@ -65,7 +93,39 @@ defmodule Lightning.DownloadAdaptorRegistryCacheTest do DownloadAdaptorRegistryCache.run(["--path", file_path]) end) - assert File.exists?(file_path) + assert [record] = + file_path |> File.read!() |> Jason.decode!(keys: :atoms) + + assert %{ + name: @package, + source: "npm", + latest_version: @latest_version, + versions: [%{version: @latest_version}] + } = record end end + + defp build_packument do + %{ + "name" => @package, + "description" => "HTTP adaptor", + "repository" => %{"url" => "git+https://github.com/OpenFn/adaptors.git"}, + "license" => "LGPL-3.0", + "dist-tags" => %{"latest" => @latest_version}, + "time" => %{@latest_version => "2024-06-01T12:00:00.000Z"}, + "versions" => %{ + @latest_version => %{ + "dependencies" => %{}, + "peerDependencies" => %{}, + "dist" => %{"integrity" => "sha512-abc", "unpackedSize" => 12_345} + } + } + } + end + + defp json_resp(conn, status, body) do + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.resp(status, Jason.encode!(body)) + end end diff --git a/test/lightning/jobs_test.exs b/test/lightning/jobs_test.exs index 2b5eb5a2dc6..72a7ce010b6 100644 --- a/test/lightning/jobs_test.exs +++ b/test/lightning/jobs_test.exs @@ -192,6 +192,8 @@ defmodule Lightning.JobsTest do describe "create_job/2" do setup do + Lightning.AdaptorTestHelpers.ensure_adaptor("@openfn/language-common") + %{actor: insert(:user)} end diff --git a/test/lightning/metadata_service_test.exs b/test/lightning/metadata_service_test.exs index f1ac4f73760..be8e3646c06 100644 --- a/test/lightning/metadata_service_test.exs +++ b/test/lightning/metadata_service_test.exs @@ -3,6 +3,13 @@ defmodule Lightning.MetadataServiceTest do alias Lightning.MetadataService + # Seeds the one name the "succeeds" cases below use; the "not in the + # registry" cases rely on their name staying unseeded. + setup do + insert(:adaptor, name: "@openfn/language-common") + :ok + end + describe "fetch/2" do test "returns the metadata when it exists" do path = @@ -174,6 +181,20 @@ defmodule Lightning.MetadataServiceTest do } end + test "returns an error, rather than raising, for a malformed adaptor string" do + credential = + insert(:credential) + |> with_body(%{name: "main", body: %{"username" => "user"}}) + + assert MetadataService.fetch("not a valid package!!", credential) == { + :error, + %Lightning.MetadataService.Error{ + type: "no_matching_adaptor", + __exception__: true + } + } + end + test "refuses a well-formed adaptor that is not in the registry (whitelist)" do path = Briefly.create!(extname: ".json") diff --git a/test/lightning/projects/provisioner_test.exs b/test/lightning/projects/provisioner_test.exs index 0d1446cf96c..d360e9e7f5e 100644 --- a/test/lightning/projects/provisioner_test.exs +++ b/test/lightning/projects/provisioner_test.exs @@ -169,6 +169,8 @@ defmodule Lightning.Projects.ProvisionerTest do end test "rejects a job with an adaptor that is not in the registry" do + insert(:adaptor, name: "@openfn/language-common") + %{body: body} = valid_document() body = diff --git a/test/lightning/setup_utils_test.exs b/test/lightning/setup_utils_test.exs index 2921608419b..5b1fbf110b5 100644 --- a/test/lightning/setup_utils_test.exs +++ b/test/lightning/setup_utils_test.exs @@ -8,6 +8,15 @@ defmodule Lightning.SetupUtilsTest do alias Lightning.Accounts.{User, UserToken} alias Lightning.Credentials.{Credential} + # The demo projects' jobs are built through `Job.changeset/2`, which only + # accepts adaptors present in the catalogue. + setup do + Enum.each( + ~w(@openfn/language-common @openfn/language-dhis2 @openfn/language-http), + &Lightning.AdaptorTestHelpers.ensure_adaptor/1 + ) + end + describe "Setup demo site seed data" do setup do Lightning.SetupUtils.setup_demo(create_super: true) diff --git a/test/lightning/workflows/job_test.exs b/test/lightning/workflows/job_test.exs index aaded90395a..cc7e5cad0d1 100644 --- a/test/lightning/workflows/job_test.exs +++ b/test/lightning/workflows/job_test.exs @@ -494,6 +494,9 @@ defmodule Lightning.Workflows.JobTest do end test "accepts well-formed, registry-listed adaptor strings" do + insert(:adaptor, name: "@openfn/language-common") + insert(:adaptor, name: "@openfn/language-http") + [ "@openfn/language-common@latest", "@openfn/language-http@1.2.3", @@ -502,12 +505,39 @@ defmodule Lightning.Workflows.JobTest do "@openfn/language-common" ] |> Enum.each(fn adaptor -> - errors = Job.changeset(%Job{}, %{adaptor: adaptor}) |> errors_on() + errors = + Job.changeset(%Job{}, %{ + name: "job", + body: "fn(state => state)", + adaptor: adaptor + }) + |> errors_on() + refute errors[:adaptor], "expected #{inspect(adaptor)} to be accepted" end) end + test "rejects an unrecognised adaptor whether or not the catalogue has rows for the active source" do + # `job.adaptor` reaches the worker's install step unfiltered, so an + # empty catalogue must permit nothing — not even an `@openfn/` name. + params = %{ + name: "job", + body: "fn(state => state)", + adaptor: "@openfn/language-totally-unseeded-xyz@1.0.0" + } + + assert Job.changeset(%Job{}, params) |> errors_on() |> Map.get(:adaptor) == + ["is not a recognised adaptor"] + + insert(:adaptor, name: "@openfn/language-http") + + assert Job.changeset(%Job{}, params) |> errors_on() |> Map.get(:adaptor) == + ["is not a recognised adaptor"] + end + test "rejects a well-formed adaptor that is not in the registry" do + insert(:adaptor, name: "@openfn/language-http") + # The registry membership check only runs on an otherwise-valid changeset, # so name and body are supplied here. [ diff --git a/test/lightning_web/channels/workflow_channel_broadcast_test.exs b/test/lightning_web/channels/workflow_channel_broadcast_test.exs index b69ab3db358..873cffce595 100644 --- a/test/lightning_web/channels/workflow_channel_broadcast_test.exs +++ b/test/lightning_web/channels/workflow_channel_broadcast_test.exs @@ -25,6 +25,8 @@ defmodule LightningWeb.WorkflowChannelBroadcastTest do # Stub the broadcast calls that save_workflow makes Mox.stub(LightningMock, :broadcast, fn _topic, _message -> :ok end) + Lightning.AdaptorTestHelpers.ensure_adaptor("@openfn/language-common") + user = insert(:user) project = insert(:project, project_users: [%{user: user, role: :owner}]) workflow = insert(:workflow, project: project) diff --git a/test/mix/tasks/seed_adaptors_from_file_test.exs b/test/mix/tasks/seed_adaptors_from_file_test.exs new file mode 100644 index 00000000000..c82dfe4113d --- /dev/null +++ b/test/mix/tasks/seed_adaptors_from_file_test.exs @@ -0,0 +1,149 @@ +defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFileTest do + use Lightning.DataCase + + import ExUnit.CaptureIO + + alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Mix.Tasks.Lightning.SeedAdaptorsFromFile + + @moduletag :tmp_dir + + defp write_snapshot(tmp_dir, records) do + path = Path.join(tmp_dir, "snapshot.json") + File.write!(path, Jason.encode_to_iodata!(records)) + path + end + + describe "run/1" do + test "upserts every record in the file into the given source", %{ + tmp_dir: tmp_dir + } do + path = + write_snapshot(tmp_dir, [ + %{ + name: "@openfn/language-http", + latest_version: "2.1.0", + versions: [%{version: "2.1.0"}, %{version: "2.0.0"}] + } + ]) + + capture_io(fn -> + SeedAdaptorsFromFile.run(["--path", path]) + end) + + assert %{latest_version: "2.1.0"} = + AdaptorsRepo.get_adaptor("@openfn/language-http", :npm) + + assert length(AdaptorsRepo.list_versions("@openfn/language-http", :npm)) == + 2 + end + + test "seeds the :local source when --source local is given", %{ + tmp_dir: tmp_dir + } do + path = + write_snapshot(tmp_dir, [ + %{ + name: "@openfn/language-common", + latest_version: "1.0.0", + versions: [] + } + ]) + + capture_io(fn -> + SeedAdaptorsFromFile.run(["--path", path, "--source", "local"]) + end) + + assert AdaptorsRepo.get_adaptor("@openfn/language-common", :npm) == nil + + assert %{source: :local} = + AdaptorsRepo.get_adaptor("@openfn/language-common", :local) + end + + test "--replace deletes existing rows for the source before seeding", %{ + tmp_dir: tmp_dir + } do + insert(:adaptor, name: "@openfn/language-stale", source: :npm) + + path = + write_snapshot(tmp_dir, [ + %{name: "@openfn/language-http", latest_version: "1.0.0", versions: []} + ]) + + capture_io(fn -> + SeedAdaptorsFromFile.run(["--path", path, "--replace"]) + end) + + assert AdaptorsRepo.get_adaptor("@openfn/language-stale", :npm) == nil + assert AdaptorsRepo.get_adaptor("@openfn/language-http", :npm) != nil + end + + test "--replace rolls back the delete when a later record fails to upsert", + %{tmp_dir: tmp_dir} do + insert(:adaptor, name: "@openfn/language-stale", source: :npm) + + path = + write_snapshot(tmp_dir, [ + %{ + name: "@openfn/language-http", + latest_version: "1.0.0", + versions: [] + }, + # Missing `latest_version`, required by the Adaptor changeset, so + # this fails inside `upsert_adaptor/1` rather than at normalize. + %{name: "@openfn/language-broken", versions: []} + ]) + + assert_raise ArgumentError, fn -> + capture_io(fn -> + SeedAdaptorsFromFile.run(["--path", path, "--replace"]) + end) + end + + assert AdaptorsRepo.get_adaptor("@openfn/language-stale", :npm) != nil + assert AdaptorsRepo.get_adaptor("@openfn/language-http", :npm) == nil + end + + test "round-trips a snapshot in the shape the download task emits", %{ + tmp_dir: tmp_dir + } do + # Same shape `mix lightning.download_adaptor_registry_cache` writes: + # an atom-keyed adaptor_record (see Lightning.Adaptors.Strategy) plus + # :source, run through Jason.encode_to_iodata!/1. + record = %{ + name: "@openfn/language-http", + description: "HTTP adaptor", + homepage: nil, + repository: "git+https://github.com/OpenFn/adaptors.git", + license: "LGPL-3.0", + latest_version: "2.1.0", + deprecated: false, + schema_data: nil, + schema_sha256: nil, + source: :npm, + versions: [ + %{ + version: "2.1.0", + integrity: "sha512-abc", + tarball_url: "https://example.com/http-2.1.0.tgz", + size_bytes: 12_345, + dependencies: %{"axios" => "^1.5.0"}, + peer_dependencies: %{}, + published_at: DateTime.utc_now(), + deprecated: false + } + ] + } + + path = tmp_dir |> Path.join("snapshot.json") + File.write!(path, Jason.encode_to_iodata!([record])) + + capture_io(fn -> + SeedAdaptorsFromFile.run(["--path", path]) + end) + + assert [%{name: "@openfn/language-http", versions: ["2.1.0"]}] = + Lightning.Adaptors.catalogue() + end + end +end diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex index 63bfd6a7f3b..b730402aca9 100644 --- a/test/support/adaptor_test_helpers.ex +++ b/test/support/adaptor_test_helpers.ex @@ -7,7 +7,11 @@ defmodule Lightning.AdaptorTestHelpers do application and is shared across the test suite: its Cachex persists across the `Ecto.Adapters.SQL.Sandbox` boundary, so tests that seed rows via the `:adaptor` factory (`insert(:adaptor, attrs)`) must - clear the cache to make them visible to facade reads. + clear the cache to make them visible to Cachex-backed facade reads + (`packages/1`, `schema/2`, `versions/2`, `icon/3`, via `Store`). + `get_adaptor/1` and `resolve_version/2` read `Repo` directly with no + Cachex in the path, so seeding for those (see `ensure_adaptor/1` + below) doesn't need a cache clear. For tests that run their own isolated supervisor instead of the production one, see `test/lightning/adaptors_test.exs` and @@ -42,6 +46,28 @@ defmodule Lightning.AdaptorTestHelpers do row end + @doc """ + Seed the catalogue row an adaptor spec needs to clear + `Lightning.Workflows.Job`'s validation, unless it's already there. + + Deliberately leaves the Cachex alone: the changeset check reads the DB + directly, and clearing a cache shared with other async tests would + disturb them. + """ + @spec ensure_adaptor(String.t()) :: :ok + def ensure_adaptor(spec) when is_binary(spec) do + case Lightning.Adaptors.parse_spec(spec) do + {name, _version} when is_binary(name) -> + if is_nil(Lightning.Adaptors.get_adaptor(name)), + do: insert(:adaptor, name: name) + + :ok + + _ -> + raise ArgumentError, "not a well-formed adaptor spec: #{inspect(spec)}" + end + end + @doc """ Seed a credential schema row keyed by short name (e.g. `"postgresql"`), reading the JSON body from `test/fixtures/schemas/.json`. diff --git a/test/support/fixtures/jobs_fixtures.ex b/test/support/fixtures/jobs_fixtures.ex index 2c3113e2225..9afe8c5afc7 100644 --- a/test/support/fixtures/jobs_fixtures.ex +++ b/test/support/fixtures/jobs_fixtures.ex @@ -23,15 +23,17 @@ defmodule Lightning.JobsFixtures do workflow_fixture(project_id: attrs[:project_id]).id end) - {:ok, job} = - attrs - |> Enum.into(%{ + attrs = + Enum.into(attrs, %{ body: "fn(state => state)", enabled: true, name: "some name", adaptor: "@openfn/language-common" }) - |> Lightning.Jobs.create_job(insert(:user)) + + Lightning.AdaptorTestHelpers.ensure_adaptor(attrs.adaptor) + + {:ok, job} = Lightning.Jobs.create_job(attrs, insert(:user)) job end diff --git a/tooling/adaptor_cache/README.md b/tooling/adaptor_cache/README.md index d65eb29d57d..8ff29f125fd 100644 --- a/tooling/adaptor_cache/README.md +++ b/tooling/adaptor_cache/README.md @@ -35,9 +35,9 @@ the full command list. Point Lightning at the cache by exporting these: ```sh -export ADAPTOR_REGISTRY_URL=http://localhost:4874/npm -export ADAPTOR_JSDELIVR_URL=http://localhost:4874/jsdelivr -export ADAPTOR_GITHUB_URL=http://localhost:4874/github +export ADAPTORS_NPM_REGISTRY_URL=http://localhost:4874/npm +export ADAPTORS_NPM_JSDELIVR_URL=http://localhost:4874/jsdelivr +export ADAPTORS_NPM_GITHUB_URL=http://localhost:4874/github ``` `bin/adaptor_cache up` prints these for you with the right port baked in, so you @@ -94,17 +94,17 @@ already builds full paths under each — the proxy just rewrites the host: | jsDelivr schema fetch (`schema.ex`) | `http://localhost:4874/jsdelivr/npm/@openfn/language-http@2.1.0/configuration-schema.json` | `/jsdelivr/` | `https://cdn.jsdelivr.net/...` | | GitHub icon fetch (`github.ex`) | `http://localhost:4874/github/OpenFn/adaptors/main/packages/http/assets/square.png` | `/github/` | `https://raw.githubusercontent.com/...` | -Setting `ADAPTOR_REGISTRY_URL`, `ADAPTOR_JSDELIVR_URL` and `ADAPTOR_GITHUB_URL` -to the proxy's `/npm`, `/jsdelivr` and `/github` base URLs is all that's needed -— the strategy code appends the same paths it always did, just against a -different host. +Setting `ADAPTORS_NPM_REGISTRY_URL`, `ADAPTORS_NPM_JSDELIVR_URL` and +`ADAPTORS_NPM_GITHUB_URL` to the proxy's `/npm`, `/jsdelivr` and `/github` base +URLs is all that's needed — the strategy code appends the same paths it always +did, just against a different host. ## Caveats - **The legacy `Lightning.AdaptorRegistry` and `mix lightning.install_schemas` bypass this entirely.** Both have hardcoded upstream URLs and don't read the - `ADAPTOR_*` env vars, so they'll always hit the real internet regardless of - whether the cache is up. + `ADAPTORS_NPM_*` env vars, so they'll always hit the real internet regardless + of whether the cache is up. - **Redirects bypass the cache.** Tesla's `FollowRedirects` middleware requests the absolute `Location` URL, which points at the real upstream even when the redirect is same-host, so anything that 30x's is fetched live. @@ -128,8 +128,8 @@ request usually means the upstream is refusing the request outright (check status code) rather than a caching problem. **Port already in use.** Set `ADAPTOR_CACHE_PORT` to something else before -`bin/adaptor_cache up`, and update the three `ADAPTOR_*_URL` exports to match -the new port. +`bin/adaptor_cache up`, and update the three `ADAPTORS_NPM_*_URL` exports to +match the new port. **Stale or wrong data cached.** `bin/adaptor_cache purge` drops the on-disk cache volume entirely (unlike `down`, which keeps it); `up` again to rebuild diff --git a/tooling/adaptor_cache/nginx.conf b/tooling/adaptor_cache/nginx.conf index b38d46210f7..c71b4614851 100644 --- a/tooling/adaptor_cache/nginx.conf +++ b/tooling/adaptor_cache/nginx.conf @@ -1,9 +1,9 @@ # Caching reverse proxy for the three Lightning.Adaptors.* upstreams. # # Point Lightning at it with: -# export ADAPTOR_REGISTRY_URL=http://localhost:4874/npm -# export ADAPTOR_JSDELIVR_URL=http://localhost:4874/jsdelivr -# export ADAPTOR_GITHUB_URL=http://localhost:4874/github +# export ADAPTORS_NPM_REGISTRY_URL=http://localhost:4874/npm +# export ADAPTORS_NPM_JSDELIVR_URL=http://localhost:4874/jsdelivr +# export ADAPTORS_NPM_GITHUB_URL=http://localhost:4874/github # # Workers run as root deliberately. The cache lives on a Docker named volume # mounted at /var/cache/nginx/adaptors, which Docker creates root-owned; the From 6750e0da8f6713aa6a363adcddf3a824a7b23c43 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Mon, 31 Aug 2026 16:56:46 +0200 Subject: [PATCH 04/37] Randomize the test-server port and rewrite bin/adaptor_cache - Test-server port randomized so mix test can run across parallel worktrees - bin/adaptor_cache rewritten as a self-contained record-and-replay proxy --- .credo.exs | 3 +- .formatter.exs | 4 +- RUNNINGLOCAL.md | 8 +- bin/adaptor_cache | 234 +---------- config/test.exs | 14 +- .../controllers/api/job_controller_test.exs | 6 +- .../api/project_controller_test.exs | 9 +- .../controllers/webhooks_controller_test.exs | 2 +- tooling/adaptor_cache/.gitignore | 1 + tooling/adaptor_cache/README.md | 133 +++--- tooling/adaptor_cache/docker-compose.yml | 25 -- tooling/adaptor_cache/lib/cache.ex | 89 ++++ tooling/adaptor_cache/lib/cli.ex | 393 ++++++++++++++++++ tooling/adaptor_cache/lib/publish.ex | 162 ++++++++ tooling/adaptor_cache/lib/router.ex | 126 ++++++ tooling/adaptor_cache/lib/scenario.ex | 57 +++ tooling/adaptor_cache/nginx.conf | 151 ------- 17 files changed, 953 insertions(+), 464 deletions(-) create mode 100644 tooling/adaptor_cache/.gitignore delete mode 100644 tooling/adaptor_cache/docker-compose.yml create mode 100644 tooling/adaptor_cache/lib/cache.ex create mode 100644 tooling/adaptor_cache/lib/cli.ex create mode 100644 tooling/adaptor_cache/lib/publish.ex create mode 100644 tooling/adaptor_cache/lib/router.ex create mode 100644 tooling/adaptor_cache/lib/scenario.ex delete mode 100644 tooling/adaptor_cache/nginx.conf diff --git a/.credo.exs b/.credo.exs index 2dfa19776b5..8f9a5eafdfa 100644 --- a/.credo.exs +++ b/.credo.exs @@ -30,7 +30,8 @@ "apps/*/lib/", "apps/*/src/", "apps/*/test/", - "apps/*/web/" + "apps/*/web/", + "tooling/adaptor_cache/lib/" ], excluded: [~r"/_build/", ~r"/deps/", ~r"/node_modules/"] }, diff --git a/.formatter.exs b/.formatter.exs index ac615d55520..581ee697e61 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -4,7 +4,9 @@ inputs: [ "*.{heex,ex,exs}", "priv/*/seeds.exs", - "{config,lib,test}/**/*.{heex,ex,exs}" + "{config,lib,test}/**/*.{heex,ex,exs}", + "tooling/adaptor_cache/lib/**/*.ex", + "bin/adaptor_cache" ], subdirectories: ["priv/*/migrations"], line_length: 81 diff --git a/RUNNINGLOCAL.md b/RUNNINGLOCAL.md index ae27966897d..1edb14ee0cf 100644 --- a/RUNNINGLOCAL.md +++ b/RUNNINGLOCAL.md @@ -214,12 +214,12 @@ next run! Every adaptor registry refresh (background scheduler tick, or a manual `mix lightning.refresh_adaptors`) makes a handful of npm, jsDelivr and GitHub requests per changed package, which gets chatty fast if you're iterating on the -subsystem or just running `refresh_adaptors` repeatedly by hand. A local caching -reverse proxy under `tooling/adaptor_cache/` makes the second and every later -run local, with no network needed at all. +subsystem or just running `refresh_adaptors` repeatedly by hand. A local +record-and-replay reverse proxy under `tooling/adaptor_cache/` makes the second +and every later run local, with no network needed at all. ```sh -bin/adaptor_cache up # start the proxy (first time must be online) +bin/adaptor_cache up # start the proxy bin/adaptor_cache check # prove all three upstreams cache correctly ``` diff --git a/bin/adaptor_cache b/bin/adaptor_cache index 53f20ff4158..c6536f781a8 100755 --- a/bin/adaptor_cache +++ b/bin/adaptor_cache @@ -1,233 +1,29 @@ -#!/usr/bin/env bash +#!/usr/bin/env elixir # ============================================================================= # Lightning Adaptor Cache # ============================================================================= # -# A local caching reverse proxy in front of the three upstreams the -# Lightning.Adaptors.* subsystem reads from: registry.npmjs.org, +# A self-contained record-and-replay reverse proxy in front of the three +# upstreams the Lightning.Adaptors.* subsystem reads from: registry.npmjs.org, # cdn.jsdelivr.net and raw.githubusercontent.com. See # tooling/adaptor_cache/README.md. # # A forward proxy would not work: Lightning talks to these upstreams through -# Tesla over the Finch adapter, and neither honours HTTP_PROXY/HTTPS_PROXY. -# Each upstream does have a configurable base URL, which is what this reverse -# proxy plugs into. -# +# Req/Finch, and neither honours HTTP_PROXY/HTTPS_PROXY. Each upstream does +# have a configurable base URL, which is what this reverse proxy plugs into. # ============================================================================= -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" - -COMPOSE_DIR="${PROJECT_ROOT}/tooling/adaptor_cache" -COMPOSE_FILE="${COMPOSE_DIR}/docker-compose.yml" - -# Fixed project name, not derived from the directory, so every git worktree -# shares one cache instead of each spinning up its own. -PROJECT_NAME="adaptor-cache" - -PORT="${ADAPTOR_CACHE_PORT:-4874}" -BASE_URL="http://localhost:${PORT}" - -compose() { - ADAPTOR_CACHE_PORT="${PORT}" \ - docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" "$@" -} - -show_help() { - cat <<'EOF' -Lightning Adaptor Cache - -A local caching reverse proxy in front of the three upstreams the -Lightning.Adaptors.* subsystem reads from: registry.npmjs.org, -cdn.jsdelivr.net and raw.githubusercontent.com. See -tooling/adaptor_cache/README.md. - -Usage: bin/adaptor_cache - - up Start the proxy and print the export lines - down Stop the proxy, keeping the cache on disk - status Show container state and reachability - purge Stop the proxy AND drop the cache volume - logs Tail the access log (cache=HIT / cache=MISS) - check Probe all three prefixes, prove MISS then HIT - --help This message - -Environment variables: - - ADAPTOR_CACHE_PORT Host port to bind (default: 4874) -EOF -} - -require_docker() { - if ! command -v docker >/dev/null 2>&1; then - echo "adaptor_cache: docker not found on PATH." >&2 - exit 1 - fi - - if ! docker compose version >/dev/null 2>&1; then - echo "adaptor_cache: 'docker compose' is not available." >&2 - exit 1 - fi - - if [[ ! -f "${COMPOSE_FILE}" ]]; then - echo "adaptor_cache: missing ${COMPOSE_FILE}" >&2 - exit 1 - fi -} - -wait_for_healthz() { - local _attempt - for _attempt in $(seq 1 30); do - if curl -fsS -o /dev/null --max-time 2 "${BASE_URL}/_healthz"; then - return 0 - fi - sleep 1 - done - - echo "adaptor_cache: proxy did not come up on ${BASE_URL} after 30s." >&2 - echo "adaptor_cache: run 'bin/adaptor_cache logs' to see why." >&2 - return 1 -} - -print_exports() { - cat <&1)"; then - printf ' %-9s FAIL could not reach %s\n' "${label}" "${url}" - printf ' %s\n' "${headers}" - return 1 - fi - - status="$(printf '%s' "${headers}" | tr -d '\r' | awk 'NR==1 {print $2}')" - cache_1="$(printf '%s' "${headers}" | tr -d '\r' \ - | awk -F': ' 'tolower($1)=="x-cache-status" {print $2}' | tail -1)" - - headers="$(curl -sS -o /dev/null -D - --max-time 30 "${url}")" - cache_2="$(printf '%s' "${headers}" | tr -d '\r' \ - | awk -F': ' 'tolower($1)=="x-cache-status" {print $2}' | tail -1)" - - printf ' %-9s %-4s first=%-12s second=%-12s %s\n' \ - "${label}" "${status:-???}" "${cache_1:--}" "${cache_2:--}" "${url}" - - if [[ "${cache_2}" != "HIT" ]]; then - printf ' ^ expected the second request to be a HIT\n' - return 1 - fi - - return 0 -} - -cmd_up() { - require_docker - compose up -d - wait_for_healthz - echo "adaptor_cache: up on ${BASE_URL}" - print_exports -} - -cmd_down() { - require_docker - compose down - echo "adaptor_cache: down. The cache volume is intact; 'purge' drops it." -} - -cmd_purge() { - require_docker - compose down -v - echo "adaptor_cache: down and cache volume removed." -} - -cmd_status() { - require_docker - compose ps - echo - if curl -fsS -o /dev/null --max-time 2 "${BASE_URL}/_healthz"; then - echo "adaptor_cache: reachable at ${BASE_URL}" - else - echo "adaptor_cache: NOT reachable at ${BASE_URL}" - fi -} - -cmd_logs() { - require_docker - if [[ $# -gt 0 ]]; then - compose logs "$@" - else - compose logs -f --tail 100 nginx - fi -} - -cmd_check() { - require_docker - - if ! curl -fsS -o /dev/null --max-time 2 "${BASE_URL}/_healthz"; then - echo "adaptor_cache: not running. Start it with 'bin/adaptor_cache up'." >&2 - exit 1 - fi - - echo "adaptor_cache: probing all three prefixes twice each" - echo - - local failed=0 - - probe npm "${BASE_URL}/npm/-/v1/search?text=@openfn&size=1" || failed=1 - probe jsdelivr \ - "${BASE_URL}/jsdelivr/npm/@openfn/language-common/package.json" || failed=1 - probe github \ - "${BASE_URL}/github/OpenFn/adaptors/main/packages/common/assets/square.png" \ - || failed=1 - - echo - if [[ "${failed}" -ne 0 ]]; then - echo "adaptor_cache: check FAILED" >&2 - exit 1 - fi - - echo "adaptor_cache: check passed — all three prefixes cache." -} +Mix.install([ + {:bandit, "~> 1.5"}, + {:plug, "~> 1.16"}, + {:req, "~> 0.5"} +]) -main() { - local command="${1:-help}" - [[ $# -gt 0 ]] && shift || true +script_path = Path.expand(__ENV__.file) +lib_dir = Path.join(Path.dirname(script_path), "../tooling/adaptor_cache/lib") - case "${command}" in - up) cmd_up "$@" ;; - down) cmd_down "$@" ;; - purge) cmd_purge "$@" ;; - status) cmd_status "$@" ;; - logs) cmd_logs "$@" ;; - check) cmd_check "$@" ;; - help | --help | -h) show_help ;; - *) - echo "adaptor_cache: unknown command '${command}'" >&2 - echo >&2 - show_help >&2 - exit 1 - ;; - esac -} +~w(cache.ex router.ex publish.ex scenario.ex cli.ex) +|> Enum.each(&Code.require_file(Path.join(lib_dir, &1))) -main "$@" +AdaptorCache.Cli.main(System.argv(), script: script_path) diff --git a/config/test.exs b/config/test.exs index 1057be2db8a..f3d1b8a07d8 100644 --- a/config/test.exs +++ b/config/test.exs @@ -52,15 +52,25 @@ config :lightning, Lightning.Vault, # We don't run a server during test. If one is required, # you can enable the server option below. +# +# The port is randomized (unless TEST_PORT is set) so `mix test` can run +# concurrently across multiple git worktrees on the same machine without +# port clashes. +test_port = + case System.get_env("TEST_PORT") do + nil -> Enum.random(4100..4800) + port -> String.to_integer(port) + end + config :lightning, LightningWeb.Endpoint, - http: [port: 4002], + http: [port: test_port], url: [scheme: "http"], secret_key_base: "/8zedVJLxvmGGFoRExE3e870g7CGZZQ1Vq11A5MbQGPKOpK57MahVsPW6Wkkv61n", server: true config :lightning, Lightning.Runtime.RuntimeManager, - ws_url: "ws://localhost:4002/worker" + ws_url: "ws://localhost:#{test_port}/worker" config :lightning, :workers, private_key: """ diff --git a/test/lightning_web/controllers/api/job_controller_test.exs b/test/lightning_web/controllers/api/job_controller_test.exs index e1655b01eb6..82de783537b 100644 --- a/test/lightning_web/controllers/api/job_controller_test.exs +++ b/test/lightning_web/controllers/api/job_controller_test.exs @@ -36,7 +36,7 @@ defmodule LightningWeb.API.JobControllerTest do "attributes" => %{"name" => "some name"}, "id" => job.id, "links" => %{ - "self" => "http://localhost:4002/api/jobs/#{job.id}" + "self" => "#{LightningWeb.Endpoint.url()}/api/jobs/#{job.id}" }, "relationships" => %{}, "type" => "jobs" @@ -63,7 +63,7 @@ defmodule LightningWeb.API.JobControllerTest do "attributes" => %{"name" => "some name"}, "id" => job.id, "links" => %{ - "self" => "http://localhost:4002/api/jobs/#{job.id}" + "self" => "#{LightningWeb.Endpoint.url()}/api/jobs/#{job.id}" }, "relationships" => %{}, "type" => "jobs" @@ -88,7 +88,7 @@ defmodule LightningWeb.API.JobControllerTest do "attributes" => %{"name" => "some name"}, "id" => job.id, "links" => %{ - "self" => "http://localhost:4002/api/jobs/#{job.id}" + "self" => "#{LightningWeb.Endpoint.url()}/api/jobs/#{job.id}" }, "relationships" => %{}, "type" => "jobs" diff --git a/test/lightning_web/controllers/api/project_controller_test.exs b/test/lightning_web/controllers/api/project_controller_test.exs index 04169ff4809..f5a9dd0a661 100644 --- a/test/lightning_web/controllers/api/project_controller_test.exs +++ b/test/lightning_web/controllers/api/project_controller_test.exs @@ -81,7 +81,8 @@ defmodule LightningWeb.API.ProjectControllerTest do }, "id" => project.id, "links" => %{ - "self" => "http://localhost:4002/api/projects/#{project.id}" + "self" => + "#{LightningWeb.Endpoint.url()}/api/projects/#{project.id}" }, "relationships" => %{}, "type" => "projects" @@ -111,7 +112,8 @@ defmodule LightningWeb.API.ProjectControllerTest do "attributes" => %{"name" => "a-test-project"}, "id" => project.id, "links" => %{ - "self" => "http://localhost:4002/api/projects/#{project.id}" + "self" => + "#{LightningWeb.Endpoint.url()}/api/projects/#{project.id}" }, "relationships" => %{}, "type" => "projects" @@ -145,7 +147,8 @@ defmodule LightningWeb.API.ProjectControllerTest do }, "id" => project.id, "links" => %{ - "self" => "http://localhost:4002/api/projects/#{project.id}" + "self" => + "#{LightningWeb.Endpoint.url()}/api/projects/#{project.id}" }, "relationships" => %{}, "type" => "projects" diff --git a/test/lightning_web/controllers/webhooks_controller_test.exs b/test/lightning_web/controllers/webhooks_controller_test.exs index a515436c279..762cb3e34a4 100644 --- a/test/lightning_web/controllers/webhooks_controller_test.exs +++ b/test/lightning_web/controllers/webhooks_controller_test.exs @@ -92,7 +92,7 @@ defmodule LightningWeb.WebhooksControllerTest do assert {:ok, %Tesla.Env{status: 413, body: "Request Entity Too Large"}} = [ - {Tesla.Middleware.BaseUrl, "http://localhost:4002"}, + {Tesla.Middleware.BaseUrl, LightningWeb.Endpoint.url()}, Tesla.Middleware.JSON ] |> Tesla.client() diff --git a/tooling/adaptor_cache/.gitignore b/tooling/adaptor_cache/.gitignore new file mode 100644 index 00000000000..cff063eab02 --- /dev/null +++ b/tooling/adaptor_cache/.gitignore @@ -0,0 +1 @@ +/scenarios/ diff --git a/tooling/adaptor_cache/README.md b/tooling/adaptor_cache/README.md index 8ff29f125fd..4cfe1043c48 100644 --- a/tooling/adaptor_cache/README.md +++ b/tooling/adaptor_cache/README.md @@ -1,34 +1,33 @@ # Adaptor Cache -> A local caching reverse proxy sitting in front of the three upstreams -> `Lightning.Adaptors.*` reads from. +> A local record-and-replay reverse proxy sitting in front of the three +> upstreams `Lightning.Adaptors.*` reads from. Every `Lightning.Adaptors.Scheduler` refresh tick makes one npm `/-/v1/search` call, then one packument call and one jsDelivr schema call per changed package, plus up to four `raw.githubusercontent.com` calls per package for icons (two shapes x the png-then-svg fallback). Iterating on the subsystem means running -that loop over and over against the real internet. This proxy caches all of it +that loop over and over against the real internet. This proxy records all of it to disk so the second and every later run is local, and the whole thing works offline (on a plane, on bad wifi, wherever). Driven by `bin/adaptor_cache` from the repo root; see that script's `--help` for -the full command list. +the full command list. It's a self-contained script (`Mix.install` pulls in +Bandit + Req the first time it runs). ## What's in this folder -| File | Purpose | -| -------------------- | -------------------------------------------------------------------------------------------- | -| `docker-compose.yml` | Runs the `nginx` container, bound to `127.0.0.1` only | -| `nginx.conf` | The three `/npm/`, `/jsdelivr/`, `/github/` proxy locations and the persistent on-disk cache | +| File/dir | Purpose | +| ------------ | ------------------------------------------------------------- | +| `lib/` | The proxy, `publish`, and `scenario` implementation | +| `scenarios/` | Saved `scenario save` snapshots (untracked, see `.gitignore`) | -## Prerequisites +## Recorded responses have no TTL -- Docker + Docker Compose -- **Bring the cache up at least once while online.** nginx resolves - `registry.npmjs.org`, `cdn.jsdelivr.net` and `raw.githubusercontent.com` at - container startup, not per-request. Starting it offline for the first time - fails with `host not found in upstream` — do the first `bin/adaptor_cache up` - with a real connection, after that it's fine offline. +A recorded response is authoritative until you `purge` it — there is no expiry. +That's deliberate: the recorded files double as hand-editable fixtures, so the +cache is both the everyday dev cache and the mechanism for driving the `publish` +scenarios below, by editing exactly the files it already wrote. ## Environment variables @@ -46,47 +45,52 @@ don't have to remember them. - `ADAPTOR_CACHE_PORT` — host port to bind (default: `4874`). Set it before any `bin/adaptor_cache` command if `4874` is taken, and update the three exports above to match. +- `ADAPTOR_CACHE_DIR` — where recorded responses live (default: + `/tmp/adaptor_cache`). Several distros age-clean `/tmp` (systemd-tmpfiles: 10 + days on Fedora/Arch) — if a fixture goes missing for no obvious reason, that's + likely it. Set this to somewhere outside `/tmp` if you want the cache to + survive indefinitely. ## Usage ```sh -bin/adaptor_cache up # start the proxy and print the export lines -bin/adaptor_cache down # stop the proxy, keeping the cache on disk -bin/adaptor_cache status # show container state and reachability -bin/adaptor_cache purge # stop the proxy AND drop the cache volume -bin/adaptor_cache logs # tail the access log (cache=HIT / cache=MISS) -bin/adaptor_cache check # probe all three prefixes, prove MISS then HIT -bin/adaptor_cache --help # full usage +bin/adaptor_cache up # start the proxy and print the export lines +bin/adaptor_cache down # stop the proxy, keeping the cache on disk +bin/adaptor_cache status # show whether it's running and reachable +bin/adaptor_cache purge # clear all recorded responses +bin/adaptor_cache logs # tail the access log (cache=HIT / cache=MISS) +bin/adaptor_cache check # probe all three prefixes, prove MISS then HIT +bin/adaptor_cache publish # record a synthetic adaptor/version +bin/adaptor_cache scenario save # snapshot the live cache under that name +bin/adaptor_cache scenario restore # replace the live cache with that snapshot +bin/adaptor_cache --help # full usage ``` With the cache up and the three vars exported, run -`mix lightning.refresh_adaptors` as usual. The first run populates the cache; -every run after that should be fast and work with no network at all. +`mix lightning.refresh_adaptors` as usual. The first run records the cache; +every run after that is local, with no network needed at all. ### Reading `bin/adaptor_cache logs` Each line is one proxied request: ``` -2026-08-25T10:00:00+00:00 status=200 cache=HIT GET /npm/-/v1/search?text=@openfn&size=250 +2026-08-31T10:00:00Z status=200 cache=HIT GET /npm/-/v1/search?text=%40openfn&size=250 ``` - `cache=HIT` — served entirely from disk, no upstream request made. -- `cache=MISS` — not in the cache (or expired), fetched from the real upstream - and stored. -- `cache=EXPIRED` / `cache=REVALIDATED` — the entry had aged out, so nginx did - go upstream. REVALIDATED means it sent a conditional request, got a 304, and - reused what was on disk. -- `cache=STALE` / `cache=UPDATING` — nginx served what it had on disk because - the upstream was unreachable, or because another request was already - refetching. This is what you see offline. +- `cache=MISS` — not recorded yet, fetched from the real upstream and saved. +- `cache=ERROR` — the live fetch itself failed (offline, upstream down); nothing + is recorded, so the next attempt tries live again. On a warm cache, MISS should only appear for packages the cache has never seen. ## How the URL mapping works -The proxy has one location block per upstream, and Lightning's `NPM` strategy -already builds full paths under each — the proxy just rewrites the host: +Lightning's `NPM` strategy already builds full paths under each of the three +base URLs — the proxy fetches the same path from the real upstream and caches it +under the _original_ request path, following any redirect itself first, so the +cached entry reflects the final resolved resource, not an intermediate redirect: | Lightning request | Through the proxy | Prefix | Real upstream | | -------------------------------------- | ------------------------------------------------------------------------------------------ | ------------ | --------------------------------------- | @@ -94,10 +98,40 @@ already builds full paths under each — the proxy just rewrites the host: | jsDelivr schema fetch (`schema.ex`) | `http://localhost:4874/jsdelivr/npm/@openfn/language-http@2.1.0/configuration-schema.json` | `/jsdelivr/` | `https://cdn.jsdelivr.net/...` | | GitHub icon fetch (`github.ex`) | `http://localhost:4874/github/OpenFn/adaptors/main/packages/http/assets/square.png` | `/github/` | `https://raw.githubusercontent.com/...` | -Setting `ADAPTORS_NPM_REGISTRY_URL`, `ADAPTORS_NPM_JSDELIVR_URL` and -`ADAPTORS_NPM_GITHUB_URL` to the proxy's `/npm`, `/jsdelivr` and `/github` base -URLs is all that's needed — the strategy code appends the same paths it always -did, just against a different host. +## Recorded files as fixtures + +A recorded response is two files: the raw body, plus a `.meta` sidecar with its +status and content type. The path mirrors the request, so — for example — the +`@openfn/language-http` packument lands at +`/tmp/adaptor_cache/npm/@openfn/language-http`, and the search response (the one +query Lightning ever sends) lands at +`/tmp/adaptor_cache/npm/-/v1/search?text=%40openfn&size=250`. Both are plain +JSON — open and edit them directly to hand-craft a scenario. + +## Driving both `publish` scenarios + +```sh +bin/adaptor_cache publish @openfn/language-brand-new 1.0.0 # new adaptor appears +bin/adaptor_cache publish @openfn/language-http 9.9.9 # new version of an existing adaptor +``` + +Either form updates the packument _and_ the search response's `latest_version` +together in one call — `scheduler.ex`'s change-detection compares the search +response against the DB to decide whether to bother fetching the packument at +all, so updating only one is a silent no-op. Run +`mix lightning.refresh_adaptors` (or reopen the picker) afterwards to see it +take effect. + +## Scenarios + +```sh +bin/adaptor_cache scenario save drill-1 # snapshot the live cache +bin/adaptor_cache purge # ...break/change things... +bin/adaptor_cache scenario restore drill-1 # back to exactly that state +``` + +Scenarios live under `tooling/adaptor_cache/scenarios//` and stay +untracked (not checked into git) for now. ## Caveats @@ -105,23 +139,15 @@ did, just against a different host. bypass this entirely.** Both have hardcoded upstream URLs and don't read the `ADAPTORS_NPM_*` env vars, so they'll always hit the real internet regardless of whether the cache is up. -- **Redirects bypass the cache.** Tesla's `FollowRedirects` middleware requests - the absolute `Location` URL, which points at the real upstream even when the - redirect is same-host, so anything that 30x's is fetched live. - **Never set this as your global npm registry in `~/.npmrc`.** The `/npm/` prefix is a transparent GET proxy of registry.npmjs.org, so npm would mostly - work, badly: this cache ignores Cache-Control and holds 200s for seven days, - so `npm install` resolves against a week-stale packument, and npm records the - registry it fetched from in `package-lock.json`'s `resolved` URLs, giving you - a lockfile that only installs on a machine running this container. + work, badly: this cache never expires what it records, so `npm install` could + resolve against an arbitrarily stale packument, and npm records the registry + it fetched from in `package-lock.json`'s `resolved` URLs, giving you a + lockfile that only installs on a machine running this proxy. ## Troubleshooting -**`host not found in upstream` on startup.** You brought the container up -offline for the first time. Get online and run `bin/adaptor_cache up` once so -nginx can resolve the three upstream hostnames, then it's fine offline after -that. - **`bin/adaptor_cache check` fails on one prefix.** Run `bin/adaptor_cache logs` and look for the failing request — a `cache=MISS` on the _second_ identical request usually means the upstream is refusing the request outright (check @@ -131,6 +157,5 @@ status code) rather than a caching problem. `bin/adaptor_cache up`, and update the three `ADAPTORS_NPM_*_URL` exports to match the new port. -**Stale or wrong data cached.** `bin/adaptor_cache purge` drops the on-disk -cache volume entirely (unlike `down`, which keeps it); `up` again to rebuild -from empty. +**Stale or wrong data cached.** `bin/adaptor_cache purge` drops every recorded +response; `up` keeps running and the next request re-fetches live. diff --git a/tooling/adaptor_cache/docker-compose.yml b/tooling/adaptor_cache/docker-compose.yml deleted file mode 100644 index c5ea519c754..00000000000 --- a/tooling/adaptor_cache/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Caching reverse proxy for the Lightning.Adaptors.* upstreams. -# -# Driven by bin/adaptor_cache; see README.md in this directory for the why and -# the env vars to export. -services: - nginx: - image: nginx:1.27.4-alpine - container_name: adaptor-cache - ports: - - '127.0.0.1:${ADAPTOR_CACHE_PORT:-4874}:80' - volumes: - - ./nginx.conf:/etc/nginx/nginx.conf:ro - # Named volume, so `docker compose down` keeps the cache; only - # `bin/adaptor_cache purge` drops it. - - adaptor-cache-data:/var/cache/nginx/adaptors - restart: unless-stopped - healthcheck: - test: ['CMD', 'wget', '-q', '-O', '-', 'http://127.0.0.1/_healthz'] - interval: 5s - timeout: 3s - retries: 12 - start_period: 2s - -volumes: - adaptor-cache-data: {} diff --git a/tooling/adaptor_cache/lib/cache.ex b/tooling/adaptor_cache/lib/cache.ex new file mode 100644 index 00000000000..e6c9e573ed3 --- /dev/null +++ b/tooling/adaptor_cache/lib/cache.ex @@ -0,0 +1,89 @@ +defmodule AdaptorCache.Cache do + @moduledoc """ + On-disk record-and-replay store. + + A response lives as two sibling files: the raw body, and a `.meta` JSON + sidecar holding the status, content type and etag. No TTL — once written, a + file is authoritative until `purge/0` removes it. + """ + + @data_dirs ~w(npm jsdelivr github) + + def root, do: System.get_env("ADAPTOR_CACHE_DIR", "/tmp/adaptor_cache") + + def data_dirs, do: @data_dirs + + def run_dir, do: Path.join(root(), ".run") + + @doc """ + Builds the on-disk path for a request, mirroring its path segments as + directories so recorded files stay human-navigable and hand-editable. A + non-empty query string is appended to the leaf filename, with any `/` + replaced first — unlike a URL path, a query string may legally contain + unescaped `/` and `..`, so it can't be trusted to stay inside the leaf + segment. + + Final defense: `Path.expand` the result and verify it's still under + `root()`. This is what actually blocks traversal (from either the path or + the query) — the `.`/`..` segment check above is just a fast, readable + rejection for the common case. + """ + def key_path(prefix, path, query) when prefix in @data_dirs do + segments = path |> String.trim_leading("/") |> String.split("/") + + if Enum.any?(segments, &(&1 in [".", ".."])) or + Enum.any?(segments, &(&1 == "")) do + {:error, :invalid_path} + else + safe_query = String.replace(query || "", "/", "%2F") + + leaf = + if safe_query == "", + do: List.last(segments), + else: List.last(segments) <> "?" <> safe_query + + dirs = Enum.slice(segments, 0..-2//1) + root = Path.expand(root()) + file = Path.expand(Path.join([root, prefix] ++ dirs ++ [leaf])) + + if String.starts_with?(file, root <> "/"), + do: {:ok, file}, + else: {:error, :invalid_path} + end + end + + def read(file) do + with true <- File.regular?(file), + {:ok, meta_json} <- File.read(file <> ".meta"), + {:ok, meta} <- JSON.decode(meta_json), + {:ok, body} <- File.read(file) do + {:ok, + %{ + status: meta["status"], + content_type: meta["content_type"], + etag: meta["etag"], + body: body + }} + else + _ -> :miss + end + end + + def write(file, status, content_type, etag, body) do + File.mkdir_p!(Path.dirname(file)) + File.write!(file, body) + + File.write!( + file <> ".meta", + JSON.encode!(%{status: status, content_type: content_type, etag: etag}) + ) + + :ok + end + + @doc "Wipes recorded responses, leaving the daemon's pidfile/log (under `.run/`) intact." + def purge do + Enum.each(@data_dirs, &File.rm_rf!(Path.join(root(), &1))) + :ok + end +end diff --git a/tooling/adaptor_cache/lib/cli.ex b/tooling/adaptor_cache/lib/cli.ex new file mode 100644 index 00000000000..e6ec3f847ea --- /dev/null +++ b/tooling/adaptor_cache/lib/cli.ex @@ -0,0 +1,393 @@ +defmodule AdaptorCache.Cli do + @moduledoc "Command dispatch for `bin/adaptor_cache`." + + alias AdaptorCache.Cache + alias AdaptorCache.Publish + alias AdaptorCache.Scenario + + def main(argv, script: script) do + ctx = %{ + script: script, + port: System.get_env("ADAPTOR_CACHE_PORT", "4874"), + pidfile: Path.join(Cache.run_dir(), "adaptor_cache.pid"), + logfile: Path.join(Cache.run_dir(), "adaptor_cache.log") + } + + dispatch(argv, ctx) + end + + defp dispatch(["up"], ctx), + do: up(ctx.script, ctx.port, ctx.pidfile, ctx.logfile) + + defp dispatch(["down"], ctx), do: down(ctx.pidfile) + defp dispatch(["status"], ctx), do: status(ctx.pidfile, ctx.port) + defp dispatch(["logs"], ctx), do: logs(ctx.logfile) + defp dispatch(["purge"], _ctx), do: purge() + defp dispatch(["check"], ctx), do: check(ctx.port) + defp dispatch(["publish", name, version], _ctx), do: publish(name, version) + defp dispatch(["scenario", "save", name], _ctx), do: scenario(:save, name) + + defp dispatch(["scenario", "restore", name], _ctx), + do: scenario(:restore, name) + + defp dispatch(["_serve", port], _ctx), do: serve(String.to_integer(port)) + defp dispatch([], _ctx), do: help() + defp dispatch(["help"], _ctx), do: help() + defp dispatch(["--help"], _ctx), do: help() + defp dispatch(argv, _ctx), do: unknown(argv) + + # --- lifecycle ------------------------------------------------------- + + defp up(script, port, pidfile, logfile) do + if alive?(pidfile) do + IO.puts("adaptor_cache: already up on #{base_url(port)}") + else + File.mkdir_p!(Cache.run_dir()) + + elixir_bin = + System.find_executable("elixir") || raise "elixir not found on PATH" + + # setsid isn't part of the base install on macOS (it ships with + # util-linux, Linux-only) — fall back to plain nohup there. Detaching + # stdin/stdout/stderr already covers surviving a closed terminal; + # setsid additionally detaches the process group, which only matters + # for a parent shell that sends SIGHUP to its whole group on exit. + detach = + if System.find_executable("setsid"), do: "setsid nohup", else: "nohup" + + cmd = """ + #{detach} "#{elixir_bin}" "#{script}" _serve #{port} > "#{logfile}" 2>&1 < /dev/null & + echo $! > "#{pidfile}" + """ + + case System.shell(cmd) do + {_, 0} -> + :ok + + {output, status} -> + IO.puts( + :stderr, + "adaptor_cache: failed to launch (exit #{status}): #{output}" + ) + + System.halt(1) + end + + cond do + not wait_for_healthz(port) -> + IO.puts( + :stderr, + "adaptor_cache: did not come up on #{base_url(port)} after 30s." + ) + + IO.puts( + :stderr, + "adaptor_cache: run 'bin/adaptor_cache logs' to see why." + ) + + System.halt(1) + + not alive?(pidfile) -> + # healthz answered, but not from the process we just spawned — + # something else (a stray daemon from an earlier crash, or an + # unrelated process) already holds the port. + IO.puts( + :stderr, + "adaptor_cache: #{base_url(port)} is reachable, but not from a process this command started." + ) + + IO.puts( + :stderr, + "adaptor_cache: something else is bound to port #{port} — set ADAPTOR_CACHE_PORT to another value, or stop it." + ) + + System.halt(1) + + true -> + IO.puts("adaptor_cache: up on #{base_url(port)}") + end + end + + print_exports(port) + end + + defp down(pidfile) do + case read_pid(pidfile) do + nil -> + IO.puts("adaptor_cache: not running.") + + pid -> + System.cmd("kill", [pid], stderr_to_stdout: true) + wait_for_exit(pid) + File.rm(pidfile) + + IO.puts( + "adaptor_cache: down. The cache on disk is intact; 'purge' clears it." + ) + end + end + + # Gives the OS a moment to actually release the port before we return, so + # a following `up` doesn't race the still-shutting-down old process. + defp wait_for_exit(pid) do + Enum.reduce_while(1..20, :ok, fn _, _ -> + if match?({_, 0}, System.cmd("kill", ["-0", pid], stderr_to_stdout: true)) do + Process.sleep(250) + {:cont, :ok} + else + {:halt, :ok} + end + end) + end + + defp status(pidfile, port) do + if alive?(pidfile) do + IO.puts("adaptor_cache: running (pid #{read_pid(pidfile)})") + else + IO.puts("adaptor_cache: not running") + end + + if healthz?(port) do + IO.puts("adaptor_cache: reachable at #{base_url(port)}") + else + IO.puts("adaptor_cache: NOT reachable at #{base_url(port)}") + end + end + + defp logs(logfile) do + if File.exists?(logfile) do + System.cmd("tail", ["-f", "-n", "100", logfile], + into: IO.stream(:stdio, :line) + ) + else + IO.puts( + "adaptor_cache: no log file yet at #{logfile} — has it been started?" + ) + end + end + + defp purge do + Cache.purge() + IO.puts("adaptor_cache: cache cleared. Next request re-fetches live.") + end + + defp serve(port) do + {:ok, _} = + Bandit.start_link( + plug: AdaptorCache.Router, + port: port, + ip: {127, 0, 0, 1} + ) + + Process.sleep(:infinity) + end + + # --- publish / scenario ----------------------------------------------- + + defp publish(name, version) do + case Publish.run(name, version) do + :ok -> + IO.puts( + "adaptor_cache: published #{name}@#{version} (packument + search response updated)." + ) + + {:error, reason} -> + IO.puts(:stderr, "adaptor_cache: #{reason}") + System.halt(1) + end + end + + defp scenario(:save, name) do + case Scenario.save(name) do + :ok -> + IO.puts("adaptor_cache: saved scenario #{inspect(name)}.") + + {:error, reason} -> + IO.puts(:stderr, "adaptor_cache: #{reason}") + System.halt(1) + end + end + + defp scenario(:restore, name) do + case Scenario.restore(name) do + :ok -> + IO.puts("adaptor_cache: restored scenario #{inspect(name)}.") + + {:error, reason} -> + IO.puts(:stderr, "adaptor_cache: #{reason}") + System.halt(1) + end + end + + # --- check ------------------------------------------------------------- + + defp check(port) do + base = base_url(port) + + unless healthz?(port) do + IO.puts( + :stderr, + "adaptor_cache: not running. Start it with 'bin/adaptor_cache up'." + ) + + System.halt(1) + end + + IO.puts("adaptor_cache: probing all three prefixes twice each\n") + + results = [ + # %40, not a literal @: matches the wire query Tesla's default query + # encoding actually sends (see AdaptorCache.Publish), so this probe + # exercises the same cache key `publish` and real traffic use. + probe("npm", base <> "/npm/-/v1/search?text=%40openfn&size=250"), + probe( + "jsdelivr", + base <> "/jsdelivr/npm/@openfn/language-common/package.json" + ), + probe( + "github", + base <> "/github/OpenFn/adaptors/main/packages/common/assets/square.png" + ) + ] + + IO.puts("") + + if Enum.all?(results, & &1) do + IO.puts("adaptor_cache: check passed — all three prefixes cache.") + else + IO.puts(:stderr, "adaptor_cache: check FAILED") + System.halt(1) + end + end + + defp probe(label, url) do + with {:ok, first} <- Req.get(url, decode_body: false), + {:ok, second} <- Req.get(url, decode_body: false) do + cache_2 = + second.headers |> Map.get("x-cache-status", ["-"]) |> List.first() + + cache_1 = first.headers |> Map.get("x-cache-status", ["-"]) |> List.first() + + IO.puts( + " #{String.pad_trailing(label, 9)} #{first.status} first=#{cache_1} second=#{cache_2} #{url}" + ) + + if cache_2 != "HIT" do + IO.puts(" ^ expected the second request to be a HIT") + false + else + true + end + else + {:error, reason} -> + IO.puts(" #{String.pad_trailing(label, 9)} FAIL #{inspect(reason)}") + false + end + end + + # --- helpers ------------------------------------------------------------- + + defp base_url(port), do: "http://localhost:#{port}" + + defp print_exports(port) do + base = base_url(port) + + IO.puts(""" + + Point Lightning at the cache by exporting these: + + export ADAPTORS_NPM_REGISTRY_URL=#{base}/npm + export ADAPTORS_NPM_JSDELIVR_URL=#{base}/jsdelivr + export ADAPTORS_NPM_GITHUB_URL=#{base}/github + + Then: mix lightning.refresh_adaptors + Watch it work: bin/adaptor_cache logs + Prove it works: bin/adaptor_cache check + """) + end + + defp wait_for_healthz(port) do + Enum.reduce_while(1..30, false, fn _, _ -> + if healthz?(port) do + {:halt, true} + else + Process.sleep(1000) + {:cont, false} + end + end) + end + + defp healthz?(port) do + case Req.get(base_url(port) <> "/_healthz", + receive_timeout: 2_000, + retry: false + ) do + {:ok, %{status: 200}} -> true + _ -> false + end + rescue + _ -> false + end + + defp alive?(pidfile) do + case read_pid(pidfile) do + nil -> false + pid -> running_and_ours?(pid) + end + end + + # kill -0 alone only proves the PID is alive, not that it's still our + # daemon — after a crash the PID can be reused by an unrelated process. + # `ps -o command=` (not /proc, which doesn't exist on macOS) catches that. + defp running_and_ours?(pid) do + case System.cmd("ps", ["-p", pid, "-o", "command="], stderr_to_stdout: true) do + {output, 0} -> String.contains?(output, "adaptor_cache") + _ -> false + end + end + + defp read_pid(pidfile) do + case File.read(pidfile) do + {:ok, pid} -> String.trim(pid) + _ -> nil + end + end + + defp unknown(argv) do + IO.puts( + :stderr, + "adaptor_cache: unknown command #{inspect(Enum.join(argv, " "))}\n" + ) + + help() + System.halt(1) + end + + defp help do + IO.puts(""" + Lightning Adaptor Cache + + A local record-and-replay reverse proxy in front of the three upstreams + the Lightning.Adaptors.* subsystem reads from: registry.npmjs.org, + cdn.jsdelivr.net and raw.githubusercontent.com. See + tooling/adaptor_cache/README.md. + + Usage: bin/adaptor_cache + + up Start the proxy and print the export lines + down Stop the proxy, keeping the cache on disk + status Show whether it's running and reachable + purge Clear all recorded responses + logs Tail the access log (cache=HIT / cache=MISS) + check Probe all three prefixes, prove MISS then HIT + publish Record a synthetic adaptor/version + scenario save Snapshot the live cache under that name + scenario restore Replace the live cache with that snapshot + --help This message + + Environment variables: + + ADAPTOR_CACHE_PORT Host port to bind (default: 4874) + """) + end +end diff --git a/tooling/adaptor_cache/lib/publish.ex b/tooling/adaptor_cache/lib/publish.ex new file mode 100644 index 00000000000..d7a25fb62a0 --- /dev/null +++ b/tooling/adaptor_cache/lib/publish.ex @@ -0,0 +1,162 @@ +defmodule AdaptorCache.Publish do + @moduledoc """ + `publish ` — makes both scenarios (new adaptor, new + version) reproducible in one step by always updating the recorded npm + search response and packument together. + + `scheduler.ex`'s `fetch_if_changed/4` only fetches a packument when the + search response's `latest_version` for that package differs from the DB — + so a packument-only bump is a silent no-op. This module never lets the two + drift apart: both updates are computed before either is written, so a + malformed hand-edited fixture fails before touching disk instead of + leaving one file updated and the other stale. + """ + + alias AdaptorCache.Cache + + # registry.ex builds this with `query: [text: "@openfn", size: 250]`, and + # Tesla's default www-form query encoding percent-encodes `@` — the wire + # query is "text=%40openfn&size=250", not the literal text. Not templated + # per package, so it's still the one deterministic key for "the" recorded + # search response. + @search_query "text=%40openfn&size=250" + + def run(name, version) do + packument_file = key_path!("npm", "/" <> name, "") + search_file = key_path!("npm", "/-/v1/search", @search_query) + + with {:ok, packument} <- update_packument(packument_file, name, version), + {:ok, search} <- update_search(search_file, name, version) do + Cache.write( + packument_file, + 200, + "application/json", + nil, + JSON.encode!(packument) + ) + + Cache.write( + search_file, + 200, + "application/json", + nil, + JSON.encode!(search) + ) + + :ok + end + end + + defp key_path!(prefix, path, query) do + {:ok, file} = Cache.key_path(prefix, path, query) + file + end + + defp update_packument(file, name, version) do + now = DateTime.utc_now() |> DateTime.to_iso8601() + + version_entry = %{ + "dist" => %{ + "tarball" => + "https://registry.npmjs.org/#{name}/-/#{basename(name)}-#{version}.tgz", + "integrity" => "sha512-adaptorcache", + "unpackedSize" => 0 + }, + "dependencies" => %{}, + "peerDependencies" => %{} + } + + with {:ok, existing} <- decode_cached(file) do + packument = + case existing do + :miss -> + %{ + "name" => name, + "dist-tags" => %{"latest" => version}, + "versions" => %{version => version_entry}, + "time" => %{version => now} + } + + packument -> + # Map.update/4 only falls back to the default when the key is + # *absent* — a hand-edited "versions": null still passes nil to + # the updater, so each field guards with `|| %{}` the same way. + packument + |> Map.put( + "dist-tags", + Map.put(packument["dist-tags"] || %{}, "latest", version) + ) + |> Map.put( + "versions", + Map.put(packument["versions"] || %{}, version, version_entry) + ) + |> Map.put("time", Map.put(packument["time"] || %{}, version, now)) + end + + {:ok, packument} + end + end + + defp update_search(file, name, version) do + minimal_entry = %{"package" => %{"name" => name, "version" => version}} + + with {:ok, existing} <- decode_cached(file) do + search = + case existing do + :miss -> + %{ + "objects" => [minimal_entry], + "total" => 1, + "time" => DateTime.utc_now() |> DateTime.to_iso8601() + } + + search -> + objects = search["objects"] || [] + + objects = + if Enum.any?(objects, &(&1["package"]["name"] == name)) do + # Only bump the version field, so a real recorded entry's + # other fields (maintainers, license, downloads, ...) + # survive intact. + Enum.map(objects, fn + %{"package" => %{"name" => ^name}} = object -> + put_in(object, ["package", "version"], version) + + other -> + other + end) + else + objects ++ [minimal_entry] + end + + search + |> Map.put("objects", objects) + |> Map.put("total", length(objects)) + end + + {:ok, search} + end + end + + # README.md explicitly invites hand-editing these files, so malformed JSON + # is expected input, not an exceptional one — fail with a message naming + # the file, not a bare JSON.DecodeError stacktrace. + defp decode_cached(file) do + case Cache.read(file) do + {:ok, %{body: body}} -> + case JSON.decode(body) do + {:ok, decoded} -> + {:ok, decoded} + + {:error, reason} -> + {:error, + "#{file} is not valid JSON (#{inspect(reason)}) — fix it or purge"} + end + + :miss -> + {:ok, :miss} + end + end + + defp basename(name), do: name |> String.split("/") |> List.last() +end diff --git a/tooling/adaptor_cache/lib/router.ex b/tooling/adaptor_cache/lib/router.ex new file mode 100644 index 00000000000..87c0eb10195 --- /dev/null +++ b/tooling/adaptor_cache/lib/router.ex @@ -0,0 +1,126 @@ +defmodule AdaptorCache.Router do + @moduledoc """ + Record-and-replay reverse proxy for the three upstreams + `Lightning.Adaptors.NPM` reads from. A hit is served straight off disk; a + miss is fetched live (following redirects itself, so the *final* resource + is what gets cached under the *originally requested* key), recorded, then + served. + """ + use Plug.Router + + alias AdaptorCache.Cache + + plug :match + plug :dispatch + + @upstreams %{ + "npm" => "https://registry.npmjs.org", + "jsdelivr" => "https://cdn.jsdelivr.net", + "github" => "https://raw.githubusercontent.com" + } + + get "/_healthz" do + send_resp(conn, 200, "ok\n") + end + + get("/npm/*rest", do: proxy(conn, "npm", rest)) + get("/jsdelivr/*rest", do: proxy(conn, "jsdelivr", rest)) + get("/github/*rest", do: proxy(conn, "github", rest)) + + match _ do + send_resp(conn, 404, "adaptor_cache: use /npm/, /jsdelivr/ or /github/\n") + end + + defp proxy(conn, prefix, rest_segments) do + path = "/" <> Enum.join(rest_segments, "/") + query = conn.query_string + + case Cache.key_path(prefix, path, query) do + {:error, :invalid_path} -> + send_resp(conn, 400, "adaptor_cache: invalid path\n") + + {:ok, file} -> + case Cache.read(file) do + {:ok, resp} -> + respond(conn, resp, "HIT") + + :miss -> + fetch_and_record(conn, file, prefix, path, query) + end + end + end + + defp fetch_and_record(conn, file, prefix, path, query) do + url = + @upstreams[prefix] <> path <> if(query == "", do: "", else: "?" <> query) + + case Req.get(url, + redirect: true, + decode_body: false, + receive_timeout: 30_000 + ) do + {:ok, %Req.Response{status: status, body: body, headers: headers}} -> + content_type = + header(headers, "content-type") || "application/octet-stream" + + etag = header(headers, "etag") + + # Only 2xx and 404 are content worth freezing forever. Everything + # else (5xx, and npm's 429 rate-limit) is transient upstream + # trouble — recording it would mean a rate-limit sticks until + # someone runs `purge`. 404 is kept deliberately: github.ex's + # png-then-svg fallback and the "adaptor disappeared" scenario both + # rely on it being cacheable. + if status in 200..299 or status == 404, + do: Cache.write(file, status, content_type, etag, body) + + respond( + conn, + %{status: status, content_type: content_type, etag: etag, body: body}, + "MISS" + ) + + {:error, reason} -> + log(conn, 502, "ERROR") + + send_resp( + conn, + 502, + "adaptor_cache: upstream error: #{inspect(reason)}\n" + ) + end + end + + defp respond(conn, resp, cache_status) do + log(conn, resp.status, cache_status) + + conn + # put_resp_content_type/2 always appends "; charset=utf-8", which + # duplicates one the upstream already sent — set the header directly to + # replay it byte-for-byte instead. + |> put_resp_header("content-type", resp.content_type) + |> put_resp_header("x-cache-status", cache_status) + |> maybe_put_etag(resp.etag) + |> send_resp(resp.status, resp.body) + end + + defp maybe_put_etag(conn, nil), do: conn + defp maybe_put_etag(conn, etag), do: put_resp_header(conn, "etag", etag) + + defp header(headers, name), do: headers |> Map.get(name, []) |> List.first() + + defp log(conn, status, cache_status) do + uri = + if conn.query_string == "", + do: conn.request_path, + else: conn.request_path <> "?" <> conn.query_string + + # Strip control chars (a crafted request path could otherwise inject + # ANSI escapes into whatever terminal is tailing this with `logs`). + uri = String.replace(uri, ~r/[\x00-\x1f\x7f]/, "") + + IO.puts( + "#{DateTime.utc_now() |> DateTime.to_iso8601()} status=#{status} cache=#{cache_status} GET #{uri}" + ) + end +end diff --git a/tooling/adaptor_cache/lib/scenario.ex b/tooling/adaptor_cache/lib/scenario.ex new file mode 100644 index 00000000000..06b23a02b56 --- /dev/null +++ b/tooling/adaptor_cache/lib/scenario.ex @@ -0,0 +1,57 @@ +defmodule AdaptorCache.Scenario do + @moduledoc """ + `scenario save`/`restore` — named, repeatable cache states, checked out of + git (see `.gitignore` in this directory). + """ + + alias AdaptorCache.Cache + + def scenarios_root, do: Path.expand("../scenarios", __DIR__) + + def save(name) do + with {:ok, dir} <- scenario_dir(name) do + File.rm_rf!(dir) + File.mkdir_p!(dir) + + Enum.each(Cache.data_dirs(), fn data_dir -> + src = Path.join(Cache.root(), data_dir) + if File.dir?(src), do: File.cp_r!(src, Path.join(dir, data_dir)) + end) + + :ok + end + end + + def restore(name) do + with {:ok, dir} <- scenario_dir(name), + {:ok, dir} <- require_dir(dir, name) do + Cache.purge() + + Enum.each(Cache.data_dirs(), fn data_dir -> + src = Path.join(dir, data_dir) + if File.dir?(src), do: File.cp_r!(src, Path.join(Cache.root(), data_dir)) + end) + + :ok + end + end + + defp require_dir(dir, name) do + if File.dir?(dir), + do: {:ok, dir}, + else: + {:error, "no scenario named #{inspect(name)} under #{scenarios_root()}"} + end + + # A scenario name becomes a raw directory segment — contain it under + # scenarios_root() the same way Cache.key_path contains a request path, + # so `scenario save ../../..` can't walk out of this directory. + defp scenario_dir(name) do + root = scenarios_root() + dir = Path.expand(Path.join(root, name)) + + if String.starts_with?(dir, root <> "/"), + do: {:ok, dir}, + else: {:error, "invalid scenario name #{inspect(name)}"} + end +end diff --git a/tooling/adaptor_cache/nginx.conf b/tooling/adaptor_cache/nginx.conf deleted file mode 100644 index c71b4614851..00000000000 --- a/tooling/adaptor_cache/nginx.conf +++ /dev/null @@ -1,151 +0,0 @@ -# Caching reverse proxy for the three Lightning.Adaptors.* upstreams. -# -# Point Lightning at it with: -# export ADAPTORS_NPM_REGISTRY_URL=http://localhost:4874/npm -# export ADAPTORS_NPM_JSDELIVR_URL=http://localhost:4874/jsdelivr -# export ADAPTORS_NPM_GITHUB_URL=http://localhost:4874/github -# -# Workers run as root deliberately. The cache lives on a Docker named volume -# mounted at /var/cache/nginx/adaptors, which Docker creates root-owned; the -# image's default `nginx` worker user could not write into it and every request -# would log a permission error and bypass the cache. This is a local dev tool -# bound to 127.0.0.1, so root workers are an acceptable trade for not needing -# an entrypoint chown. -user root; -worker_processes 1; -error_log /dev/stderr warn; -pid /var/run/nginx.pid; - -events { - worker_connections 256; -} - -http { - default_type application/octet-stream; - - # $upstream_cache_status says whether a request was answered from disk (HIT) - # or went out to the upstream (MISS, EXPIRED, REVALIDATED). - # `bin/adaptor_cache check` parses it. - log_format cache '$time_iso8601 status=$status cache=$upstream_cache_status ' - '$request_method $request_uri'; - access_log /dev/stdout cache; - - sendfile on; - tcp_nopush on; - keepalive_timeout 65; - - # Generous on purpose: nothing should be evicted mid-session, and the whole - # @openfn/language-* corpus is a few hundred MB at worst. - proxy_cache_path /var/cache/nginx/adaptors - levels=1:2 - keys_zone=adaptors:64m - max_size=20g - inactive=30d - use_temp_path=off; - - proxy_http_version 1.1; - - # All three upstreams are SNI-dependent CDNs. Without proxy_ssl_server_name - # the TLS handshake goes out with no server name and the upstream serves a - # default cert or refuses outright. proxy_ssl_name defaults to $proxy_host, - # which is the right value here. - proxy_ssl_server_name on; - - proxy_cache adaptors; - - # Cache aggressively regardless of upstream freshness headers. npm sends - # short max-age values and Set-Cookie; jsDelivr varies on Accept-Encoding. - # This is a dev cache, so serving a stale body is fine. - proxy_ignore_headers Cache-Control Expires Set-Cookie Vary; - - proxy_cache_valid 200 301 302 7d; - # 404s are frequent and expected: NPM.GitHub tries .png before - # .svg, so every SVG-only adaptor produces a 404 on the png attempt. - # Cache them so the fan-out is cheap, but briefly, so a newly-added icon - # shows up the same day. - proxy_cache_valid 404 10m; - proxy_cache_valid any 1m; - - # One request per cache key reaches the upstream; the rest wait on it. - # Without this, NPM.GitHub's icon fan-out opens as many upstream connections - # as its max concurrency allows on a cold cache. - proxy_cache_lock on; - proxy_cache_lock_timeout 30s; - - # Offline: serve whatever is on disk when the upstream cannot be reached. - proxy_cache_use_stale error timeout invalid_header updating - http_429 http_500 http_502 http_503 http_504; - proxy_cache_background_update on; - proxy_cache_revalidate on; - - # `always` matters for the 404s: add_header's default status list covers 2xx - # and 3xx but not 404, and cached 404s are routine here because of the - # png-then-svg icon fallback. - add_header X-Cache-Status $upstream_cache_status always; - add_header X-Adaptor-Cache-Upstream $proxy_host always; - - proxy_connect_timeout 10s; - proxy_send_timeout 30s; - proxy_read_timeout 30s; - - server { - listen 80; - server_name _; - - location = /_healthz { - access_log off; - return 200 "ok\n"; - } - - # nginx does not merge proxy_set_header across configuration levels: the - # moment a location declares one, every proxy_set_header from an enclosing - # level is dropped. So each location below repeats the same three headers - # rather than inheriting them, and Accept-Encoding must not be hoisted. - # - # Accept-Encoding is pinned to identity because Lightning's Tesla clients - # have no decompress middleware and NPM.Schema takes sha256 over the raw - # response bytes. If a gzip-capable client (curl --compressed) populated - # the cache, replaying that entry to Lightning would give it the wrong - # bytes and the wrong schema_sha256. Identity keeps every cached body - # byte-identical for every client. - - # GET /npm/-/v1/search?text=@openfn&size=250 - # GET /npm/@openfn/language-http - location /npm/ { - proxy_set_header Host registry.npmjs.org; - proxy_set_header Connection close; - proxy_set_header Accept-Encoding ""; - # A literal hostname (not a variable) so nginx resolves it once at - # startup from the container's resolver. Using a variable would require - # a `resolver` directive and runtime DNS. - proxy_pass https://registry.npmjs.org/; - } - - # GET /jsdelivr/npm/@/configuration-schema.json - location /jsdelivr/ { - proxy_set_header Host cdn.jsdelivr.net; - proxy_set_header Connection close; - proxy_set_header Accept-Encoding ""; - proxy_pass https://cdn.jsdelivr.net/; - } - - # GET /github/OpenFn/adaptors//packages//assets/. - # - # The conditional-GET path still works through the cache. NPM.GitHub sends - # if-none-match from the etag it stored last time and treats a 304 as - # :not_modified. nginx caches the upstream 200 along with its ETag, and on - # a hit its not-modified filter compares the client's If-None-Match against - # that ETag and downgrades the 200 to a 304 itself. There is no upstream - # hop, and Lightning's etag bookkeeping sees what GitHub would have sent. - location /github/ { - proxy_set_header Host raw.githubusercontent.com; - proxy_set_header Connection close; - proxy_set_header Accept-Encoding ""; - proxy_pass https://raw.githubusercontent.com/; - } - - location / { - return 404 "adaptor_cache: use /npm/, /jsdelivr/ or /github/\n"; - } - } -} From 05c8ece757edad1f08728d52f816a8f9de10b674 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Mon, 31 Aug 2026 17:19:29 +0200 Subject: [PATCH 05/37] Supervise the adaptors subsystem one_for_one, drop unused build steps - Adaptors subsystem supervised :one_for_one - ADAPTORS_ICONS_PATH documented so the icon cache can outlive a restart - New Credential type list now read from Lightning.Adaptors - Now-unused adaptor icon/schema build steps dropped - WorkflowChannel replies with an error instead of crashing on unrecognised messages --- DEPLOYMENT.md | 1 + Dockerfile | 4 - Dockerfile-dev | 2 - RUNNINGLOCAL.md | 2 - bin/bootstrap.d/common.sh | 2 - lib/lightning/adaptors/supervisor.ex | 15 +-- .../channels/workflow_channel.ex | 35 ++++++- .../credential_form_component.ex | 33 +++---- .../adaptors/channel_broadcaster_test.exs | 2 +- test/lightning/adaptors/config_test.exs | 7 ++ test/lightning/adaptors/invalidator_test.exs | 2 +- test/lightning/adaptors/node_monitor_test.exs | 2 +- test/lightning/adaptors/scheduler_test.exs | 4 +- .../adaptors/supervisor_integration_test.exs | 95 +++++++------------ .../channels/workflow_channel_test.exs | 42 ++++++-- .../live/credential_live_test.exs | 2 +- test/support/adaptor_test_helpers.ex | 30 ++++-- 17 files changed, 156 insertions(+), 124 deletions(-) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index b6047daa1b6..bb723481c2d 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -185,6 +185,7 @@ For SMTP, the following environment variables are required: | **Variable** | Description | | ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ADAPTORS_ICONS_PATH` | Directory the on-disk adaptor icon cache is written to. Defaults to a subdirectory of the system temp directory, which most container platforms wipe on restart; point it at a persistent volume so cached icons survive a restart. | | `ADAPTORS_PATH` | Where you store your locally installed adaptors | | `ALLOW_SIGNUP` | Set to `true` to enable user access to the registration page. Set to `false` to disable new user registrations and block access to the registration page.
Default is `false`. | | `CORS_ORIGIN` | A list of acceptable hosts for browser/cors requests (',' separated) | diff --git a/Dockerfile b/Dockerfile index 15d3991eba6..6ffcbb0a054 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,8 +62,6 @@ COPY lib lib COPY assets assets RUN mix lightning.install_runtime -RUN mix lightning.install_adaptor_icons -RUN mix lightning.install_schemas RUN npm install --prefix assets # compile assets @@ -121,12 +119,10 @@ ENV ADAPTORS_PATH=/app/priv/openfn # Only copy the final release and the adaptor directory from the build stage COPY --from=builder --chown=lightning:root /app/_build/${MIX_ENV}/rel/lightning ./ COPY --from=builder --chown=lightning:root /app/priv/openfn ./priv/openfn -COPY --from=builder --chown=lightning:root /app/priv/schemas ./priv/schemas COPY --from=builder --chown=lightning:root /app/priv/github ./priv/github USER lightning -ENV SCHEMAS_PATH="/app/priv/schemas" ENV COMMIT=${COMMIT} ENV BRANCH=${BRANCH} ENV IMAGE_TAG=${IMAGE_TAG} diff --git a/Dockerfile-dev b/Dockerfile-dev index 787d38a582d..4930c1ef6d2 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -67,8 +67,6 @@ RUN mix deps.compile COPY priv priv COPY lib lib RUN mix lightning.install_runtime -RUN mix lightning.install_schemas -RUN mix lightning.install_adaptor_icons COPY bin bin diff --git a/RUNNINGLOCAL.md b/RUNNINGLOCAL.md index 1edb14ee0cf..5fda8dcb06e 100644 --- a/RUNNINGLOCAL.md +++ b/RUNNINGLOCAL.md @@ -79,8 +79,6 @@ mix local.rebar --force [[ $(uname -m) == 'arm64' ]] && CPATH=/opt/homebrew/include LIBRARY_PATH=/opt/homebrew/lib mix deps.compile enacl # Force compile enacl if on M1 [[ $(uname -m) == 'arm64' ]] && mix compile.rambo # Force compile rambo if on M1 mix lightning.install_runtime -mix lightning.install_schemas -mix lightning.install_adaptor_icons mix ecto.create mix ecto.migrate npm install --prefix assets diff --git a/bin/bootstrap.d/common.sh b/bin/bootstrap.d/common.sh index ed744535c6d..2ead15c3943 100755 --- a/bin/bootstrap.d/common.sh +++ b/bin/bootstrap.d/common.sh @@ -192,8 +192,6 @@ setup_project_directory() { step "Installing Lightning components" mix lightning.install_runtime - mix lightning.install_schemas - mix lightning.install_adaptor_icons } setup_project_database() { diff --git a/lib/lightning/adaptors/supervisor.ex b/lib/lightning/adaptors/supervisor.ex index 2cc47205ddb..ace6c948c86 100644 --- a/lib/lightning/adaptors/supervisor.ex +++ b/lib/lightning/adaptors/supervisor.ex @@ -2,12 +2,13 @@ defmodule Lightning.Adaptors.Supervisor do @moduledoc """ Per-instance supervisor for the `Lightning.Adaptors.*` subsystem. - The entire subsystem boots, crashes, and is supervised as a unit - under `:rest_for_one`. `Cachex` is the first child: if it crashes, - the supervisor restarts it and every child listed after it - (`Task.Supervisor`, `Invalidator`, `NodeMonitor`, `ChannelBroadcaster`, - and the `HighlanderPG`-wrapped `Scheduler`) so they re-bind to the - fresh Cachex name on the way back up. + Children boot in list order — `Cachex` and the `Task.Supervisor` + before the processes that use them — but are supervised + `:one_for_one`. Every child addresses its collaborators by registered + name, resolved per call, so a restarted `Cachex` or `Task.Supervisor` + re-registers under the same name and nothing has to be rebuilt around + it. Cascading a restart would only cost the `HighlanderPG`-wrapped + `Scheduler` its advisory lock and force a needless re-election. No registered name, Cachex table name, PubSub topic, `Task.Supervisor` name, or `HighlanderPG` lock key is hardcoded. Every name is derived @@ -118,7 +119,7 @@ defmodule Lightning.Adaptors.Supervisor do ) ] - Supervisor.init(children, strategy: :rest_for_one) + Supervisor.init(children, strategy: :one_for_one) end @doc """ diff --git a/lib/lightning_web/channels/workflow_channel.ex b/lib/lightning_web/channels/workflow_channel.ex index e0301698ddc..b11fb8e8a18 100644 --- a/lib/lightning_web/channels/workflow_channel.ex +++ b/lib/lightning_web/channels/workflow_channel.ex @@ -944,6 +944,16 @@ defmodule LightningWeb.WorkflowChannel do {:reply, {:ok, %{}}, socket} end + # Catch-all for any event this channel doesn't recognise (e.g. a stale + # client tab still sending an event removed in a later deploy). Replies + # with an error instead of raising FunctionClauseError, which would kill + # the channel process and disconnect every collaborator in the room. + @impl true + def handle_in(event, _payload, socket) do + warn_unhandled_message("handle_in", event) + {:reply, {:error, %{reason: "unknown event: #{event}"}}, socket} + end + @impl true def handle_info({:yjs, chunk}, socket) do push(socket, "yjs", {:binary, chunk}) @@ -1163,12 +1173,13 @@ defmodule LightningWeb.WorkflowChannel do {:noreply, socket} end + # Catch-all for any internal message this channel doesn't recognise (e.g. a + # PubSub broadcast for an event type removed in a later deploy). Logs and + # keeps the channel alive instead of raising FunctionClauseError, which + # would kill the process and disconnect every collaborator in the room. @impl true def handle_info(message, socket) do - Logger.warning(fn -> - "WorkflowChannel: unhandled message #{inspect(message, limit: 5)} " <> - "on workflow #{socket.assigns[:workflow_id]}" - end) + warn_unhandled_message("handle_info", unhandled_message_type(message)) {:noreply, socket} end @@ -1248,6 +1259,22 @@ defmodule LightningWeb.WorkflowChannel do {:noreply, socket} end + # Logs and reports to Sentry that a channel message went unhandled, by + # event name only. The full message/payload is never logged since it may + # carry user or workflow data. + defp warn_unhandled_message(kind, event) do + Logger.warning("WorkflowChannel: unhandled #{kind} event: #{event}") + + Sentry.capture_message( + "WorkflowChannel: unhandled #{kind} event: #{event}", + level: :warning + ) + end + + defp unhandled_message_type(%{event: event}), do: event + defp unhandled_message_type(%struct{}), do: inspect(struct) + defp unhandled_message_type(_msg), do: "unrecognised" + defp list_all_packages do case Lightning.Adaptors.packages() do {:ok, pkgs} -> pkgs diff --git a/lib/lightning_web/live/credential_live/credential_form_component.ex b/lib/lightning_web/live/credential_live/credential_form_component.ex index 8dd48c2c789..6fe68b20293 100644 --- a/lib/lightning_web/live/credential_live/credential_form_component.ex +++ b/lib/lightning_web/live/credential_live/credential_form_component.ex @@ -4,8 +4,10 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do """ use LightningWeb, :live_component + alias Lightning.Adaptors alias Lightning.Credentials alias Lightning.OauthClients + alias LightningWeb.AdaptorIconURL alias LightningWeb.Components.NewInputs alias LightningWeb.CredentialLive.GenericOauthComponent alias LightningWeb.CredentialLive.Helpers @@ -1170,22 +1172,14 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do "Environment names organize credential configurations by deployment stage. When workflows run in sandbox projects (e.g., env: 'staging'), they automatically use the matching credential environment. Choose names that align with your project environments: 'production' for live systems, 'staging' for testing, 'development' for local work. Consistent naming ensures the right secrets are used in each environment." end - defp get_type_options(schemas_path) do - schemas_options = - Path.wildcard("#{schemas_path}/*.json") - |> Enum.map(fn p -> - name = p |> Path.basename() |> String.replace(".json", "") - - image_path = - Routes.static_path( - LightningWeb.Endpoint, - "/images/adaptors/#{name}-square.png" - ) - - {name, name, image_path, nil} - end) + defp get_type_options do + adaptor_options = + case Adaptors.packages() do + {:ok, packages} -> Enum.map(packages, &adaptor_type_option/1) + {:error, _} -> [] + end - schemas_options + adaptor_options |> Enum.reject(fn {_, name, _, _} -> name in ["googlesheets", "gmail", "collections"] end) @@ -1199,6 +1193,10 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do |> Enum.sort_by(&String.downcase(elem(&1, 0)), :asc) end + defp adaptor_type_option(%{name: name} = meta) do + {name, name, AdaptorIconURL.build(name, meta, :square), nil} + end + defp list_users do Lightning.Accounts.list_users() |> Enum.map(fn user -> @@ -1324,9 +1322,6 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do type_options = if action == :new do - {:ok, schemas_path} = - Application.fetch_env(:lightning, :schemas_path) - keychain_option = if socket.assigns[:from_collab_editor] do [ @@ -1340,7 +1335,7 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do [] end - get_type_options(schemas_path) + get_type_options() |> Enum.concat( Enum.map(oauth_clients, fn client -> {client.name, client.id, "/images/oauth-2.png", "oauth"} diff --git a/test/lightning/adaptors/channel_broadcaster_test.exs b/test/lightning/adaptors/channel_broadcaster_test.exs index de6c447da2f..1eff41bc120 100644 --- a/test/lightning/adaptors/channel_broadcaster_test.exs +++ b/test/lightning/adaptors/channel_broadcaster_test.exs @@ -12,7 +12,7 @@ defmodule Lightning.Adaptors.ChannelBroadcasterTest do setup do sup = :"cb_test_#{System.unique_integer([:positive])}" - # :rest_for_one starts the ChannelBroadcaster automatically, registered + # The supervisor starts the ChannelBroadcaster automatically, registered # under `channel_broadcaster_name(sup)`. start_supervised!( {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} diff --git a/test/lightning/adaptors/config_test.exs b/test/lightning/adaptors/config_test.exs index cce00b3d670..7a3af965c44 100644 --- a/test/lightning/adaptors/config_test.exs +++ b/test/lightning/adaptors/config_test.exs @@ -63,6 +63,13 @@ defmodule Lightning.Adaptors.ConfigTest do assert Config.cache_timeout_ms() == 15_000 end + + test "icon_path/0 defaults to a subdirectory of the system temp dir" do + delete_parent_key(:icon_path) + + assert Config.icon_path() == + Path.join(System.tmp_dir!(), "lightning/adaptor_icons") + end end defp put_parent(key, value) do diff --git a/test/lightning/adaptors/invalidator_test.exs b/test/lightning/adaptors/invalidator_test.exs index 365925a6822..f0a096b8265 100644 --- a/test/lightning/adaptors/invalidator_test.exs +++ b/test/lightning/adaptors/invalidator_test.exs @@ -6,7 +6,7 @@ defmodule Lightning.Adaptors.InvalidatorTest do setup do sup = :"inv_test_#{System.unique_integer([:positive])}" - # :rest_for_one starts the Invalidator automatically, registered under + # The supervisor starts the Invalidator automatically, registered under # `invalidator_name(sup)`. start_supervised!( {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} diff --git a/test/lightning/adaptors/node_monitor_test.exs b/test/lightning/adaptors/node_monitor_test.exs index f7888e1142d..e4b0132340e 100644 --- a/test/lightning/adaptors/node_monitor_test.exs +++ b/test/lightning/adaptors/node_monitor_test.exs @@ -11,7 +11,7 @@ defmodule Lightning.Adaptors.NodeMonitorTest do setup do sup = :"nm_test_#{System.unique_integer([:positive])}" - # :rest_for_one starts the NodeMonitor automatically, registered under + # The supervisor starts the NodeMonitor automatically, registered under # `node_monitor_name(sup)`. start_supervised!( {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index 12eb6696352..ae807d27ef8 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -16,8 +16,8 @@ defmodule Lightning.Adaptors.SchedulerTest do setup :verify_on_exit! # Each test owns an isolated supervisor. The supervisor starts its own - # Scheduler as part of the :rest_for_one child list, but with the - # test-env `refresh_interval: 0` it's an inert no-op. Individual tests + # Scheduler as part of its child list, but with the test-env + # `refresh_interval: 0` it's an inert no-op. Individual tests # call `start_scheduler/2` to replace it with a controlled-interval # Scheduler under `start_supervised!/1` (so Mox expectations can be # registered before init fires). diff --git a/test/lightning/adaptors/supervisor_integration_test.exs b/test/lightning/adaptors/supervisor_integration_test.exs index 52726311f4c..47835a9f5b6 100644 --- a/test/lightning/adaptors/supervisor_integration_test.exs +++ b/test/lightning/adaptors/supervisor_integration_test.exs @@ -1,10 +1,9 @@ defmodule Lightning.Adaptors.SupervisorIntegrationTest do @moduledoc """ Integration-level tests for `Lightning.Adaptors.Supervisor`: prove the full - child list boots under a single `start_supervised!` call, and that the - `:rest_for_one` cascade restarts Invalidator when Cachex restarts (§6.5a). - Invalidator subscribes at init, so without that restart the cache would go - stale. + child list boots under a single `start_supervised!` call, and that a crash in + any of the siblings restarts only that sibling, leaving the HighlanderPG + Scheduler holding its advisory lock. """ use Lightning.DataCase, async: false @@ -110,8 +109,13 @@ defmodule Lightning.Adaptors.SupervisorIntegrationTest do end end - describe ":rest_for_one strategy" do - test "Cachex crash cascades to Invalidator / ChannelBroadcaster / Scheduler", + describe ":one_for_one strategy" do + # Two victims only: the supervisor's default max_restarts is 3 in 5s, + # and hitting that ceiling would take the whole instance down for + # reasons that have nothing to do with what we're asserting. `:tasks` + # is the interesting one — the Scheduler uses it on every tick, and + # still doesn't need restarting alongside it. + test "a sibling crash restarts only that sibling, leaving the Scheduler's leadership intact", %{sup: sup} do start_supervised!( {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.Local} @@ -121,71 +125,36 @@ defmodule Lightning.Adaptors.SupervisorIntegrationTest do # globally so we have a baseline pid to compare against. assert_eventually(is_pid(scheduler_pid(sup)), @scheduler_wait_ms) - before = pids_by_role(sup) + leader = scheduler_pid(sup) + highlander = Process.whereis(AdaptorsSupervisor.highlander_name(sup)) - cachex_pid = Map.fetch!(before, :cache) - assert is_pid(cachex_pid) + for role <- [:tasks, :broadcaster] do + victim = Map.fetch!(pids_by_role(sup), role) + assert is_pid(victim) - ref = Process.monitor(cachex_pid) - Process.exit(cachex_pid, :kill) - assert_receive {:DOWN, ^ref, :process, ^cachex_pid, _}, 1_000 + ref = Process.monitor(victim) + Process.exit(victim, :kill) + assert_receive {:DOWN, ^ref, :process, ^victim, _}, 1_000 - after_pids = wait_for_restart(sup, before) + assert_eventually( + is_pid(current_pid(sup, role)) and current_pid(sup, role) != victim, + 1_000 + ) - # Cachex itself comes back under a fresh pid. - assert Map.fetch!(after_pids, :cache) != cachex_pid + # HighlanderPG holds the advisory lock for as long as its wrapped + # child lives, so an unchanged Scheduler pid is an unchanged + # leader: no re-election, no lock handover. + assert scheduler_pid(sup) == leader, + "killing #{role} re-elected the Scheduler " <> + "(before=#{inspect(leader)}, after=#{inspect(scheduler_pid(sup))})" - # §6.5a: under :rest_for_one, all children that depend on Cachex - # (Invalidator, Broadcaster, Scheduler) must restart too so they - # re-bind to the fresh cache. - for role <- [:invalidator, :broadcaster, :scheduler] do - old = Map.fetch!(before, role) - new = Map.fetch!(after_pids, role) - assert is_pid(old) - assert is_pid(new) + assert Process.alive?(leader) - assert new != old, - "expected #{role} to restart after Cachex crash " <> - "(before=#{inspect(old)}, after=#{inspect(new)})" + assert Process.whereis(AdaptorsSupervisor.highlander_name(sup)) == + highlander end end end - # Polls `pids_by_role/1` until the children we expect to be restarted - # show new PIDs, or we hit the deadline. Returns the post-restart map. - # The Scheduler restart goes through HighlanderPG (lock + poll cycle), - # so allow a slightly longer deadline than for the locally-registered - # children alone. - defp wait_for_restart(sup, before, deadline_ms \\ 3_000) do - start = System.monotonic_time(:millisecond) - roles_expected = [:invalidator, :broadcaster, :scheduler] - do_wait_for_restart(sup, before, roles_expected, start, deadline_ms) - end - - defp do_wait_for_restart(sup, before, roles, start, deadline_ms) do - current = pids_by_role(sup) - - changed? = - Enum.all?(roles, fn role -> - case {Map.get(before, role), Map.get(current, role)} do - {old, new} when is_pid(old) and is_pid(new) -> old != new - _ -> false - end - end) - - cond do - changed? -> - current - - System.monotonic_time(:millisecond) - start > deadline_ms -> - flunk( - "supervisor children did not restart within #{deadline_ms}ms; " <> - "before=#{inspect(before)} after=#{inspect(current)}" - ) - - true -> - Process.sleep(20) - do_wait_for_restart(sup, before, roles, start, deadline_ms) - end - end + defp current_pid(sup, role), do: Map.get(pids_by_role(sup), role) end diff --git a/test/lightning_web/channels/workflow_channel_test.exs b/test/lightning_web/channels/workflow_channel_test.exs index e2ea537642a..0e1d7b64136 100644 --- a/test/lightning_web/channels/workflow_channel_test.exs +++ b/test/lightning_web/channels/workflow_channel_test.exs @@ -4690,12 +4690,7 @@ defmodule LightningWeb.WorkflowChannelTest do } end - test "does not push adaptors_updated for unrelated events on client topic", - %{socket: socket} do - Process.flag(:trap_exit, true) - Process.unlink(socket.channel_pid) - ref = Process.monitor(socket.channel_pid) - + test "does not push adaptors_updated for unrelated events on client topic" do capture_log(fn -> Phoenix.PubSub.broadcast( Lightning.PubSub, @@ -4704,11 +4699,44 @@ defmodule LightningWeb.WorkflowChannelTest do ) refute_push "adaptors_updated", _, 50 - assert_receive {:DOWN, ^ref, :process, _, _}, 200 end) end end + describe "unrecognised channel messages" do + test "handle_in replies with an error instead of crashing the channel", + %{socket: socket} do + log = + capture_log(fn -> + ref = push(socket, "request_project_adaptors", %{}) + + assert_reply ref, :error, %{ + reason: "unknown event: request_project_adaptors" + } + end) + + assert log =~ "unhandled handle_in event: request_project_adaptors" + assert Process.alive?(socket.channel_pid) + end + + test "handle_info logs and stays alive for an unrecognised internal broadcast", + %{socket: socket, workflow: workflow} do + log = + capture_log(fn -> + Phoenix.PubSub.broadcast( + Lightning.PubSub, + "workflow:collaborate:#{workflow.id}", + %{event: "some_future_event", payload: %{}} + ) + + refute_push "some_future_event", _, 50 + end) + + assert log =~ "unhandled handle_info event: some_future_event" + assert Process.alive?(socket.channel_pid) + end + end + describe "request_history" do test "returns work orders with runs for workflow", %{ socket: socket, diff --git a/test/lightning_web/live/credential_live_test.exs b/test/lightning_web/live/credential_live_test.exs index cc89c3730d0..8b89058d4e6 100644 --- a/test/lightning_web/live/credential_live_test.exs +++ b/test/lightning_web/live/credential_live_test.exs @@ -2770,7 +2770,7 @@ defmodule LightningWeb.CredentialLiveTest do adaptor_icon = Floki.find(adaptor_label, "object") assert length(adaptor_icon) > 0 img_src = adaptor_icon |> Floki.attribute("data") |> List.first() - assert img_src =~ "/images/adaptors/#{adaptor}-square.png" + assert img_src =~ "/adaptors/icons/#{adaptor}/square-" end end end diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex index b730402aca9..8ef8e0b5d2f 100644 --- a/test/support/adaptor_test_helpers.ex +++ b/test/support/adaptor_test_helpers.ex @@ -106,15 +106,29 @@ defmodule Lightning.AdaptorTestHelpers do """ @spec seed_all_credential_schemas() :: :ok def seed_all_credential_schemas do - Path.wildcard("test/fixtures/schemas/*.json") - |> Enum.each(fn path -> - # Skip empty fixture files — some (e.g. `asana.json`, - # `primero.json`) are intentional empty placeholders. - if File.stat!(path).size > 0 do + metas = + Path.wildcard("test/fixtures/schemas/*.json") + |> Enum.reject(fn path -> File.stat!(path).size == 0 end) + |> Enum.map(fn path -> short_name = path |> Path.basename(".json") - seed_credential_schema(short_name) - end - end) + row = seed_credential_schema(short_name) + + %{ + name: short_name, + latest_version: row.latest_version, + description: nil, + deprecated: false, + icon_square_ext: "png", + icon_rectangle_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, short_name <> "-square"), + icon_rectangle_sha256: + :crypto.hash(:sha256, short_name <> "-rectangle") + } + end) + + cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) + source = AdaptorsSupervisor.source(Lightning.Adaptors) + Cachex.put(cache, {:packages, source}, {:ok, metas}) :ok end From 54987b5cec0f54e47a876d610e29c56a423e745f Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Wed, 2 Sep 2026 07:24:12 +0200 Subject: [PATCH 06/37] Cache the catalogue projection through Store, restore picker filters - Lightning.Adaptors waits for the catalogue's first load - Catalogue projection and stamp cached through Store - Adaptor exclusion filter restored, source/strategy fallback fixed, adaptor mix tasks renamed - Local-mode's "local" picker display restored after the registry rewrite --- .claude/rules/adaptors-docs.md | 49 ++ .claude/rules/adaptors-otp.md | 42 ++ .env.example | 6 +- RUNNINGLOCAL.md | 8 +- .../collaborative-editor/hooks/useChannel.ts | 12 +- .../stores/createWorkflowStore.ts | 5 +- .../createWorkflowStore.reconcile.test.ts | 3 +- .../stores/createWorkflowStore.test.ts | 6 +- config/test.exs | 14 +- lib/lightning/adaptor_service.ex | 4 +- lib/lightning/adaptors.ex | 457 +++++++++------ .../adaptors/{repo.ex => catalogue.ex} | 115 ++-- .../{repo_adaptor.ex => catalogue_adaptor.ex} | 38 +- ...ersion.ex => catalogue_adaptor_version.ex} | 19 +- lib/lightning/adaptors/config.ex | 78 +-- lib/lightning/adaptors/invalidator.ex | 9 +- lib/lightning/adaptors/package_name.ex | 82 ++- lib/lightning/adaptors/scheduler.ex | 254 +++++--- lib/lightning/adaptors/seed.ex | 82 +++ lib/lightning/adaptors/store.ex | 215 ++++--- lib/lightning/adaptors/supervisor.ex | 126 +--- lib/lightning/ai_assistant/ai_assistant.ex | 12 +- lib/lightning/collaboration/session.ex | 44 +- lib/lightning/release.ex | 2 +- lib/lightning/workflows/job.ex | 14 +- lib/lightning_web/channels/run_channel.ex | 8 +- .../channels/run_with_options.ex | 53 +- .../channels/workflow_channel.ex | 541 ++++++++++-------- .../controllers/adaptor_controller.ex | 26 +- .../credential_form_component.ex | 4 +- .../live/maintenance_live/index.ex | 54 +- lib/mix/tasks/lightning.adaptors.dump.ex | 80 +++ ...m_file.ex => lightning.adaptors.import.ex} | 14 +- lib/mix/tasks/lightning.adaptors.refresh.ex | 83 +++ ...ache.ex => lightning.adaptors.snapshot.ex} | 15 +- lib/mix/tasks/lightning.refresh_adaptors.ex | 59 -- test/integration/web_and_worker_test.exs | 2 - test/lightning/adaptor_service_test.exs | 110 ++++ ...or_test.exs => catalogue_adaptor_test.exs} | 28 +- ...exs => catalogue_adaptor_version_test.exs} | 6 +- ...ue_test.exs => catalogue_listing_test.exs} | 60 +- .../{repo_test.exs => catalogue_test.exs} | 190 +++--- test/lightning/adaptors/config_test.exs | 32 +- test/lightning/adaptors/invalidator_test.exs | 4 +- test/lightning/adaptors/node_monitor_test.exs | 8 +- test/lightning/adaptors/package_name_test.exs | 44 +- test/lightning/adaptors/readiness_test.exs | 304 ++++++++++ test/lightning/adaptors/scheduler_test.exs | 165 +++++- test/lightning/adaptors/seed_test.exs | 132 +++++ test/lightning/adaptors/store_test.exs | 247 +++++++- test/lightning/adaptors_test.exs | 195 ++++--- .../ai_assistant/ai_assistant_test.exs | 10 +- .../ai_assistant/unsaved_job_test.exs | 2 +- .../collaboration/no_change_snapshot_test.exs | 2 + .../collaboration/session_readiness_test.exs | 170 ++++++ test/lightning/collaboration/session_test.exs | 4 + test/lightning/projects/provisioner_test.exs | 28 + test/lightning/workflows/job_test.exs | 24 +- test/lightning/workflows_test.exs | 56 ++ .../channels/run_channel_test.exs | 40 +- .../channels/run_with_options_test.exs | 62 +- .../channels/workflow_channel_test.exs | 96 +++- .../controllers/adaptor_controller_test.exs | 21 +- .../adaptor_icon_controller_test.exs | 4 +- .../live/credential_live_test.exs | 2 - .../live/maintenance_live/index_test.exs | 33 +- test/lightning_web/live/project_live_test.exs | 3 - .../live/workflow_live/collaborate_test.exs | 3 - test/mix/tasks/gen_workflow_hash_test.exs | 7 + .../tasks/lightning.adaptors.dump_test.exs | 209 +++++++ ...exs => lightning.adaptors.import_test.exs} | 36 +- .../tasks/lightning.adaptors.refresh_test.exs | 116 ++++ .../lightning.adaptors.snapshot_test.exs} | 10 +- .../tasks/lightning.refresh_adaptors_test.exs | 85 --- test/support/adaptor_test_helpers.ex | 247 ++------ test/support/factories.ex | 2 +- test/test_helper.exs | 5 + tooling/adaptor_cache/README.md | 4 +- tooling/adaptor_cache/lib/cli.ex | 2 +- 79 files changed, 3760 insertions(+), 1683 deletions(-) create mode 100644 .claude/rules/adaptors-docs.md create mode 100644 .claude/rules/adaptors-otp.md rename lib/lightning/adaptors/{repo.ex => catalogue.ex} (78%) rename lib/lightning/adaptors/{repo_adaptor.ex => catalogue_adaptor.ex} (72%) rename lib/lightning/adaptors/{repo_adaptor_version.ex => catalogue_adaptor_version.ex} (70%) create mode 100644 lib/lightning/adaptors/seed.ex create mode 100644 lib/mix/tasks/lightning.adaptors.dump.ex rename lib/mix/tasks/{seed_adaptors_from_file.ex => lightning.adaptors.import.ex} (74%) create mode 100644 lib/mix/tasks/lightning.adaptors.refresh.ex rename lib/mix/tasks/{download_adaptor_registry_cache.ex => lightning.adaptors.snapshot.ex} (73%) delete mode 100644 lib/mix/tasks/lightning.refresh_adaptors.ex create mode 100644 test/lightning/adaptor_service_test.exs rename test/lightning/adaptors/{repo_adaptor_test.exs => catalogue_adaptor_test.exs} (90%) rename test/lightning/adaptors/{repo_adaptor_version_test.exs => catalogue_adaptor_version_test.exs} (96%) rename test/lightning/adaptors/{repo_catalogue_test.exs => catalogue_listing_test.exs} (69%) rename test/lightning/adaptors/{repo_test.exs => catalogue_test.exs} (62%) create mode 100644 test/lightning/adaptors/readiness_test.exs create mode 100644 test/lightning/adaptors/seed_test.exs create mode 100644 test/lightning/collaboration/session_readiness_test.exs create mode 100644 test/mix/tasks/lightning.adaptors.dump_test.exs rename test/mix/tasks/{seed_adaptors_from_file_test.exs => lightning.adaptors.import_test.exs} (73%) create mode 100644 test/mix/tasks/lightning.adaptors.refresh_test.exs rename test/{lightning/download_adaptor_registry_test.exs => mix/tasks/lightning.adaptors.snapshot_test.exs} (91%) delete mode 100644 test/mix/tasks/lightning.refresh_adaptors_test.exs diff --git a/.claude/rules/adaptors-docs.md b/.claude/rules/adaptors-docs.md new file mode 100644 index 00000000000..cd2b1862828 --- /dev/null +++ b/.claude/rules/adaptors-docs.md @@ -0,0 +1,49 @@ +--- +paths: + - "lib/lightning/adaptors.ex" + - "lib/lightning/adaptors/**" + - "lib/lightning_web/controllers/adaptor_icon_controller.ex" + - "lib/mix/tasks/lightning.adaptors.refresh.ex" + - "lib/mix/tasks/lightning.adaptors.import.ex" + - "lib/mix/tasks/lightning.adaptors.dump.ex" + - "lib/mix/tasks/lightning.adaptors.snapshot.ex" + - "test/lightning/adaptors_test.exs" + - "test/lightning/adaptors/**" + - "test/mix/tasks/lightning.adaptors.*.exs" + - "assets/js/collaborative-editor/stores/createAdaptorStore.ts" + - "assets/js/collaborative-editor/types/adaptor.ts" + - ".context/adaptors/**" +--- + +# Adaptors: which documents to trust + +Most of `.context/adaptors/` is archaeology from designs that were abandoned before they +shipped. Grep will find it and it reads convincingly. Everything at the top level of that +folder is current, and there are five things: + +- `README.md` — the entry point, and the shortest thing to read. +- `ATLAS.md` — the architecture in seven diagrams, stamped with the commit it describes. + Start here to understand the shape of the subsystem. +- `REWRITE-2026-05.md` — the canonical spec. Per-callback contracts and the reasoning behind + each decision. Grep it, don't read it end to end. +- `07-channel-live-update-findings-2026-06-03.md` — two decisions still open, still blocking. +- `NOTES.md` — a running log of open questions and irregularities hit while working on the + subsystem, newest entry on top. Dated entries, none of them acted on yet. Read it before + concluding you have found a new bug. + +Everything under `.context/adaptors/archive/` is superseded, and every file there carries an +ARCHIVED banner saying why. Do not cite it, follow it, or use it to answer a question about +how the subsystem works. Two traps worth naming: `archive/ARCHITECTURE.md` diagrams the +abandoned PR #4473 design (blob table plus Oban) in convincing detail and shares almost +nothing with what shipped, and `archive/NOTES.md` is a dead namesake of the live `NOTES.md` +above — check which one you opened. + +Live status is the PR, not the folder: `gh pr view 4801 --json body -q .body`. Don't +reconstruct that checklist anywhere else, and don't infer completion state from the archived +phase-A/phase-B PRDs — both phases shipped, so those describe code that already exists. + +If you change the subsystem's shape, update `ATLAS.md` and move its commit stamp. The reason +that folder needed archiving is that nobody did this last time. + +Process and naming conventions for this subsystem are a separate rule: +`.claude/rules/adaptors-otp.md`. diff --git a/.claude/rules/adaptors-otp.md b/.claude/rules/adaptors-otp.md new file mode 100644 index 00000000000..51755120bb9 --- /dev/null +++ b/.claude/rules/adaptors-otp.md @@ -0,0 +1,42 @@ +--- +paths: + - "lib/lightning/adaptors.ex" + - "lib/lightning/adaptors/**/*.ex" + - "test/lightning/adaptors_test.exs" + - "test/lightning/adaptors/**/*.exs" +--- + +# Adaptors: naming and dependency injection + +Every process in this subsystem derives its name from the single `:name` opt passed +to `Lightning.Adaptors.Supervisor`. Nothing is hardcoded, which is what lets the +integration suite run isolated instances in one BEAM under `async: true`. Adding a +process that breaks this forces the whole suite serial. + +When adding or changing a process here: + +- Take `:name` from opts (`Keyword.fetch!(opts, :name)`) and derive any child, + cache, topic or lock name from it. Follow the helpers at + `lib/lightning/adaptors/supervisor.ex:159-205`. +- Add it to the fixed child list in `init/1` (`supervisor.ex:101-122`) with its + collaborators passed in the child spec. Do not add a `Registry`: the child set is + fixed and the registered atom already addresses it. +- Public functions that talk to a running process lead with the server ref, + defaulted: `def refresh(sup \\ @sup, name)`. `start_link` takes `name:` in + trailing opts. +- The `Scheduler` is a cluster singleton behind `HighlanderPG` and registers under + `global_scheduler_name/1` (`supervisor.ex:196`). `Process.whereis` will not find + it. + +Known wart, do not copy it: `strategy` and `source` are published to +`:persistent_term` in `init/1` (`supervisor.ex:74-77`) and re-read at call time by +`Scheduler` (`scheduler.ex:156-160`, `:166`, `:194`) and `Store`. New code should +take them from process state or the child spec instead. `Scheduler` already does +this correctly for `source` (`scheduler.ex:111`, `:127-135`). + +In tests, prefer `Mox.allow(StrategyMock, self(), pid)` and keep `async: true`, as +`test/lightning/adaptors/store_test.exs:115` does. `set_mox_global` costs the file +its async, and is only justified where the hop graph is genuinely dynamic, as in +`highlander_integration_test.exs:27`. + +Full reasoning: `.claude/guidelines/testable-supervision-trees.md`. diff --git a/.env.example b/.env.example index a48e63a3bfa..4b6b8ca7df9 100644 --- a/.env.example +++ b/.env.example @@ -245,10 +245,12 @@ # To boot from a static adaptor catalogue snapshot instead of reaching npm, # seed it before starting the app. In dev/CI (Mix available): -# mix lightning.seed_adaptors_from_file --path /path/to/snapshot.json +# mix lightning.adaptors.import --path /path/to/snapshot.json # In a release (no Mix), from the app directory instead: # bin/lightning eval 'Lightning.Release.seed_adaptors("/path/to/snapshot.json")' -# See `mix help lightning.seed_adaptors_from_file`. +# See `mix help lightning.adaptors.import`. To create a snapshot file, use +# `mix lightning.adaptors.dump` (from an existing catalogue) or +# `mix lightning.adaptors.snapshot` (straight from npm, no DB needed). # # Enable local adaptors mode. OPENFN_ADAPTORS_REPO takes one repo path, or a # comma-separated list to merge several. See RUNNINGLOCAL.md for the details. diff --git a/RUNNINGLOCAL.md b/RUNNINGLOCAL.md index 5fda8dcb06e..f1fca559c66 100644 --- a/RUNNINGLOCAL.md +++ b/RUNNINGLOCAL.md @@ -210,11 +210,11 @@ next run! ### Caching the adaptor upstreams Every adaptor registry refresh (background scheduler tick, or a manual -`mix lightning.refresh_adaptors`) makes a handful of npm, jsDelivr and GitHub +`mix lightning.adaptors.refresh`) makes a handful of npm, jsDelivr and GitHub requests per changed package, which gets chatty fast if you're iterating on the -subsystem or just running `refresh_adaptors` repeatedly by hand. A local -record-and-replay reverse proxy under `tooling/adaptor_cache/` makes the second -and every later run local, with no network needed at all. +subsystem or just running `lightning.adaptors.refresh` repeatedly by hand. A +local record-and-replay reverse proxy under `tooling/adaptor_cache/` makes the +second and every later run local, with no network needed at all. ```sh bin/adaptor_cache up # start the proxy diff --git a/assets/js/collaborative-editor/hooks/useChannel.ts b/assets/js/collaborative-editor/hooks/useChannel.ts index e8eea9aa494..822db55e4b0 100644 --- a/assets/js/collaborative-editor/hooks/useChannel.ts +++ b/assets/js/collaborative-editor/hooks/useChannel.ts @@ -59,6 +59,7 @@ export interface ChannelError { | 'optimistic_lock_error' | 'limit_error' | 'nesting_too_deep' + | 'adaptor_catalogue_unavailable' | undefined; /** @@ -71,11 +72,16 @@ export interface ChannelError { export async function channelRequest( channel: Channel, message: string, - payload: object + payload: object, + timeout?: number ): Promise { return new Promise((resolve, reject) => { - channel - .push(message, payload) + const push = + timeout === undefined + ? channel.push(message, payload) + : channel.push(message, payload, timeout); + + push .receive('ok', (response: T) => { resolve(response); }) diff --git a/assets/js/collaborative-editor/stores/createWorkflowStore.ts b/assets/js/collaborative-editor/stores/createWorkflowStore.ts index 0c9f6a5b422..cbd080fa434 100644 --- a/assets/js/collaborative-editor/stores/createWorkflowStore.ts +++ b/assets/js/collaborative-editor/stores/createWorkflowStore.ts @@ -154,6 +154,7 @@ const logger = _logger.ns('WorkflowStore').seal(); const JobShape = JobSchema.shape; const EdgeShape = EdgeSchema.shape; +const SAVE_TIMEOUT_MS = 75_000; // Helper to update derived state (defined first to avoid hoisting issues) function updateDerivedState(draft: Workflow.State) { @@ -1572,7 +1573,7 @@ export const createWorkflowStore = ( saved_at: string; lock_version: number; workflow: BaseWorkflow; - }>(provider.channel, 'save_workflow', payload); + }>(provider.channel, 'save_workflow', payload, SAVE_TIMEOUT_MS); logger.debug('Saved workflow', response); @@ -1789,7 +1790,7 @@ export const createWorkflowStore = ( lock_version: number; repo: string; workflow: BaseWorkflow; - }>(provider.channel, 'save_and_sync', payload); + }>(provider.channel, 'save_and_sync', payload, SAVE_TIMEOUT_MS); logger.debug('Saved and synced workflow to GitHub', response); diff --git a/assets/test/collaborative-editor/stores/createWorkflowStore.reconcile.test.ts b/assets/test/collaborative-editor/stores/createWorkflowStore.reconcile.test.ts index 8e69fed0b8c..3a4020e64d4 100644 --- a/assets/test/collaborative-editor/stores/createWorkflowStore.reconcile.test.ts +++ b/assets/test/collaborative-editor/stores/createWorkflowStore.reconcile.test.ts @@ -115,7 +115,8 @@ describe('WorkflowStore - Save path does not reconcile dangling references', () cron_cursor_job_id: 'job-added-by-collaborator', }), ], - }) + }), + 75_000 ); } ); diff --git a/assets/test/collaborative-editor/stores/createWorkflowStore.test.ts b/assets/test/collaborative-editor/stores/createWorkflowStore.test.ts index 521ac2d70d0..89410329f0e 100644 --- a/assets/test/collaborative-editor/stores/createWorkflowStore.test.ts +++ b/assets/test/collaborative-editor/stores/createWorkflowStore.test.ts @@ -99,7 +99,8 @@ describe('WorkflowStore - Save Workflow', () => { id: 'workflow-123', name: 'Test Workflow', lock_version: null, // Original value at time of save - }) + }), + 75_000 ); }); @@ -235,7 +236,8 @@ describe('WorkflowStore - Save Workflow', () => { positions: { 'job-1': { x: 100, y: 200 }, }, - }) + }), + 75_000 ); }); diff --git a/config/test.exs b/config/test.exs index f3d1b8a07d8..5f63d5036c1 100644 --- a/config/test.exs +++ b/config/test.exs @@ -108,20 +108,14 @@ config :lightning, :workers, # In test we don't send emails. config :lightning, Lightning.Mailer, adapter: Swoosh.Adapters.Test -# Adaptors.Supervisor config for test boot. -# -# - `:strategy` — the production `Lightning.Adaptors.Supervisor` mounted in -# `application.ex` would default to `Lightning.Adaptors.NPM` and try to -# hit the network on the first Scheduler tick. Replace it with the -# Mox-backed `StrategyMock` so the application-level supervisor (under -# the production name `Lightning.Adaptors`) is a no-op for tests that -# exercise the facade directly. -# - `:refresh_interval` — `0` disables Scheduler tick scheduling entirely. -# Per-test isolated supervisors set their own interval as needed. config :lightning, Lightning.Adaptors, strategy: Lightning.Adaptors.StrategyMock, refresh_interval: 0 +# `Config.source_for/1` only maps the two real strategies; the mock has to +# declare its catalogue source like any other third-party strategy would. +config :lightning, Lightning.Adaptors.StrategyMock, source: :npm + config :hammer, backend: {Hammer.Backend.ETS, diff --git a/lib/lightning/adaptor_service.ex b/lib/lightning/adaptor_service.ex index e3c0905489b..2c3cf3978eb 100644 --- a/lib/lightning/adaptor_service.ex +++ b/lib/lightning/adaptor_service.ex @@ -21,7 +21,7 @@ defmodule Lightning.AdaptorService do Every install is gated on the adaptor catalogue (`Lightning.Adaptors`): `install/2` refuses to run `npm install` for a package name the catalogue - doesn't recognise, including when the catalogue is empty. + doesn't recognise. ## Looking up adaptors @@ -314,7 +314,7 @@ defmodule Lightning.AdaptorService do defp known?(nil), do: false - defp known?(name), do: Adaptors.get_adaptor(name) != nil + defp known?(name), do: match?({:ok, _}, Adaptors.fetch_adaptor(name)) @spec install!(Agent.agent(), package_spec()) :: {:ok, InstalledAdaptor.t()} diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index 530b8b2d214..e0fc35ef2fc 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -1,138 +1,239 @@ defmodule Lightning.Adaptors do @moduledoc """ - Public interface to the adaptor catalogue. + The adaptor catalogue: which npm adaptors exist, their versions, schemas + and icons. - ## Catalogue reads - - * `packages/0,1` - every adaptor for the active source - * `versions/2` - published versions of one adaptor - * `get_adaptor/1` - one adaptor as a `Lightning.Adaptors.Package`, - or `nil` - * `catalogue/0` and `catalogue_stamp/0` - the full catalogue and its - ETag basis - * `schema/1,2` and `icon/2,3` - per-adaptor assets + A scheduler fetches the catalogue from the configured source and + persists it. Reads check an in-memory cache first and the database + second. The first read against an empty catalogue triggers the initial + load and waits for it, bounded by the first-load timeout, and returns an + error if the load does not complete in time. ## Adaptor specs - An adaptor spec is the `"name@version"` string a job stores, where the - version may be a semver, `latest`, `local`, or absent. - - * `parse_spec/1` - split a spec into `{name, version}` - * `valid_format?/1` - does a string match the strict spec format - * `resolve_version/2` - turn `latest`/`local` into a concrete version - * `to_wire/1` - render a spec for the worker's install step - - ## Refreshing - - * `refresh_now/0,1`, `refresh_package/1,2`, `refresh_icons/0,1` - * `seed_from_file/2` - populate the catalogue from a JSON snapshot, - used by `mix lightning.seed_adaptors_from_file` and - `Lightning.Release` - - Most functions come in a dual-arity shape: the zero-/single-arg form - passes the compile-time default supervisor name `@sup`; the extra-arity - form accepts an explicit supervisor name for test isolation. - `get_adaptor/1`, `resolve_version/2`, `catalogue/0`, and - `catalogue_stamp/0` are exceptions — none of them go through `Store`'s - cache process. `get_adaptor/1` and `resolve_version/2` read the active - source from `Config.current_source/0` (a process-independent - `Application.get_env` read), so there's nothing to swap. `catalogue/0` - and `catalogue_stamp/0` read it from `AdaptorsSupervisor.source/1` - instead — a boot-time snapshot owned by a running supervisor — so - those two are only correct for `@sup`, the default supervisor. + A spec is the `name@version` string a job stores, where the version is + a semver or one of two sentinels: `latest` resolves to the catalogue's + current version when the job runs, and `local` points the worker at an + adaptor on its own filesystem. The version may also be omitted. + + ## Configuration + + Under `config :lightning, Lightning.Adaptors`: + + * `:strategy` - the module that fetches from the source + * `:refresh_interval` - how often the scheduler re-checks the source, + in milliseconds; `0` disables the periodic tick + * `:first_load_timeout` - how long a read waits for the initial load + * `:cache_timeout_ms` - how long a read waits for a cache fill + * `:icon_path` - where fetched icons are written + + Defaults are in `Lightning.Adaptors.Config`. + + ## Testing + + Every function that talks to a running process takes the supervisor + name as an optional first argument, defaulting to `Lightning.Adaptors`. + Start a `Lightning.Adaptors.Supervisor` under another name and pass + that name to run a test against its own catalogue and cache. """ + alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Config alias Lightning.Adaptors.PackageName - alias Lightning.Adaptors.Repo alias Lightning.Adaptors.Scheduler alias Lightning.Adaptors.Store alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor defmodule Package do @moduledoc """ - One catalogue adaptor, as seen by callers outside - `Lightning.Adaptors`. Not wire-serialised and not an `Ecto.Schema`. + One catalogue adaptor. """ + defmodule Version do + @moduledoc """ + One published version of a catalogue adaptor. + """ + + @type t :: %__MODULE__{ + version: String.t(), + integrity: String.t() | nil, + size_bytes: integer() | nil, + published_at: DateTime.t() | nil, + deprecated: boolean() + } + + defstruct [ + :version, + :integrity, + :size_bytes, + :published_at, + deprecated: false + ] + end + @type t :: %__MODULE__{ name: String.t(), source: :npm | :local, - latest_version: String.t() | nil + latest_version: String.t() | nil, + description: String.t() | nil, + deprecated: boolean(), + icon_square_ext: String.t() | nil, + icon_rectangle_ext: String.t() | nil, + icon_square_sha256: binary() | nil, + icon_rectangle_sha256: binary() | nil } - defstruct [:name, :source, :latest_version] + defstruct [ + :name, + :source, + :latest_version, + :description, + :icon_square_ext, + :icon_rectangle_ext, + :icon_square_sha256, + :icon_rectangle_sha256, + deprecated: false + ] end @sup Lightning.Adaptors - @type package_meta :: Store.package_meta() - @type version_meta :: Store.version_meta() - - @spec packages() :: {:ok, [package_meta()]} | {:error, :timeout | term()} - def packages, do: packages(@sup) - - @spec packages(atom()) :: {:ok, [package_meta()]} | {:error, :timeout | term()} - def packages(sup), do: Store.packages(sup) - - @spec versions(atom(), String.t()) :: - {:ok, [version_meta()]} | {:error, term()} - def versions(sup, pkg), do: Store.versions(sup, pkg) - - @spec schema(String.t()) :: {:ok, String.t()} | {:error, term()} - def schema(pkg), do: schema(@sup, pkg) + @doc """ + Returns every adaptor in the catalogue as `Package` structs. + """ + @spec packages(atom()) :: {:ok, [Package.t()]} | {:error, :timeout | term()} + def packages(sup \\ @sup) do + with {:ok, metas} <- Store.packages(sup) do + source = AdaptorsSupervisor.source(sup) + {:ok, Enum.map(metas, &to_package(&1, source))} + end + end + @doc """ + Returns the credential schema of the adaptor named `pkg`, as a JSON + binary. + """ @spec schema(atom(), String.t()) :: {:ok, String.t()} | {:error, term()} - def schema(sup, pkg), do: Store.schema(sup, pkg) - - @spec icon(String.t(), :square | :rectangle) :: - {:ok, Path.t()} | {:error, term()} - def icon(pkg, shape), do: icon(@sup, pkg, shape) + def schema(sup \\ @sup, pkg), do: Store.schema(sup, pkg) + @doc """ + Returns the on-disk path of the adaptor's `:square` or `:rectangle` + icon, fetching it on the first request. + """ @spec icon(atom(), String.t(), :square | :rectangle) :: {:ok, Path.t()} | {:error, term()} - def icon(sup, pkg, shape), do: Store.icon(sup, pkg, shape) + def icon(sup \\ @sup, pkg, shape), do: Store.icon(sup, pkg, shape) @doc """ - Full catalogue for the active source: every adaptor's `name`, - `latest_version`, `repository`, icon fields, and full version list. - Reads `Repo` directly, like `resolve_version/2`. + Returns the picker catalogue as `{{latest_updated_at, count}, entries}`: + every adaptor with its full version list and icon URLs, rendered once + per change rather than per request, alongside the ETag basis for it. + + One read, so the stamp always describes the entries it comes with. """ - @spec catalogue() :: [Repo.catalogue_entry()] - def catalogue, do: Repo.catalogue(AdaptorsSupervisor.source(@sup)) + @spec catalogue_with_stamp(atom()) :: + {{DateTime.t() | nil, non_neg_integer()}, [Store.catalogue_entry()]} + def catalogue_with_stamp(sup \\ @sup) do + {:ok, catalogue} = Store.catalogue(sup) + catalogue + end @doc """ - ETag basis for `catalogue/0` — see `Repo.catalogue_stamp/1`. + Returns the adaptor named `name`, or `nil`. + + Takes a bare package name, not a spec; see `parse_spec/1`. Never waits + for the catalogue to load; see `fetch_adaptor/2` for that. """ - @spec catalogue_stamp() :: {DateTime.t() | nil, non_neg_integer()} - def catalogue_stamp, do: Repo.catalogue_stamp(AdaptorsSupervisor.source(@sup)) + @spec get_adaptor(atom(), String.t()) :: Package.t() | nil + def get_adaptor(sup \\ @sup, name) when is_binary(name), + do: lookup(sup, name) @doc """ - One adaptor from the active source's catalogue, by bare package name. + Returns `{:ok, adaptor}` for the adaptor named `name`, waiting for the + catalogue's first load if it has never loaded. + + Errors: - Takes a name, not a `name@version` spec — use `parse_spec/1` first if - you have a spec. Returns `nil` when the catalogue has no such adaptor, - including when it is empty. + * `{:error, :not_found}` - the loaded catalogue has no such adaptor + * `{:error, :timeout}` - the first load did not finish within + `Lightning.Adaptors.Config.first_load_timeout/0` + * `{:error, :unavailable}` - no Scheduler process is reachable + * `{:error, :not_ready}` - the load ran but left the catalogue empty """ - @spec get_adaptor(String.t()) :: Package.t() | nil - def get_adaptor(name) when is_binary(name) do - case Repo.get_adaptor(name, Config.current_source()) do + @spec fetch_adaptor(atom(), String.t()) :: + {:ok, Package.t()} + | {:error, :not_found | :timeout | :unavailable | :not_ready} + def fetch_adaptor(sup \\ @sup, name) when is_binary(name) do + case lookup(sup, name) do + %Package{} = package -> + {:ok, package} + nil -> - nil - - adaptor -> - %Package{ - name: adaptor.name, - source: adaptor.source, - latest_version: adaptor.latest_version - } + if ready?(sup), + do: {:error, :not_found}, + else: load_then_fetch(sup, name) end end + defp load_then_fetch(sup, name) do + with :ok <- load(sup) do + case lookup(sup, name) do + %Package{} = package -> {:ok, package} + nil -> {:error, :not_found} + end + end + end + + # Cache first, then the row itself: the cached list can lag a Scheduler + # write until the Invalidator drops it. + defp lookup(sup, name) do + source = AdaptorsSupervisor.source(sup) + + cached = + case Store.packages(sup) do + {:ok, metas} -> Enum.find(metas, &(&1.name == name)) + {:error, _} -> nil + end + + case cached || Catalogue.get_adaptor(name, source) do + nil -> nil + meta -> to_package(meta, source) + end + end + + defp to_package(meta, source) do + struct(Package, meta |> Map.delete(:__struct__) |> Map.put(:source, source)) + end + @doc """ - Split an adaptor spec into `{name, version}`, with `version` `nil` when - the spec carries none. Returns `{nil, nil}` for a string that isn't a - well-formed spec. + Waits until the catalogue has loaded at least once, triggering the + first load if needed. + + Returns `:ok`, or one of the `fetch_adaptor/2` errors other than + `:not_found`. + """ + @spec ensure_loaded(atom()) :: + :ok | {:error, :timeout | :unavailable | :not_ready} + def ensure_loaded(sup \\ @sup) do + if ready?(sup), do: :ok, else: load(sup) + end + + defp load(sup) do + case refresh(sup, await: true) do + {:error, :timeout} -> {:error, :timeout} + {:error, :unavailable} -> {:error, :unavailable} + # A successful cycle can still leave the source empty, and a failed + # one can land on rows a seed already wrote. + _ -> if ready?(sup), do: :ok, else: {:error, :not_ready} + end + end + + defp ready?(sup), + do: Catalogue.max_checked_at(AdaptorsSupervisor.source(sup)) != nil + + @doc """ + Splits an adaptor spec into `{name, version}`, with `version` `nil` when + the spec carries none, and `{nil, nil}` for a malformed spec. """ @spec parse_spec(String.t()) :: {String.t() | nil, String.t() | nil} def parse_spec(spec) when is_binary(spec) do @@ -144,118 +245,120 @@ defmodule Lightning.Adaptors do end @doc """ - Whether a string is a well-formed adaptor spec: a package name plus an - optional `@version`, with no newlines or shell metacharacters. + Returns whether `spec` is a well-formed adaptor spec: a package name + plus an optional `@version`, with no newlines or shell metacharacters. """ @spec valid_format?(String.t()) :: boolean() def valid_format?(spec) when is_binary(spec), do: Regex.match?(PackageName.strict_format(), spec) @doc """ - Render an adaptor spec for the worker's install step, resolving - `latest` to a concrete version and preserving `local`. + Renders an adaptor spec for the worker: `latest` becomes the + catalogue's current version, `local` is kept, and a `:local` source + forces `name@local`. A `nil` spec renders as `""`. """ - @spec to_wire(String.t() | nil) :: String.t() - defdelegate to_wire(spec), to: PackageName - - @spec resolve_version(String.t(), String.t()) :: - {:ok, String.t()} | {:error, :not_found} - def resolve_version(name, requested) when requested in ["latest", "local"] do - case Repo.get_adaptor(name, Config.current_source()) do - %{latest_version: v} -> {:ok, v} - nil -> {:error, :not_found} - end - end + @spec to_wire(atom(), String.t() | nil) :: + {:ok, String.t()} + | {:error, :not_found | :timeout | :unavailable | :not_ready} + def to_wire(sup \\ @sup, spec) - def resolve_version(_name, version), do: {:ok, version} + def to_wire(_sup, nil), do: {:ok, ""} - @spec refresh_now() :: :ok | {:error, term()} - def refresh_now, do: refresh_now(@sup) + def to_wire(sup, spec) when is_binary(spec) do + source = AdaptorsSupervisor.source(sup) - @spec refresh_now(atom()) :: :ok | {:error, term()} - def refresh_now(sup), - do: Scheduler.refresh_now(AdaptorsSupervisor.global_scheduler_name(sup)) + case parse_spec(spec) do + {name, "latest"} when source != :local -> + with {:ok, %Package{latest_version: latest}} <- fetch_adaptor(sup, name) do + {:ok, PackageName.to_wire(spec, source: source, latest: latest)} + end - @spec refresh_package(String.t()) :: :ok | {:error, :not_found | term()} - def refresh_package(name) when is_binary(name), do: refresh_package(@sup, name) + _ -> + {:ok, PackageName.to_wire(spec, source: source)} + end + end + + @doc """ + Starts a catalogue refresh, or joins one already running. + + Returns `:ok` as soon as the refresh is underway. With `await: true`, + blocks until the cycle completes, bounded by `:timeout` (default + `Lightning.Adaptors.Config.first_load_timeout/0`), and returns + `{:ok, counts}` or `{:error, reason}`; see + `Lightning.Adaptors.Scheduler.await_refresh/2` for the counts. + """ + @spec refresh(atom(), keyword()) :: + :ok | {:ok, Scheduler.refresh_counts()} | {:error, term()} + def refresh(sup \\ @sup, opts \\ []) do + scheduler = AdaptorsSupervisor.global_scheduler_name(sup) + + if opts[:await] do + timeout = opts[:timeout] || Config.first_load_timeout() + Scheduler.await_refresh(scheduler, timeout) + else + Scheduler.refresh_now(scheduler) + end + catch + :exit, {:timeout, _} -> {:error, :timeout} + :exit, _reason -> {:error, :unavailable} + end + @doc """ + Refetches the adaptor named `name` from the source and persists it, + whether or not its version changed. + """ @spec refresh_package(atom(), String.t()) :: :ok | {:error, :not_found | term()} - def refresh_package(sup, name) when is_binary(name), - do: - Scheduler.refresh_package( - AdaptorsSupervisor.global_scheduler_name(sup), - name - ) - - @spec refresh_icons() :: - {:ok, %{updated: non_neg_integer(), unchanged: non_neg_integer()}} - | {:error, term()} - def refresh_icons, do: refresh_icons(@sup) + def refresh_package(sup \\ @sup, name) when is_binary(name) do + Scheduler.refresh_package( + AdaptorsSupervisor.global_scheduler_name(sup), + name + ) + catch + :exit, {:timeout, _} -> {:error, :timeout} + :exit, _reason -> {:error, :unavailable} + end + @doc """ + Refetches every adaptor's icons and updates those whose bytes changed. + Returns `{:ok, %{updated: n, unchanged: m}}`. + """ @spec refresh_icons(atom()) :: {:ok, %{updated: non_neg_integer(), unchanged: non_neg_integer()}} | {:error, term()} - def refresh_icons(sup), - do: Scheduler.refresh_icons(AdaptorsSupervisor.global_scheduler_name(sup)) - - @doc false - def icon_meta(name), do: icon_meta(@sup, name) - - @doc false - def icon_meta(sup, name), do: Store.icon_meta(sup, name) + def refresh_icons(sup \\ @sup) do + Scheduler.refresh_icons(AdaptorsSupervisor.global_scheduler_name(sup)) + catch + :exit, {:timeout, _} -> {:error, :timeout} + :exit, _reason -> {:error, :unavailable} + end @doc """ - Populate the adaptor catalogue from a JSON snapshot file, without - reaching npm. - - The file is a JSON array of adaptor records in the shape - `Lightning.Adaptors.Repo.upsert_adaptor/1` accepts — the same shape - `mix lightning.download_adaptor_registry_cache` writes. + Returns the stored extension and sha256 of each icon shape for the + adaptor named `name`, without touching disk. See + `t:Lightning.Adaptors.Store.icon_meta/0`. + """ + @spec icon_meta(atom(), String.t()) :: + {:ok, Store.icon_meta()} | {:error, :not_found} + def icon_meta(sup \\ @sup, name), do: Store.icon_meta(sup, name) - `opts`: + @doc """ + Subscribes the calling process to catalogue update broadcasts. - * `:source` - `:npm` (default) or `:local`. - * `:replace` - when `true`, deletes every existing row for that - source before seeding, so the file becomes the source's entire - contents rather than a merge. The delete and every upsert run in - one transaction, so a bad record aborts the whole seed rather - than leaving the source partially replaced. + Updates arrive as `%{event: "adaptors_updated", payload: %{names: [...]}}` + messages. """ - @spec seed_from_file(Path.t(), keyword()) :: {:ok, non_neg_integer()} - def seed_from_file(path, opts \\ []) do - source = Keyword.get(opts, :source, :npm) - replace? = Keyword.get(opts, :replace, false) - - records = - path - |> File.read!() - |> Jason.decode!() - |> Enum.map(&normalize_snapshot_record(&1, source)) - - {:ok, _} = - Lightning.Repo.transaction(fn -> - if replace?, do: Repo.delete_all_for_source(source) - Enum.each(records, &Repo.upsert_adaptor/1) - end) - - {:ok, length(records)} + @spec subscribe_to_updates(atom()) :: :ok | {:error, term()} + def subscribe_to_updates(sup \\ @sup) do + Phoenix.PubSub.subscribe( + Lightning.PubSub, + AdaptorsSupervisor.client_topic(sup) + ) end - # Top-level record keys and per-version keys map onto known schema - # fields, so they can be turned into existing atoms. `dependencies` and - # `peer_dependencies` values are left with string keys — that's the - # shape the `:map` columns already store. - defp normalize_snapshot_record(record, source) when is_map(record) do - record - |> atomize_known_keys() - |> Map.put(:source, source) - |> Map.update(:versions, [], fn versions -> - Enum.map(versions, &atomize_known_keys/1) - end) - end - - defp atomize_known_keys(map) do - Map.new(map, fn {k, v} -> {String.to_existing_atom(k), v} end) - end + @doc """ + Populates the catalogue from a JSON snapshot file. See + `Lightning.Adaptors.Seed.seed_from_file/2`. + """ + defdelegate seed_from_file(path, opts \\ []), to: Lightning.Adaptors.Seed end diff --git a/lib/lightning/adaptors/repo.ex b/lib/lightning/adaptors/catalogue.ex similarity index 78% rename from lib/lightning/adaptors/repo.ex rename to lib/lightning/adaptors/catalogue.ex index b15d6da7c8f..fceced9fe22 100644 --- a/lib/lightning/adaptors/repo.ex +++ b/lib/lightning/adaptors/catalogue.ex @@ -1,30 +1,19 @@ -defmodule Lightning.Adaptors.Repo do +defmodule Lightning.Adaptors.Catalogue do @moduledoc """ - Query and write helpers over the `adaptors` and `adaptor_versions` tables. - - Despite the name, this is **not** an `Ecto.Repo` — it is a thin - data-access module that wraps `Lightning.Repo` (the real - `Ecto.Repo`). The two schemas it targets live as siblings: - `Lightning.Adaptors.Repo.Adaptor` and - `Lightning.Adaptors.Repo.AdaptorVersion`. - - Every read helper takes the desired `:source` (`:npm | :local`) - explicitly; the module itself stays source-agnostic. Callers resolve - the active source via `Lightning.Adaptors.Config.current_source/0`. - - `upsert_adaptor/1` is the only writer the Scheduler uses. It is - idempotent, transactional, and diff-aware: `checked_at` advances on - every call, while `updated_at` only advances when the row's - meaningful fields differ from what was already in the DB. Version - rows are replaced inside the same transaction so a partial failure - cannot leave the table half-rewritten. + Reads and writes for the `adaptors` and `adaptor_versions` tables. + + Every read takes the `:source` (`:npm | :local`) explicitly. + `upsert_adaptor/1` is idempotent: `checked_at` advances on every call, + `updated_at` only when a field actually changed, and version rows are + replaced in the same transaction. """ import Ecto.Query alias Ecto.Multi - alias Lightning.Adaptors.Repo.Adaptor - alias Lightning.Adaptors.Repo.AdaptorVersion + alias Lightning.Adaptors.Catalogue.Adaptor + alias Lightning.Adaptors.Catalogue.AdaptorVersion + alias Lightning.Repo @type source :: :npm | :local @@ -55,15 +44,25 @@ defmodule Lightning.Adaptors.Repo do size_bytes dependencies peer_dependencies published_at deprecated)a + # Packages that exist on npm but should never be offered in the picker. + # Listing-only: `get_adaptor/2` still resolves them, so jobs already + # using one keep validating. + @excluded_names ~w(@openfn/language-devtools + @openfn/language-template + @openfn/language-fhir-jembi + @openfn/language-collections) + @doc """ Picker-facing lean projection for a source. Avoids the heavy JSONB columns (`schema_data`, `dependencies`, `peer_dependencies`). + + Excludes the packages listed in `@excluded_names`. """ @spec list_package_metas(source()) :: [package_meta()] def list_package_metas(source) do - Lightning.Repo.all( + Repo.all( from a in Adaptor, - where: a.source == ^source, + where: a.source == ^source and a.name not in ^@excluded_names, select: %{ name: a.name, latest_version: a.latest_version, @@ -84,7 +83,7 @@ defmodule Lightning.Adaptors.Repo do """ @spec list_adaptors(source()) :: [Adaptor.t()] def list_adaptors(source) do - Lightning.Repo.all(from a in Adaptor, where: a.source == ^source) + Repo.all(from a in Adaptor, where: a.source == ^source) end @doc """ @@ -93,7 +92,7 @@ defmodule Lightning.Adaptors.Repo do """ @spec get_adaptor(String.t(), source()) :: Adaptor.t() | nil def get_adaptor(name, source) do - Lightning.Repo.get_by(Adaptor, name: name, source: source) + Repo.get_by(Adaptor, name: name, source: source) end @doc """ @@ -101,7 +100,7 @@ defmodule Lightning.Adaptors.Repo do """ @spec list_versions(String.t(), source()) :: [AdaptorVersion.t()] def list_versions(name, source) do - Lightning.Repo.all( + Repo.all( from v in AdaptorVersion, join: a in Adaptor, on: v.adaptor_id == a.id, @@ -112,7 +111,8 @@ defmodule Lightning.Adaptors.Repo do @doc """ Idempotent, transactional, diff-aware upsert of one adaptor record - plus its version rows. The `:source` is read from the record. + plus its version rows. The source is read from the record, whose keys + may be atoms or strings (as a decoded JSON snapshot gives them). Behaviour: @@ -134,11 +134,12 @@ defmodule Lightning.Adaptors.Repo do {versions, adaptor_attrs} = record - |> Map.put(:checked_at, now) - |> Map.pop(:versions, []) + |> stringify_keys() + |> Map.put("checked_at", now) + |> Map.pop("versions", []) - name = Map.fetch!(adaptor_attrs, :name) - source = Map.fetch!(adaptor_attrs, :source) + name = Map.fetch!(adaptor_attrs, "name") + source = adaptor_attrs |> Map.fetch!("source") |> normalize_source() multi = Multi.new() @@ -160,13 +161,13 @@ defmodule Lightning.Adaptors.Repo do insert_version_rows(repo, adaptor.id, versions, now) end) - case Lightning.Repo.transaction(multi) do + case Repo.transaction(multi) do {:ok, %{adaptor: adaptor}} -> {:ok, adaptor} {:error, step, reason, _changes} -> raise ArgumentError, - "Lightning.Adaptors.Repo.upsert_adaptor/1 failed at #{inspect(step)}: " <> + "Lightning.Adaptors.Catalogue.upsert_adaptor/1 failed at #{inspect(step)}: " <> inspect(reason) end end @@ -176,7 +177,7 @@ defmodule Lightning.Adaptors.Repo do """ @spec delete_all_for_source(source()) :: :ok def delete_all_for_source(source) do - Lightning.Repo.delete_all(from a in Adaptor, where: a.source == ^source) + Repo.delete_all(from a in Adaptor, where: a.source == ^source) :ok end @@ -191,7 +192,7 @@ defmodule Lightning.Adaptors.Repo do def touch_checked_at(name, source) do now = DateTime.utc_now() - Lightning.Repo.update_all( + Repo.update_all( from(a in Adaptor, where: a.name == ^name and a.source == ^source), set: [checked_at: now] ) @@ -212,7 +213,7 @@ defmodule Lightning.Adaptors.Repo do } ] def list_missing_icons(source) do - Lightning.Repo.all( + Repo.all( from a in Adaptor, where: a.source == ^source and @@ -252,7 +253,7 @@ defmodule Lightning.Adaptors.Repo do |> Map.put(:updated_at, DateTime.utc_now()) |> Enum.into([]) - Lightning.Repo.update_all( + Repo.update_all( from(a in Adaptor, where: a.name == ^name and a.source == ^source), set: allowed ) @@ -264,7 +265,7 @@ defmodule Lightning.Adaptors.Repo do """ @spec max_checked_at(source()) :: DateTime.t() | nil def max_checked_at(source) do - Lightning.Repo.one( + Repo.one( from a in Adaptor, where: a.source == ^source, select: max(a.checked_at) @@ -274,13 +275,15 @@ defmodule Lightning.Adaptors.Repo do @doc """ Full catalogue projection for a source: every adaptor's `name`, `latest_version`, `repository`, icon fields, and full version list. + + Excludes the packages listed in `@excluded_names`. """ @spec catalogue(source()) :: [catalogue_entry()] def catalogue(source) do adaptors = - Lightning.Repo.all( + Repo.all( from a in Adaptor, - where: a.source == ^source, + where: a.source == ^source and a.name not in ^@excluded_names, order_by: [asc: a.name], select: %{ name: a.name, @@ -294,11 +297,11 @@ defmodule Lightning.Adaptors.Repo do ) versions_by_name = - Lightning.Repo.all( + Repo.all( from v in AdaptorVersion, join: a in Adaptor, on: v.adaptor_id == a.id, - where: a.source == ^source, + where: a.source == ^source and a.name not in ^@excluded_names, order_by: [asc: v.inserted_at, asc: v.version], select: {a.name, v.version} ) @@ -321,7 +324,7 @@ defmodule Lightning.Adaptors.Repo do """ @spec catalogue_stamp(source()) :: {DateTime.t() | nil, non_neg_integer()} def catalogue_stamp(source) do - Lightning.Repo.one( + Repo.one( from a in Adaptor, left_join: v in AdaptorVersion, on: v.adaptor_id == a.id, @@ -343,9 +346,7 @@ defmodule Lightning.Adaptors.Repo do defp upsert_adaptor_row(repo, %Adaptor{} = existing, attrs, now) do changeset = Adaptor.changeset(existing, attrs) - # `Ecto.Changeset.cast/3` only records a change when the cast value - # differs from the underlying struct, so the set of "real" changes - # is `:changes` minus the `:checked_at` tick we apply on every call. + # `checked_at` changes on every call, so it is excluded from the diff. meaningful_changes? = changeset.changes |> Map.delete(:checked_at) @@ -380,7 +381,11 @@ defmodule Lightning.Adaptors.Repo do defp build_version_rows(adaptor_id, records, now) do records |> Enum.reduce_while({:ok, []}, fn record, {:ok, acc} -> - attrs = Map.put(record, :adaptor_id, adaptor_id) + attrs = + record + |> stringify_keys() + |> Map.put("adaptor_id", adaptor_id) + changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) if changeset.valid? do @@ -395,6 +400,22 @@ defmodule Lightning.Adaptors.Repo do end end + # `Ecto.Changeset.cast/3` raises on a map mixing atom and string keys, so + # every map handed to a changeset here is flattened to string keys first — + # that is what a JSON snapshot gives us, and what atom-keyed callers + # convert cleanly into. + defp stringify_keys(map) do + Map.new(map, fn {k, v} -> {to_string(k), v} end) + end + + # `source` is read outside the changeset (for the existing-row lookup), + # so it needs its own cast: `Ecto.Enum` fields accept a string via + # `Changeset.cast/3`, but not via `Repo.get_by/3`'s query parameters. + defp normalize_source(source) when is_atom(source), do: source + + defp normalize_source(source) when is_binary(source), + do: String.to_existing_atom(source) + defp version_row_from_changeset(changeset, now) do changeset |> Ecto.Changeset.apply_changes() diff --git a/lib/lightning/adaptors/repo_adaptor.ex b/lib/lightning/adaptors/catalogue_adaptor.ex similarity index 72% rename from lib/lightning/adaptors/repo_adaptor.ex rename to lib/lightning/adaptors/catalogue_adaptor.ex index 1b66bff0f67..0115bd3c677 100644 --- a/lib/lightning/adaptors/repo_adaptor.ex +++ b/lib/lightning/adaptors/catalogue_adaptor.ex @@ -1,14 +1,8 @@ -defmodule Lightning.Adaptors.Repo.Adaptor do +defmodule Lightning.Adaptors.Catalogue.Adaptor do @moduledoc """ - Ecto schema for one row of the `adaptors` table — the per-package - metadata projection used by the picker and Scheduler. - - Source-tagged via `:source` (`:npm | :local`) so the same package - name can coexist across sources; the unique index is - `[:name, :source]`. - - Mirrors `Lightning.Adaptors.Strategy.adaptor_record` minus `:versions`, - which lives on `Lightning.Adaptors.Repo.AdaptorVersion`. + Ecto schema for one row of the `adaptors` table, unique on + `[:name, :source]`. Versions live on + `Lightning.Adaptors.Catalogue.AdaptorVersion`. """ use Ecto.Schema @@ -17,15 +11,9 @@ defmodule Lightning.Adaptors.Repo.Adaptor do defmodule JSONBinary do @moduledoc """ - Ecto type backing `schema_data` with a text column while preserving - JSON field order on read. - - Storage is a JSON binary in a `text` column. Inputs may be either a - binary or a map — maps are encoded with `Jason.encode!/1` at the - dumper to keep `Lightning.Factories.adaptor/2` and other direct - struct inserts compatible without forcing every caller to encode - up-front. Loads always return a binary so credential-form rendering - can re-engage `Jason.decode!(_, objects: :ordered_objects)`. + Ecto type for `schema_data`: a JSON binary in a `text` column. + Accepts a binary or a map on write and always loads a binary, so the + reader can decode with ordered objects. """ use Ecto.Type @@ -86,7 +74,7 @@ defmodule Lightning.Adaptors.Repo.Adaptor do field :license, :string field :latest_version, :string field :deprecated, :boolean, default: false - field :schema_data, Lightning.Adaptors.Repo.Adaptor.JSONBinary + field :schema_data, Lightning.Adaptors.Catalogue.Adaptor.JSONBinary field :schema_sha256, :string field :icon_square_ext, :string field :icon_rectangle_ext, :string @@ -107,11 +95,7 @@ defmodule Lightning.Adaptors.Repo.Adaptor do icon_square_etag icon_rectangle_etag)a @doc """ - Build a changeset for upserting a single adaptor row. - - This is the single clause used by every write path on - `Lightning.Adaptors.Repo` — there is no separate update path because - the writer always rewrites the full row. + Builds the changeset for inserting or fully rewriting an adaptor row. """ @spec changeset(t(), map()) :: Ecto.Changeset.t() def changeset(struct, attrs) do @@ -119,6 +103,7 @@ defmodule Lightning.Adaptors.Repo.Adaptor do |> cast(attrs, @required ++ @optional) |> validate_required(@required) |> validate_length(:name, max: 214) + |> validate_format(:name, Lightning.Adaptors.PackageName.name_format()) |> validate_inclusion(:icon_square_ext, ~w(png svg)) |> validate_inclusion(:icon_rectangle_ext, ~w(png svg)) |> validate_icon_sha256_pair(:icon_square) @@ -126,9 +111,6 @@ defmodule Lightning.Adaptors.Repo.Adaptor do |> unique_constraint([:name, :source]) end - # Enforces the §6.4 invariant: a non-nil `icon__ext` requires a - # non-nil `icon__sha256`, and vice versa. Either both fields - # are set or both are nil — half-populated pairs fail the changeset. @spec validate_icon_sha256_pair( Ecto.Changeset.t(), :icon_square | :icon_rectangle diff --git a/lib/lightning/adaptors/repo_adaptor_version.ex b/lib/lightning/adaptors/catalogue_adaptor_version.ex similarity index 70% rename from lib/lightning/adaptors/repo_adaptor_version.ex rename to lib/lightning/adaptors/catalogue_adaptor_version.ex index d0e05df3001..248f1bbd461 100644 --- a/lib/lightning/adaptors/repo_adaptor_version.ex +++ b/lib/lightning/adaptors/catalogue_adaptor_version.ex @@ -1,19 +1,14 @@ -defmodule Lightning.Adaptors.Repo.AdaptorVersion do +defmodule Lightning.Adaptors.Catalogue.AdaptorVersion do @moduledoc """ - Ecto schema for one row of the `adaptor_versions` table — per-version - metadata for an adaptor package (`integrity`, `tarball_url`, - `size_bytes`, `dependencies`, `peer_dependencies`, `published_at`, - `deprecated`). - - Belongs to `Lightning.Adaptors.Repo.Adaptor` and cascade-deletes with - its parent. Mirrors `Lightning.Adaptors.Strategy.version_record`. + Ecto schema for one row of the `adaptor_versions` table. Belongs to + `Lightning.Adaptors.Catalogue.Adaptor` and is deleted with it. """ use Ecto.Schema import Ecto.Changeset - alias Lightning.Adaptors.Repo.Adaptor + alias Lightning.Adaptors.Catalogue.Adaptor @type t :: %__MODULE__{ id: Ecto.UUID.t() | nil, @@ -55,11 +50,7 @@ defmodule Lightning.Adaptors.Repo.AdaptorVersion do published_at deprecated)a @doc """ - Build a changeset for inserting an `adaptor_versions` row. - - `Lightning.Adaptors.Repo.upsert_adaptor/1` replaces version rows with - a delete-then-insert inside a transaction, so there is no separate - update path. + Builds the changeset for inserting a version row. """ @spec changeset(t(), map()) :: Ecto.Changeset.t() def changeset(struct, attrs) do diff --git a/lib/lightning/adaptors/config.ex b/lib/lightning/adaptors/config.ex index 8f45fd8cfcf..5c7101647b7 100644 --- a/lib/lightning/adaptors/config.ex +++ b/lib/lightning/adaptors/config.ex @@ -1,23 +1,10 @@ defmodule Lightning.Adaptors.Config do @moduledoc """ - Stateless runtime configuration for the `Lightning.Adaptors.*` subsystem. + Runtime configuration for the adaptors subsystem, read from + `config :lightning, Lightning.Adaptors` on every call. - Every helper is a thin wrapper around `Application.get_env/3`. It is the - single runtime source of truth for which strategy is active, how often - the scheduler ticks, the per-call cache fetch deadline, the icon cache - root, and per-strategy opt blocks. - - ## Application key layout - - Two-tier: - - * `:lightning, Lightning.Adaptors` — subsystem-wide knobs - (`:strategy`, `:refresh_interval`, `:cache_timeout_ms`, `:icon_path`). - * `:lightning, ` — each strategy owns its own - Application key for its own knobs; read via `strategy_opts/1`. - - No GenServer, no ETS, no `:persistent_term` — every call is a fresh - `Application.get_env/3`. + Strategy-specific options live under the strategy module's own key; + see `strategy_opts/1`. """ @parent_key Lightning.Adaptors @@ -26,6 +13,7 @@ defmodule Lightning.Adaptors.Config do @default_refresh_interval :timer.hours(1) @default_cache_timeout_ms 15_000 @default_icon_path {:tmp, "lightning/adaptor_icons"} + @default_first_load_timeout :timer.seconds(60) @doc """ The active strategy module. Defaults to `Lightning.Adaptors.NPM`. @@ -36,14 +24,27 @@ defmodule Lightning.Adaptors.Config do end @doc """ - Atom mapping of `strategy/0`: `:local` for `Lightning.Adaptors.Local`, - `:npm` for any other strategy module. + The catalogue `source` a strategy writes under. + + `Lightning.Adaptors.Local` and `Lightning.Adaptors.NPM` are mapped + here; any other strategy must declare a `:source` of `:npm` or + `:local` under its own application key, or this raises rather than + guessing at `:npm`. """ - @spec current_source() :: :local | :npm - def current_source do - case strategy() do - Lightning.Adaptors.Local -> :local - _other -> :npm + @spec source_for(module()) :: :local | :npm + def source_for(Lightning.Adaptors.Local), do: :local + def source_for(Lightning.Adaptors.NPM), do: :npm + + def source_for(strategy) when is_atom(strategy) do + case Keyword.fetch(strategy_opts(strategy), :source) do + {:ok, source} when source in [:npm, :local] -> + source + + _ -> + raise ArgumentError, + "strategy #{inspect(strategy)} has no catalogue source; " <> + "map it in Lightning.Adaptors.Config.source_for/1 or set " <> + "`config :lightning, #{inspect(strategy)}, source: :npm`" end end @@ -56,7 +57,8 @@ defmodule Lightning.Adaptors.Config do end @doc """ - Per-`Cachex.fetch` courier deadline in milliseconds. Defaults to 15s. + How long a read waits for a cache fill, in milliseconds. Defaults to + 15 seconds. """ @spec cache_timeout_ms() :: non_neg_integer() def cache_timeout_ms do @@ -64,15 +66,8 @@ defmodule Lightning.Adaptors.Config do end @doc """ - Resolved filesystem path for the icon cache. - - Accepts either: - - * `{:tmp, suffix}` — resolved against `System.tmp_dir!/0` at call - time so the default does not bake a container-specific tmp path - into a compiled release. - * a plain binary path — returned verbatim. - + Filesystem path of the icon cache. A `{:tmp, suffix}` value is joined + to `System.tmp_dir!/0` at call time; a binary is returned as is. Defaults to `{:tmp, "lightning/adaptor_icons"}`. """ @spec icon_path() :: Path.t() @@ -84,9 +79,18 @@ defmodule Lightning.Adaptors.Config do end @doc """ - Per-strategy keyword opts. Parameterised on the strategy module — each - strategy is its own Application key, not nested under the parent. - Returns `[]` when the strategy's Application key is unset. + Bound, in milliseconds, on how long `Lightning.Adaptors.ensure_loaded/1` + and `Lightning.Adaptors.fetch_adaptor/2` block waiting for the + catalogue's first load. Defaults to 60 seconds. + """ + @spec first_load_timeout() :: non_neg_integer() + def first_load_timeout do + get(:first_load_timeout, @default_first_load_timeout) + end + + @doc """ + Options configured under the strategy module's own application key, or + `[]` when unset. """ @spec strategy_opts(module()) :: keyword() def strategy_opts(strategy_mod) when is_atom(strategy_mod) do diff --git a/lib/lightning/adaptors/invalidator.ex b/lib/lightning/adaptors/invalidator.ex index a548c23ee13..0b3b1f5eba0 100644 --- a/lib/lightning/adaptors/invalidator.ex +++ b/lib/lightning/adaptors/invalidator.ex @@ -4,9 +4,11 @@ defmodule Lightning.Adaptors.Invalidator do local Cachex entries, keeping each node coherent with Postgres. Subscribes to `opts[:source_topic]` on `Lightning.PubSub` at init. - On `{:changed, name, source}`, deletes the four per-adaptor cache keys - written by `Lightning.Adaptors.Store`. No source filtering on the hot - path — a broadcast for a source that isn't active on this node is a + On `{:changed, name, source}`, deletes the five cache keys written by + `Lightning.Adaptors.Store`: the three keyed by name (`:schema`, + `:versions`, `:icon_meta`) plus the two source-wide ones (`:packages`, + `:catalogue`), which any change invalidates. No source filtering on the + hot path — a broadcast for a source that isn't active on this node is a no-op because those keys simply don't exist in Cachex. """ @@ -40,6 +42,7 @@ defmodule Lightning.Adaptors.Invalidator do Cachex.del(state.cache, {:versions, name, source}) Cachex.del(state.cache, {:icon_meta, name, source}) Cachex.del(state.cache, {:packages, source}) + Cachex.del(state.cache, {:catalogue, source}) {:noreply, state} end end diff --git a/lib/lightning/adaptors/package_name.ex b/lib/lightning/adaptors/package_name.ex index 746bdc2091e..a55e0533442 100644 --- a/lib/lightning/adaptors/package_name.ex +++ b/lib/lightning/adaptors/package_name.ex @@ -1,39 +1,30 @@ defmodule Lightning.Adaptors.PackageName do @moduledoc """ - NPM-style package-name parsing and worker wire-shape recomposition for - the `Lightning.Adaptors.*` subsystem. - - This module is the single source of truth for adaptor package name - parsing and wire recomposition, read through the `Lightning.Adaptors` - facade. - - `parse/1` splits `"name@version"` strings using the same strict, - anchored format `strict_format/0` validates against, so a spec that - reaches `to_wire/1` after passing changeset validation is guaranteed to - parse to the same name — never a truncated or re-derived one. `to_wire/1` - resolves the `latest` literal through `Lightning.Adaptors.resolve_version/2`, - preserves `"name@local"` as a literal regardless of source, and emits - `"name@local"` under a `:local` strategy source. + Parses `name@version` adaptor specs and renders them for the worker. """ - alias Lightning.Adaptors - alias Lightning.Adaptors.Config - - # Anchored with \A…\z (NOT ^…$, since $ matches before a trailing \n). - # Accepts scoped (@scope/name) and unscoped names with an optional - # @version (semver, prerelease, or the tokens `latest` / `local`); - # excludes newlines and shell metacharacters. + # `\A…\z` rather than `^…$`: `$` matches before a trailing newline. @strict_format ~r{\A(@?[\w.-]+(?:/[\w.-]+)?)(?:@([\w.-]+))?\z} + @name_format ~r{\A@?[\w.-]+(?:/[\w.-]+)?\z} + @doc """ - The strict, anchored package-name format: name plus optional `@version`, - rejecting embedded newlines and shell metacharacters. Read through - `Lightning.Adaptors.valid_format?/1` and - `Lightning.Adaptors.parse_spec/1`. + Returns the spec format: a package name plus optional `@version`, with + no newlines or shell metacharacters. """ @spec strict_format() :: Regex.t() def strict_format, do: @strict_format + @doc """ + Returns the bare package-name format, with no `@version` suffix. + """ + @spec name_format() :: Regex.t() + def name_format, do: @name_format + + @doc """ + Splits a spec into `{name, version}`; `{nil, nil}` for `nil` or a + malformed spec. + """ @spec parse(nil) :: {nil, nil} def parse(nil), do: {nil, nil} @@ -46,34 +37,27 @@ defmodule Lightning.Adaptors.PackageName do end end - @spec to_wire(String.t() | nil) :: String.t() - def to_wire(adaptor) do - case parse(adaptor) do - {nil, nil} -> "" - {name, version} -> recompose(name, version, adaptor) - end - end + @doc """ + Renders a spec for the worker. - defp recompose(name, "local", _original), do: "#{name}@local" + `opts[:source]` of `:local` forces `name@local`. `opts[:latest]` is the + concrete version for a `latest` spec, and is required for one under + any other source. A `name@local` spec is always kept as is. + """ + @spec to_wire(String.t() | nil, keyword()) :: String.t() + def to_wire(adaptor, opts \\ []) do + case parse(adaptor) do + {nil, nil} -> + "" - defp recompose(name, version, original) do - case Config.current_source() do - :local -> + {name, "local"} -> "#{name}@local" - _ -> - case version do - "latest" -> - case Adaptors.resolve_version(name, "latest") do - {:ok, resolved} -> "#{name}@#{resolved}" - {:error, _} -> "#{name}@latest" - end - - nil -> - original - - _concrete -> - original + {name, version} -> + cond do + opts[:source] == :local -> "#{name}@local" + version == "latest" -> "#{name}@#{Keyword.fetch!(opts, :latest)}" + true -> adaptor end end end diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 9fe26e53f41..44e0477ea44 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -1,45 +1,26 @@ defmodule Lightning.Adaptors.Scheduler do @moduledoc """ - Cluster-singleton GenServer that periodically refreshes the active - source's ledger via the configured strategy, persists through - `Lightning.Adaptors.Repo`, and broadcasts `{:changed, name, source}`. - - Wrapped by `HighlanderPG` so only one node in the cluster runs the - Scheduler at a time. The inner GenServer registers under - `{:global, Lightning.Adaptors.Supervisor.global_scheduler_name(name)}`, - so callers on any node reach the leader transparently via Erlang - distribution. Peer nodes react to refreshes via - `Lightning.Adaptors.Invalidator` and - `Lightning.Adaptors.ChannelBroadcaster`. - - Smart-init timing: the first tick is scheduled at - `max(0, last_checked_at + interval - now)` to avoid double-refreshing - shortly after a deploy. An empty table or an overdue schedule fires - immediately (`delay = 0`). Interval `0` disables scheduling entirely. - - ## Two-pipeline refresh - - A tick runs two parallel pipelines under the per-instance - `Task.Supervisor`: - - * **Pipeline A** — `strategy.fetch_icons/1` for every adaptor. - * **Pipeline B** — `strategy.list_adaptors/0` followed by a bounded - per-adaptor fan-out (`async_stream_nolink`) calling - `strategy.fetch_adaptor/1` only for names whose `latest_version` - changed since the last tick. - - Once both pipelines complete the join step merges the icons map into - each fetched record, writes the icon bytes to disk via - `Lightning.Adaptors.IconCache.write!/5`, and upserts each adaptor in - one go. `refresh_package/2` deliberately bypasses the icon pipeline — - on-demand single-package refreshes do not refetch icons. + Cluster-singleton GenServer that refreshes the catalogue from the + strategy, on a timer and on demand, persisting through + `Lightning.Adaptors.Catalogue` and broadcasting `{:changed, name, source}` + on the source topic. + + The first tick is due one `refresh_interval` after the catalogue's most + recent check, so a restart does not refetch straight away; an empty + catalogue ticks at once. An interval of `0` disables the timer and + leaves only on-demand refreshes. + + A tick lists the source, fetches only the adaptors whose + `latest_version` changed, fetches icons in parallel, and upserts each + changed adaptor with its icons. `refresh_package/2` refetches one + adaptor without icons. """ use GenServer + alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Config alias Lightning.Adaptors.IconCache - alias Lightning.Adaptors.Repo, as: AdaptorsRepo alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor require Logger @@ -48,10 +29,8 @@ defmodule Lightning.Adaptors.Scheduler do @icons_task_timeout :timer.seconds(60) @doc """ - Start the Scheduler for the given supervisor instance. - - Required opts: `:name`, `:sup`, `:lock_key`, `:cache`, `:tasks`, - `:source_topic`. + Starts the Scheduler. Required opts: `:name`, `:sup`, `:lock_key`, + `:cache`, `:tasks`, `:source_topic`. """ @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts) do @@ -65,9 +44,7 @@ defmodule Lightning.Adaptors.Scheduler do end @doc """ - Trigger an immediate refresh tick. - - Routes via `:global` to the leader-held GenServer. + Starts a refresh tick, or lets one already in flight continue. """ @spec refresh_now(GenServer.server()) :: :ok | {:error, term()} def refresh_now(scheduler_name) do @@ -75,10 +52,8 @@ defmodule Lightning.Adaptors.Scheduler do end @doc """ - Force a single-adaptor refresh, bypassing the diff. 30-second timeout. - - Returns `{:error, :not_found}` or `{:error, term()}` from a failed - strategy fetch. + Refetches and persists one adaptor whether or not its version changed. + Waits up to 30 seconds. """ @spec refresh_package(GenServer.server(), String.t()) :: :ok | {:error, :not_found | term()} @@ -86,13 +61,40 @@ defmodule Lightning.Adaptors.Scheduler do GenServer.call(scheduler_name, {:refresh_package, name}, 30_000) end + @typedoc """ + One refresh cycle's tallies: adaptors the upstream listing returned, + how many of those had a changed `latest_version`, how many were + fetched and persisted, and how many per-adaptor fetches failed. + """ + @type refresh_counts :: %{ + listed: non_neg_integer(), + changed: non_neg_integer(), + fetched: non_neg_integer(), + errors: non_neg_integer() + } + + @doc """ + Starts a refresh cycle, or joins the one in flight, and waits for it to + complete. + + Returns `{:ok, counts}` when the listing succeeded (per-adaptor fetch + failures are counted in `counts.errors`), `{:error, reason}` when it + failed, or `{:error, {:refresh_failed, reason}}` when the cycle crashed. + """ + @spec await_refresh(GenServer.server(), timeout()) :: + {:ok, refresh_counts()} + | {:error, {:refresh_failed, term()} | term()} + def await_refresh(scheduler_name, timeout) do + GenServer.call(scheduler_name, :await_refresh, timeout) + end + @doc """ - Refresh icons only, against every source-scoped adaptor row. + Refetches every adaptor's icons and updates the rows whose icon bytes + changed, leaving other fields untouched. - Runs `strategy.fetch_icons/1` and re-applies any shape whose `sha256` - differs from what is on the row. Adaptor metadata and version rows - are not touched. Returns `{:ok, %{updated: n, unchanged: m}}` on - success or `{:error, reason}` if the bulk fetch fails. + Returns `{:ok, %{updated: n, unchanged: m}}`, `{:error, reason}` if the + fetch fails, or `{:error, {:refresh_failed, reason}}` if the task + crashed. """ @spec refresh_icons(GenServer.server()) :: {:ok, %{updated: non_neg_integer(), unchanged: non_neg_integer()}} @@ -113,7 +115,7 @@ defmodule Lightning.Adaptors.Scheduler do if interval_ms > 0 do delay = - time_until_next_ms(AdaptorsRepo.max_checked_at(source), interval_ms) + time_until_next_ms(Catalogue.max_checked_at(source), interval_ms) Process.send_after(self(), :tick, delay) @@ -131,7 +133,11 @@ defmodule Lightning.Adaptors.Scheduler do interval_ms: interval_ms, source_topic: source_topic, cache: cache, - tasks: tasks + tasks: tasks, + refresh: nil, + waiters: [], + package_refreshes: %{}, + icon_refreshes: %{} }} end @@ -141,7 +147,91 @@ defmodule Lightning.Adaptors.Scheduler do Process.send_after(self(), :tick, state.interval_ms) end - Task.Supervisor.start_child(state.tasks, fn -> do_refresh(state) end) + if state.refresh do + Logger.debug( + "Adaptors[#{state.source}]: tick coalesced into in-flight refresh" + ) + + {:noreply, state} + else + {:noreply, start_refresh(state)} + end + end + + @impl true + def handle_info({ref, result}, %{refresh: %Task{ref: ref}} = state) do + Process.demonitor(ref, [:flush]) + + Logger.info( + "Adaptors[#{state.source}]: refresh complete, replying to " <> + "#{length(state.waiters)} waiter(s)" + ) + + Enum.each(state.waiters, &GenServer.reply(&1, result)) + + {:noreply, %{state | refresh: nil, waiters: []}} + end + + def handle_info( + {:DOWN, ref, :process, _pid, reason}, + %{refresh: %Task{ref: ref}} = state + ) do + Logger.warning( + "Adaptors[#{state.source}]: refresh task crashed: #{inspect(reason)} — " <> + "replying error to #{length(state.waiters)} waiter(s)" + ) + + Enum.each( + state.waiters, + &GenServer.reply(&1, {:error, {:refresh_failed, reason}}) + ) + + {:noreply, %{state | refresh: nil, waiters: []}} + end + + def handle_info({ref, result}, state) + when is_map_key(state.package_refreshes, ref) do + Process.demonitor(ref, [:flush]) + {from, package_refreshes} = Map.pop!(state.package_refreshes, ref) + GenServer.reply(from, result) + {:noreply, %{state | package_refreshes: package_refreshes}} + end + + def handle_info({:DOWN, ref, :process, _pid, reason}, state) + when is_map_key(state.package_refreshes, ref) do + Logger.warning( + "Adaptors[#{state.source}]: refresh_package task crashed: #{inspect(reason)}" + ) + + {from, package_refreshes} = Map.pop!(state.package_refreshes, ref) + GenServer.reply(from, {:error, {:refresh_failed, reason}}) + {:noreply, %{state | package_refreshes: package_refreshes}} + end + + def handle_info({ref, result}, state) + when is_map_key(state.icon_refreshes, ref) do + Process.demonitor(ref, [:flush]) + {from, icon_refreshes} = Map.pop!(state.icon_refreshes, ref) + GenServer.reply(from, result) + {:noreply, %{state | icon_refreshes: icon_refreshes}} + end + + def handle_info({:DOWN, ref, :process, _pid, reason}, state) + when is_map_key(state.icon_refreshes, ref) do + Logger.warning( + "Adaptors[#{state.source}]: refresh_icons task crashed: #{inspect(reason)}" + ) + + {from, icon_refreshes} = Map.pop!(state.icon_refreshes, ref) + GenServer.reply(from, {:error, {:refresh_failed, reason}}) + {:noreply, %{state | icon_refreshes: icon_refreshes}} + end + + # A late task result must not crash the singleton. + def handle_info(msg, state) do + Logger.warning( + "Adaptors[#{state.source}]: scheduler ignoring unexpected message: #{inspect(msg)}" + ) {:noreply, state} end @@ -153,19 +243,43 @@ defmodule Lightning.Adaptors.Scheduler do {:reply, :ok, state} end - def handle_call({:refresh_package, name}, _from, state) do + def handle_call(:await_refresh, from, state) do + Logger.debug( + "Adaptors[#{state.source}]: await_refresh attached (#{length(state.waiters) + 1} waiters)" + ) + + state = %{state | waiters: [from | state.waiters]} + state = if state.refresh, do: state, else: start_refresh(state) + {:noreply, state} + end + + def handle_call({:refresh_package, name}, from, state) do Logger.info("Adaptors[#{state.source}]: refresh_package(#{name}) requested") strategy = AdaptorsSupervisor.strategy(state.sup) - result = force_refresh_one(strategy, name, state) - {:reply, result, state} + + task = + Task.Supervisor.async_nolink(state.tasks, fn -> + force_refresh_one(strategy, name, state) + end) + + {:noreply, put_in(state.package_refreshes[task.ref], from)} end - def handle_call(:refresh_icons, _from, state) do + def handle_call(:refresh_icons, from, state) do Logger.info("Adaptors[#{state.source}]: refresh_icons requested") strategy = AdaptorsSupervisor.strategy(state.sup) - existing = AdaptorsRepo.list_adaptors(state.source) + task = + Task.Supervisor.async_nolink(state.tasks, fn -> + do_refresh_icons(strategy, state) + end) + + {:noreply, put_in(state.icon_refreshes[task.ref], from)} + end + + defp do_refresh_icons(strategy, state) do + existing = Catalogue.list_adaptors(state.source) prior_etags = prior_etags_from_rows(existing) case strategy.fetch_icons(prior_etags: prior_etags) do @@ -178,24 +292,29 @@ defmodule Lightning.Adaptors.Scheduler do "updated=#{result.updated} unchanged=#{result.unchanged}" ) - {:reply, {:ok, result}, state} + {:ok, result} {:error, reason} -> Logger.warning( "Adaptors[#{state.source}]: refresh_icons strategy fetch failed: #{inspect(reason)}" ) - {:reply, {:error, reason}, state} + {:error, reason} end end + defp start_refresh(state) do + task = Task.Supervisor.async_nolink(state.tasks, fn -> do_refresh(state) end) + %{state | refresh: task} + end + defp do_refresh(state) do started_at = System.monotonic_time(:millisecond) strategy = AdaptorsSupervisor.strategy(state.sup) # Single DB round-trip serves both the icons-task input (prior etags) # and the version diff used below to decide which adaptors to fetch. - existing_rows = AdaptorsRepo.list_adaptors(state.source) + existing_rows = Catalogue.list_adaptors(state.source) prior_etags = prior_etags_from_rows(existing_rows) existing_by_name = @@ -246,6 +365,9 @@ defmodule Lightning.Adaptors.Scheduler do "errors=#{errors} duration=#{duration_ms}ms" ) + {:ok, + %{listed: listed, changed: changed, fetched: persisted, errors: errors}} + {:error, reason} -> Logger.warning("Scheduler: list_adaptors failed: #{inspect(reason)}") _ = await_icons(icons_task) @@ -256,7 +378,7 @@ defmodule Lightning.Adaptors.Scheduler do "touched=0 fetched=0 icons=0 errors=1 duration=#{duration_ms}ms" ) - :ok + {:error, reason} end end @@ -267,7 +389,7 @@ defmodule Lightning.Adaptors.Scheduler do state ) do if Map.get(existing_by_name, name) == version do - AdaptorsRepo.touch_checked_at(name, state.source) + Catalogue.touch_checked_at(name, state.source) :touched else case strategy.fetch_adaptor(name) do @@ -383,7 +505,7 @@ defmodule Lightning.Adaptors.Scheduler do defp heal_missing_icons(icons, state) do state.source - |> AdaptorsRepo.list_missing_icons() + |> Catalogue.list_missing_icons() |> Enum.reduce(0, fn row, acc -> package_icons = Map.get(icons, row.name, %{}) @@ -420,7 +542,7 @@ defmodule Lightning.Adaptors.Scheduler do end) if map_size(changes) > 0 do - {1, _} = AdaptorsRepo.update_icons(row.name, state.source, changes) + {1, _} = Catalogue.update_icons(row.name, state.source, changes) Phoenix.PubSub.broadcast( Lightning.PubSub, @@ -522,7 +644,7 @@ defmodule Lightning.Adaptors.Scheduler do # log their own success line outside it, so a mistake in that line crashes # rather than being reported back as a failed upsert. defp upsert_and_broadcast(record, name, state) do - {:ok, _} = AdaptorsRepo.upsert_adaptor(record) + {:ok, _} = Catalogue.upsert_adaptor(record) Phoenix.PubSub.broadcast( Lightning.PubSub, diff --git a/lib/lightning/adaptors/seed.ex b/lib/lightning/adaptors/seed.ex new file mode 100644 index 00000000000..a102968d0f2 --- /dev/null +++ b/lib/lightning/adaptors/seed.ex @@ -0,0 +1,82 @@ +defmodule Lightning.Adaptors.Seed do + @moduledoc """ + Populates the adaptor catalogue from a JSON snapshot file. + """ + + alias Lightning.Adaptors.Catalogue + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + @doc """ + Populates the catalogue from a JSON snapshot file and returns the + number of records written. + + The file is a JSON array of adaptor records in the shape + `Lightning.Adaptors.Catalogue.upsert_adaptor/1` accepts, as written by + `mix lightning.adaptors.snapshot` or `mix lightning.adaptors.dump`. + + Once the transaction commits, broadcasts `{:changed, name, source}` for + every name the seed touched, so each node's + `Lightning.Adaptors.Invalidator` drops its now-stale cache entries. In + `replace: true` mode that includes names the wipe removed. + + Options: + + * `:source` - `:npm` (default) or `:local` + * `:replace` - when `true`, deletes every existing row for the source + first, in the same transaction as the upserts + * `:sup` - supervisor instance whose topic the broadcasts go to, + defaulting to `Lightning.Adaptors` + """ + @spec seed_from_file(Path.t(), keyword()) :: {:ok, non_neg_integer()} + def seed_from_file(path, opts \\ []) do + source = Keyword.get(opts, :source, :npm) + replace? = Keyword.get(opts, :replace, false) + sup = Keyword.get(opts, :sup, Lightning.Adaptors) + + records = + path + |> File.read!() + |> Jason.decode!() + |> Enum.map(&normalize_snapshot_record(&1, source)) + + replaced_names = + if replace?, + do: Enum.map(Catalogue.list_adaptors(source), & &1.name), + else: [] + + {:ok, _} = + Lightning.Repo.transaction(fn -> + if replace?, do: Catalogue.delete_all_for_source(source) + Enum.each(records, &Catalogue.upsert_adaptor/1) + end) + + broadcast_changed( + sup, + source, + Enum.uniq(Enum.map(records, & &1["name"]) ++ replaced_names) + ) + + {:ok, length(records)} + end + + # `Lightning.Release.seed_adaptors/2` seeds through + # `Ecto.Migrator.with_repo/2`, which starts the repo without the rest of + # the app — there is no PubSub to broadcast on, and no cache to evict. + defp broadcast_changed(sup, source, names) do + if Process.whereis(Lightning.PubSub) do + topic = AdaptorsSupervisor.source_topic(sup) + + Enum.each(names, fn name -> + Phoenix.PubSub.broadcast( + Lightning.PubSub, + topic, + {:changed, name, source} + ) + end) + end + end + + defp normalize_snapshot_record(record, source) when is_map(record) do + Map.put(record, "source", source) + end +end diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index c72dbc42c4c..37fa65fb8f7 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -1,48 +1,22 @@ defmodule Lightning.Adaptors.Store do @moduledoc """ - Stateless read facade over `Cachex`, `Lightning.Adaptors.Repo`, and the - active `Lightning.Adaptors.Strategy`. - - Every public read helper wraps a `Cachex.fetch/4` whose fallback first - consults the local Postgres projection (`Lightning.Adaptors.Repo`) and - only invokes the Strategy as a last resort. Cachex's courier supplies - blocking semantics and per-key coalescing of concurrent first-callers - for free — there is no GenServer mailbox in front of the reads. - - ## Source tagging - - Each cache key carries the active `:source` (`:npm | :local`) read via - `Lightning.Adaptors.Supervisor.source/1`, so the same package name can - coexist across deployment modes without manual scrubbing. - - ## Commit vs ignore - - Successful Strategy/Repo lookups commit their projected value to the - cache. Failures — empty `packages/1` results, unknown adaptors for - `icon_meta/2`, Strategy errors — return `:ignore`, so a subsequent - caller retries fresh rather than seeing a poisoned cache entry. - - ## Icons - - `icon/3` returns a `Path.t/0` the controller serves via `send_file/3` - — no binary on the BEAM heap. The on-disk `Lightning.Adaptors.IconCache` - is the primary cache: a `cached?/4` hit short-circuits before Cachex - is touched. On a disk miss the lazy Strategy fetch is wrapped in - `Cachex.fetch/4` on `{:icon_bytes, source, name, shape}` so that - concurrent first-callers coalesce onto a single courier; the courier - returns `{:ignore, _}` so no entry is committed, and subsequent - callers re-read the now-populated file from disk. + Cached reads over `Lightning.Adaptors.Catalogue`. + + Every read checks the instance's Cachex first and falls back to the + catalogue table. `schema/2` and `versions/2` also fetch from the + strategy when the row has no data yet and persist what they get; this + only fills gaps on adaptors already in the catalogue, and an unknown + name returns `{:error, :not_found}`. `icon/3` returns a path on disk, + fetching the bytes from the strategy on the first miss. `catalogue/1` + caches the picker payload already rendered, together with the ETag + stamp that describes it. """ + alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Config alias Lightning.Adaptors.IconCache - alias Lightning.Adaptors.Repo, as: AdaptorsRepo alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor - - # Strategy and source are scoped to the supervisor instance — every - # `Store` call resolves both from the per-instance `:persistent_term` - # entry the supervisor populated at boot. No `Application.get_env` - # reads in the hot path; no global mutable state in tests. + alias LightningWeb.AdaptorIconURL @type sup :: atom() @@ -61,17 +35,24 @@ defmodule Lightning.Adaptors.Store do icon_rectangle_sha256: binary() | nil } - @type package_meta :: AdaptorsRepo.package_meta() - - @doc """ - Read the `schema_data` JSON blob for a single adaptor. + @type package_meta :: Catalogue.package_meta() + + @type catalogue_entry :: %{ + name: String.t(), + latest_version: String.t(), + versions: [String.t()], + repository: String.t() | nil, + icon_urls: %{ + square: String.t() | nil, + rectangle: String.t() | nil + } + } - Cache-then-Repo-then-Strategy. On Strategy success the full adaptor - record is upserted into Postgres and the projected schema blob is - committed to the cache. + @type catalogue :: + {{DateTime.t() | nil, non_neg_integer()}, [catalogue_entry()]} - Returns the schema as a JSON binary so that ordered-objects decoding - can re-engage at the credential-form renderer. + @doc """ + Returns the adaptor's credential schema as a JSON binary, not decoded. """ @spec schema(sup(), String.t()) :: {:ok, String.t()} | {:error, term()} def schema(sup, name) do @@ -82,7 +63,7 @@ defmodule Lightning.Adaptors.Store do |> Cachex.fetch( {:schema, name, source}, fn _key -> - case AdaptorsRepo.get_adaptor(name, source) do + case Catalogue.get_adaptor(name, source) do %{schema_data: data} when not is_nil(data) -> {:commit, {:ok, data}} @@ -96,8 +77,7 @@ defmodule Lightning.Adaptors.Store do end @doc """ - Read the version history for a single adaptor as a list of lean - per-version maps. See `t:version_meta/0` for the projected shape. + Returns the adaptor's version history as `t:version_meta/0` maps. """ @spec versions(sup(), String.t()) :: {:ok, [version_meta()]} | {:error, term()} @@ -109,7 +89,7 @@ defmodule Lightning.Adaptors.Store do |> Cachex.fetch( {:versions, name, source}, fn _key -> - case AdaptorsRepo.list_versions(name, source) do + case Catalogue.list_versions(name, source) do [] -> fetch_and_persist(sup, name, source, :versions) rows -> {:commit, {:ok, project_versions(rows)}} end @@ -120,17 +100,8 @@ defmodule Lightning.Adaptors.Store do end @doc """ - Resolve the on-disk path of one icon variant for an adaptor. - - Disk is the cache: a cache-hit on `IconCache.cached?/4` returns the - path immediately. A cache-miss is routed through `Cachex.fetch/4` on - `{:icon_bytes, source, name, shape}` so concurrent first-callers - coalesce onto one in-flight Strategy fetch — the courier returns - `{:ignore, _}` so no cache entry is committed and the next miss reads - the freshly-written file from disk. - - Returns `{:error, :not_found}` when the icon variant is absent from - the adaptor row (the row is the source of truth). + Returns the on-disk path of one icon shape for the adaptor, or + `{:error, :not_found}` when the adaptor row has no such icon. """ @spec icon(sup(), String.t(), :square | :rectangle) :: {:ok, Path.t()} | {:error, :not_found | term()} @@ -171,13 +142,8 @@ defmodule Lightning.Adaptors.Store do end @doc """ - Picker-facing lean projection: every adaptor row for the active - source, minus heavy JSONB columns (`schema_data`, `dependencies`, - `peer_dependencies`). - - An empty Repo result returns `{:ok, []}` but is **not** committed to - the cache — during cold-start the Scheduler will fill the table on - its next tick, and the next call will pick that up automatically. + Returns every adaptor for the active source, without the `schema_data`, + `dependencies` and `peer_dependencies` columns. """ @spec packages(sup()) :: {:ok, [package_meta()]} | {:error, term()} def packages(sup) do @@ -188,7 +154,7 @@ defmodule Lightning.Adaptors.Store do |> Cachex.fetch( {:packages, source}, fn _key -> - case AdaptorsRepo.list_package_metas(source) do + case Catalogue.list_package_metas(source) do [] -> {:ignore, {:ok, []}} metas -> {:commit, {:ok, metas}} end @@ -199,12 +165,35 @@ defmodule Lightning.Adaptors.Store do end @doc """ - Cheap `{icon__ext, icon__sha256}` projection for the - icon controller's sha-validation path. Pure metadata — no disk I/O. + Returns the picker catalogue for the active source as + `{stamp, rendered_entries}`. + + The ETag stamp and the payload it describes are cached as one entry so + a 304 can be answered without re-reading the projection, and so the two + can never drift apart. + """ + @spec catalogue(sup()) :: {:ok, catalogue()} | {:error, term()} + def catalogue(sup) do + cache = AdaptorsSupervisor.cache_name(sup) + source = AdaptorsSupervisor.source(sup) + + cache + |> Cachex.fetch( + {:catalogue, source}, + fn _key -> + case build_catalogue(source) do + {_stamp, []} = empty -> {:ignore, {:ok, empty}} + filled -> {:commit, {:ok, filled}} + end + end, + timeout: Config.cache_timeout_ms() + ) + |> unwrap() + end - Unknown adaptors return `{:error, :not_found}` and are **not** - cached, so a subsequent insert by the Scheduler becomes visible on - the very next call. + @doc """ + Returns the extension and sha256 of each icon shape for the adaptor, + without touching disk, or `{:error, :not_found}` for an unknown name. """ @spec icon_meta(sup(), String.t()) :: {:ok, icon_meta()} | {:error, :not_found} @@ -216,7 +205,7 @@ defmodule Lightning.Adaptors.Store do |> Cachex.fetch( {:icon_meta, name, source}, fn _key -> - case AdaptorsRepo.get_adaptor(name, source) do + case Catalogue.get_adaptor(name, source) do nil -> {:ignore, {:error, :not_found}} adaptor -> {:commit, {:ok, project_icon_meta(adaptor)}} end @@ -227,55 +216,99 @@ defmodule Lightning.Adaptors.Store do end @doc """ - Re-warm Cachex from Postgres for the active source. - - Called by `Lightning.Adaptors.NodeMonitor` on `:nodeup` — a peer - rejoining after a partition can't know which `{:changed, name, source}` - broadcasts it missed, so it treats its entire local Cachex as - suspect and overwrites from the DB. - - Uses `Cachex.put_many/2` (never `Cachex.clear/1`-then-fill) so - concurrent callers never observe an empty cache and never trigger a - spurious cold-miss Strategy fetch during the warm. + Overwrites the cached package list, icon metadata and catalogue from the + database. An empty catalogue is left uncached, as `catalogue/1` does. """ @spec warm_from_repo(sup()) :: :ok def warm_from_repo(sup) do cache = AdaptorsSupervisor.cache_name(sup) source = AdaptorsSupervisor.source(sup) - metas = AdaptorsRepo.list_package_metas(source) + metas = Catalogue.list_package_metas(source) icon_metas = Enum.map(metas, fn m -> {{:icon_meta, m.name, source}, {:ok, project_icon_meta(m)}} end) + catalogue = + case build_catalogue(source) do + {_stamp, []} -> [] + filled -> [{{:catalogue, source}, {:ok, filled}}] + end + Cachex.put_many( cache, - [{{:packages, source}, {:ok, metas}} | icon_metas] + [{{:packages, source}, {:ok, metas}} | icon_metas] ++ catalogue ) :ok end + # The stamp is read before the projection so it can only ever be older + # than the payload it describes, never newer: a lagging stamp costs a + # client one extra 200, a leading one would serve a stale 304. + @spec build_catalogue(Catalogue.source()) :: catalogue() + defp build_catalogue(source) do + stamp = Catalogue.catalogue_stamp(source) + + {stamp, + source |> Catalogue.catalogue() |> Enum.map(&render_entry(&1, source))} + end + + @spec render_entry(Catalogue.catalogue_entry(), Catalogue.source()) :: + catalogue_entry() + defp render_entry(entry, source) do + %{ + name: entry.name, + latest_version: + if(source == :local, do: "local", else: entry.latest_version), + versions: if(source == :local, do: ["local"], else: entry.versions), + repository: entry.repository, + icon_urls: %{ + square: AdaptorIconURL.build(entry.name, entry, :square), + rectangle: AdaptorIconURL.build(entry.name, entry, :rectangle) + } + } + end + + # Lazy fetches only fill gaps on adaptors already in the catalogue; + # they never add one. @spec fetch_and_persist(atom(), String.t(), :npm | :local, atom()) :: {:commit, {:ok, term()}} | {:ignore, {:error, term()}} defp fetch_and_persist(sup, name, source, field) do + if Catalogue.get_adaptor(name, source) do + fetch_and_persist_known(sup, name, source, field) + else + {:ignore, {:error, :not_found}} + end + end + + defp fetch_and_persist_known(sup, name, source, field) do case AdaptorsSupervisor.strategy(sup).fetch_adaptor(name) do - {:ok, record} -> + {:ok, %{name: ^name} = record} -> record = record |> Map.put(:source, source) |> normalize_schema_data() - {:ok, _} = AdaptorsRepo.upsert_adaptor(record) - {:commit, {:ok, Map.get(record, field)}} + {:ok, _} = Catalogue.upsert_adaptor(record) + {:commit, {:ok, record |> Map.get(field) |> project_field(field)}} + + {:ok, %{name: other}} -> + {:ignore, {:error, {:name_mismatch, other}}} {:error, reason} -> {:ignore, {:error, reason}} end end + # Both cache paths must store the same projected shape. + defp project_field(rows, :versions) when is_list(rows), + do: project_versions(rows) + + defp project_field(value, _field), do: value + # Strategies should emit `schema_data` as a JSON binary, but legacy # call paths (and tests) may still hand us a map. Normalize here so # the cached value matches what subsequent DB-backed reads return. diff --git a/lib/lightning/adaptors/supervisor.ex b/lib/lightning/adaptors/supervisor.ex index ace6c948c86..caeb934d765 100644 --- a/lib/lightning/adaptors/supervisor.ex +++ b/lib/lightning/adaptors/supervisor.ex @@ -1,42 +1,13 @@ defmodule Lightning.Adaptors.Supervisor do @moduledoc """ - Per-instance supervisor for the `Lightning.Adaptors.*` subsystem. - - Children boot in list order — `Cachex` and the `Task.Supervisor` - before the processes that use them — but are supervised - `:one_for_one`. Every child addresses its collaborators by registered - name, resolved per call, so a restarted `Cachex` or `Task.Supervisor` - re-registers under the same name and nothing has to be rebuilt around - it. Cascading a restart would only cost the `HighlanderPG`-wrapped - `Scheduler` its advisory lock and force a needless re-election. - - No registered name, Cachex table name, PubSub topic, `Task.Supervisor` - name, or `HighlanderPG` lock key is hardcoded. Every name is derived - from a single `:name` opt — which is what lets the integration suite - spin up multiple isolated instances inside one BEAM for - `async: true` tests. Production starts exactly one instance under - `name: Lightning.Adaptors`. - - ## Cluster-singleton Scheduler - - The `Lightning.Adaptors.Scheduler` is wrapped in `HighlanderPG` - (`pg_try_advisory_lock` on `lock_key/1`) so exactly one node in a - multi-node deployment runs the refresh tick. The inner Scheduler - registers under `{:global, global_scheduler_name(name)}`; callers on - any node hit the leader transparently via Erlang distribution. - - ## Strategy injection - - The active `Lightning.Adaptors.Strategy` implementation is passed in - explicitly via the `:strategy` opt. Tests instantiate an isolated - supervisor with `strategy: Lightning.Adaptors.StrategyMock` — no - `Application.put_env` mutation, no shared mutable state. The - production caller in `lib/lightning/application.ex` passes the - default from `Lightning.Adaptors.Config.strategy/0` (resolved from - Application env at boot time). - - `strategy/1` and `source/1` expose the per-instance values back to - the stateless `Lightning.Adaptors.Store` callers. + Supervises one instance of the adaptors subsystem: the cache, task + supervisor, invalidator, node monitor, channel broadcaster and the + `HighlanderPG`-wrapped scheduler. + + Every child, cache, topic and lock name derives from the `:name` opt, + so several instances can run in one BEAM. Production starts one under + `name: Lightning.Adaptors`. The scheduler is a cluster singleton + registered under `global_scheduler_name/1`. """ use Supervisor @@ -44,21 +15,15 @@ defmodule Lightning.Adaptors.Supervisor do alias Lightning.Adaptors.Config @doc """ - Start a supervisor instance. + Starts a supervisor instance. - Required opts: + Options: - * `:name` — supervisor instance name (atom). Derives every child - name via `Module.concat/2`. - - Optional opts: - - * `:strategy` — `Lightning.Adaptors.Strategy` implementation. - Defaults to `Lightning.Adaptors.Config.strategy/0`. - - * `:lock_key` — explicit `HighlanderPG` advisory-lock key. Defaults - to `lock_key(name)`. Override only in integration tests where - multiple supervisor instances must compete for the same lock. + * `:name` - required; every child name derives from it + * `:strategy` - `Lightning.Adaptors.Strategy` implementation, + defaulting to `Lightning.Adaptors.Config.strategy/0` + * `:lock_key` - `HighlanderPG` advisory-lock key, defaulting to + `lock_key(name)` """ @spec start_link(keyword()) :: Supervisor.on_start() def start_link(opts) do @@ -74,7 +39,7 @@ defmodule Lightning.Adaptors.Supervisor do :persistent_term.put(meta_key(name), %{ strategy: strategy, - source: source_for(strategy) + source: Config.source_for(strategy) }) cache = cache_name(name) @@ -123,10 +88,8 @@ defmodule Lightning.Adaptors.Supervisor do end @doc """ - The active strategy for the supervisor instance named `name`. - - Reads from `:persistent_term` populated at `init/1`. Raises if the - supervisor has not been started under that name. + Returns the strategy of the supervisor named `name`. Raises if no + supervisor has started under that name. """ @spec strategy(atom()) :: module() def strategy(name) do @@ -134,8 +97,7 @@ defmodule Lightning.Adaptors.Supervisor do end @doc """ - The active source (`:npm | :local`) for the supervisor instance - named `name`. + Returns the source (`:npm | :local`) of the supervisor named `name`. """ @spec source(atom()) :: :npm | :local def source(name) do @@ -143,11 +105,8 @@ defmodule Lightning.Adaptors.Supervisor do end @doc """ - Best-effort cleanup of the per-instance `:persistent_term` entry. - - Not called automatically — `:persistent_term.erase/1` triggers a - global GC and is expensive enough that we leave it to deliberate - teardown paths (e.g. release shutdown). + Erases the instance's `:persistent_term` entry. Not called + automatically, since `:persistent_term.erase/1` triggers a global GC. """ @spec forget(atom()) :: boolean() def forget(name) do @@ -176,67 +135,42 @@ defmodule Lightning.Adaptors.Supervisor do def node_monitor_name(name), do: Module.concat(name, NodeMonitor) @doc """ - Local `Scheduler` GenServer name for the supervisor named `name`. - - The inner Scheduler is actually registered globally — see - `global_scheduler_name/1`. This atom form is retained as the - child-spec `id` and for derived module names. + Atom form of the Scheduler name for the supervisor named `name`. The + process itself registers under `global_scheduler_name/1`. """ @spec scheduler_name(atom()) :: atom() def scheduler_name(name), do: Module.concat(name, Scheduler) @doc """ - `:global`-registered Scheduler name for the supervisor named `name`. - - The HighlanderPG-wrapped Scheduler registers itself under this name so - callers on any node reach the leader via Erlang distribution. Pass the - return value to `GenServer.call/3` directly. + `:global` Scheduler name for the supervisor named `name`; pass it to + `GenServer.call/3` directly. """ @spec global_scheduler_name(atom()) :: {:global, atom()} def global_scheduler_name(name), do: {:global, scheduler_name(name)} - @doc """ - `HighlanderPG` supervisor name for the supervisor named `name`. - - Used as the child-spec id and the `:sup_name` for introspection - (`HighlanderPG.which_children/1`, etc.). - """ + @doc "`HighlanderPG` supervisor name for the supervisor named `name`." @spec highlander_name(atom()) :: atom() def highlander_name(name), do: Module.concat(name, HighlanderPG) @doc """ - Source-side PubSub topic for the supervisor named `name`. - - Used by the `Scheduler` and `Invalidator` to broadcast and receive - `{:changed, name, source}` style events. + PubSub topic carrying `{:changed, name, source}` events for the + supervisor named `name`. """ @spec source_topic(atom()) :: String.t() def source_topic(name), do: "adaptors:#{inspect(name)}" @doc """ - Client-side PubSub topic for the supervisor named `name`. - - The `ChannelBroadcaster` republishes throttled updates from - `source_topic/1` onto this topic for `WorkflowChannel` subscribers. + PubSub topic carrying debounced `adaptors_updated` events for the + supervisor named `name`; see `Lightning.Adaptors.subscribe_to_updates/1`. """ @spec client_topic(atom()) :: String.t() def client_topic(name), do: "adaptors:client_update:#{inspect(name)}" @doc """ Postgres advisory-lock key for the supervisor named `name`. - - Derived as `:erlang.phash2({:adaptors, name})` so each supervisor - instance leases its `HighlanderPG`-wrapped `Scheduler` against a - distinct `int4` key — two concurrent test supervisors with different - names cannot collide on advisory locks. The §12.7 integration test - overrides `:lock_key` on `start_link/1` to force two supervisors to - compete for the same lock. """ @spec lock_key(atom()) :: non_neg_integer() def lock_key(name), do: :erlang.phash2({:adaptors, name}) defp meta_key(name), do: {__MODULE__, name} - - defp source_for(Lightning.Adaptors.Local), do: :local - defp source_for(_other), do: :npm end diff --git a/lib/lightning/ai_assistant/ai_assistant.ex b/lib/lightning/ai_assistant/ai_assistant.ex index 536b81c0fa9..55f0323aa07 100644 --- a/lib/lightning/ai_assistant/ai_assistant.ex +++ b/lib/lightning/ai_assistant/ai_assistant.ex @@ -550,11 +550,13 @@ defmodule Lightning.AiAssistant do @spec put_expression_and_adaptor(ChatSession.t(), String.t(), String.t()) :: ChatSession.t() def put_expression_and_adaptor(session, expression, adaptor) do - %{ - session - | expression: expression, - adaptor: Lightning.Adaptors.to_wire(adaptor) - } + wire = + case Lightning.Adaptors.to_wire(adaptor) do + {:ok, wire} -> wire + {:error, _} -> adaptor + end + + %{session | expression: expression, adaptor: wire} end @doc """ diff --git a/lib/lightning/collaboration/session.ex b/lib/lightning/collaboration/session.ex index 5769ccf6912..5a645acf742 100644 --- a/lib/lightning/collaboration/session.ex +++ b/lib/lightning/collaboration/session.ex @@ -218,6 +218,8 @@ defmodule Lightning.Collaboration.Session do - `{:error, :workflow_deleted}` - Workflow has been deleted - `{:error, :snapshot_failed}` - Snapshot creation failed; it shares the save's transaction, so the whole save rolled back and nothing persisted + - `{:error, :adaptor_catalogue_unavailable}` - The adaptor catalogue's + first load did not complete, so no validation could run - `{:error, changeset}` - Validation or persistence error ## Examples @@ -235,9 +237,14 @@ defmodule Lightning.Collaboration.Session do | :snapshot_failed | :deserialization_failed | :internal_error + | :adaptor_catalogue_unavailable | Ecto.Changeset.t()} def save_workflow(session_pid, user) do - GenServer.call(session_pid, {:save_workflow, user}, 10_000) + GenServer.call( + session_pid, + {:save_workflow, user}, + Lightning.Adaptors.Config.first_load_timeout() + 10_000 + ) end @doc """ @@ -340,8 +347,22 @@ defmodule Lightning.Collaboration.Session do end @impl true - def handle_call({:save_workflow, user}, _from, state) do - do_save_workflow(state, user, :save) + def handle_call({:save_workflow, user}, from, state) do + session = self() + + Task.start(fn -> + case ensure_catalogue_loaded() do + :ok -> + send(session, {:resume_save, from, user}) + + {:error, reason} -> + Logger.info("Adaptor catalogue not ready for save: #{inspect(reason)}") + + GenServer.reply(from, {:error, :adaptor_catalogue_unavailable}) + end + end) + + {:noreply, state} end @impl true @@ -385,6 +406,23 @@ defmodule Lightning.Collaboration.Session do end end + defp ensure_catalogue_loaded do + Lightning.Adaptors.ensure_loaded() + rescue + error -> + {:error, error} + catch + :exit, reason -> + {:error, {:exit, reason}} + end + + @impl true + def handle_info({:resume_save, from, user}, state) do + {:reply, reply, state} = do_save_workflow(state, user, :save) + GenServer.reply(from, reply) + {:noreply, state} + end + @impl true def handle_info({:yjs, reply, shared_doc_pid}, state) do Logger.debug( diff --git a/lib/lightning/release.ex b/lib/lightning/release.ex index 185761afbe2..6e59f2fe290 100644 --- a/lib/lightning/release.ex +++ b/lib/lightning/release.ex @@ -39,7 +39,7 @@ defmodule Lightning.Release do @doc """ Populate the adaptor catalogue from a JSON snapshot file, without reaching npm. The release-safe path for `Lightning.Adaptors.seed_from_file/2` - — there is no Mix in a release, so `mix lightning.seed_adaptors_from_file` + — there is no Mix in a release, so `mix lightning.adaptors.import` cannot run there; this is what `bin/lightning eval` calls instead. ## Usage diff --git a/lib/lightning/workflows/job.ex b/lib/lightning/workflows/job.ex index 00b9ec5bd31..f1139558614 100644 --- a/lib/lightning/workflows/job.ex +++ b/lib/lightning/workflows/job.ex @@ -153,16 +153,20 @@ defmodule Lightning.Workflows.Job do end end - # `job.adaptor` reaches the worker's install step unfiltered, so an - # adaptor missing from the catalogue is rejected here whatever its name - # — an empty catalogue permits nothing. defp validate_known_adaptor(changeset) do validate_change(changeset, :adaptor, fn :adaptor, adaptor -> with {name, _version} when is_binary(name) <- Adaptors.parse_spec(adaptor), - %Adaptors.Package{} <- Adaptors.get_adaptor(name) do + {:ok, _package} <- Adaptors.fetch_adaptor(name) do [] else - _ -> [adaptor: "is not a recognised adaptor"] + {:error, :not_found} -> + [adaptor: "is not a recognised adaptor"] + + {:error, _} -> + [adaptor: "adaptor catalogue is not ready yet, try again shortly"] + + _ -> + [adaptor: "is not a recognised adaptor"] end end) end diff --git a/lib/lightning_web/channels/run_channel.ex b/lib/lightning_web/channels/run_channel.ex index 70a84a9299e..b9948e84ba6 100644 --- a/lib/lightning_web/channels/run_channel.ex +++ b/lib/lightning_web/channels/run_channel.ex @@ -100,9 +100,13 @@ defmodule LightningWeb.RunChannel do @impl true def handle_in("fetch:plan", _payload, socket) do - %{run: run} = socket.assigns + case RunWithOptions.render(socket.assigns.run) do + {:ok, plan} -> + reply_with(socket, {:ok, plan}) - reply_with(socket, {:ok, RunWithOptions.render(run)}) + {:error, reason} -> + reply_with(socket, {:error, %{reason: "adaptor_#{reason}"}}) + end end def handle_in("run:start", payload, socket) do diff --git a/lib/lightning_web/channels/run_with_options.ex b/lib/lightning_web/channels/run_with_options.ex index 5f13975c15e..e45f5edc356 100644 --- a/lib/lightning_web/channels/run_with_options.ex +++ b/lib/lightning_web/channels/run_with_options.ex @@ -17,19 +17,22 @@ defmodule LightningWeb.RunWithOptions do for that mapping: it takes a bunch of Lightning resources and turns them into a Worker-executable plan for a run. """ - @spec render(Run.t()) :: map() + @spec render(Run.t()) :: {:ok, map()} | {:error, term()} def render(%Run{} = run) do - %{ - "id" => run.id, - "project_id" => run.snapshot.workflow.project_id, - "triggers" => run.snapshot.triggers |> Enum.map(&render/1), - "jobs" => run.snapshot.jobs |> Enum.map(&render/1), - "edges" => run.snapshot.edges |> Enum.map(&render/1), - "starting_node_id" => run.starting_trigger_id || run.starting_job_id, - "dataclip_id" => run.dataclip_id, - "options" => options_for_worker(run.options), - "meta" => render_meta(run) - } + with {:ok, jobs} <- render_jobs(run.snapshot.jobs) do + {:ok, + %{ + "id" => run.id, + "project_id" => run.snapshot.workflow.project_id, + "triggers" => run.snapshot.triggers |> Enum.map(&render/1), + "jobs" => jobs, + "edges" => run.snapshot.edges |> Enum.map(&render/1), + "starting_node_id" => run.starting_trigger_id || run.starting_job_id, + "dataclip_id" => run.dataclip_id, + "options" => options_for_worker(run.options), + "meta" => render_meta(run) + }} + end end def render(%Trigger{} = trigger) do @@ -39,13 +42,16 @@ defmodule LightningWeb.RunWithOptions do end def render(%Job{} = job) do - %{ - "id" => job.id, - "adaptor" => Adaptors.to_wire(job.adaptor), - "credential_id" => get_credential_id(job), - "body" => job.body, - "name" => job.name - } + with {:ok, adaptor} <- Adaptors.to_wire(job.adaptor) do + {:ok, + %{ + "id" => job.id, + "adaptor" => adaptor, + "credential_id" => get_credential_id(job), + "body" => job.body, + "name" => job.name + }} + end end def render( @@ -72,6 +78,15 @@ defmodule LightningWeb.RunWithOptions do } end + defp render_jobs(jobs) do + rendered = Enum.map(jobs, &render/1) + + case Enum.find(rendered, &match?({:error, _}, &1)) do + nil -> {:ok, Enum.map(rendered, fn {:ok, job} -> job end)} + error -> error + end + end + defp render_meta(run) do %{ "work_order_id" => run.work_order_id, diff --git a/lib/lightning_web/channels/workflow_channel.ex b/lib/lightning_web/channels/workflow_channel.ex index b11fb8e8a18..64ab99fe977 100644 --- a/lib/lightning_web/channels/workflow_channel.ex +++ b/lib/lightning_web/channels/workflow_channel.ex @@ -1,9 +1,6 @@ defmodule LightningWeb.WorkflowChannel do @moduledoc """ - Phoenix Channel for handling binary Yjs collaboration messages. - - Unlike LiveView events, Phoenix Channels properly support binary data - transmission without JSON serialization. + Phoenix Channel for binary Yjs collaboration messages. """ use LightningWeb, :channel @@ -91,10 +88,7 @@ defmodule LightningWeb.WorkflowChannel do "workflow:collaborate:#{workflow_id}" ) - Phoenix.PubSub.subscribe( - Lightning.PubSub, - Lightning.Adaptors.Supervisor.client_topic(Lightning.Adaptors) - ) + Lightning.Adaptors.subscribe_to_updates() {:ok, assign(socket, @@ -306,49 +300,28 @@ defmodule LightningWeb.WorkflowChannel do end @doc """ - Handles explicit workflow save requests from the collaborative editor. - - The save operation: - 1. Asks Session to extract and save the current Y.Doc state - 2. Session handles all Y.Doc interaction internally - 3. Returns success/error to the client + Saves the current Y.Doc state through the Session. - Note: By the time this message is processed, all prior Y.js sync messages - have been processed due to Phoenix Channel's synchronous per-socket handling. + The reply is deferred: `Session.save_workflow/2` may wait on the adaptor + catalogue's first load, so the call runs off the channel process and + the reply is sent with `Phoenix.Channel.reply/2` when it finishes. - Success response: {:ok, %{saved_at: DateTime, lock_version: integer}} - Error response: {:error, %{errors: map, type: string}} + Success: `{:ok, %{saved_at: DateTime, lock_version: integer}}` + Error: `{:error, %{errors: map, type: string}}` """ @impl true def handle_in("save_workflow", _params, socket) do - session_pid = socket.assigns.session_pid - user = socket.assigns.current_user - - with :ok <- authorize_content_edit(socket), - {:ok, workflow} <- Session.save_workflow(session_pid, user) do - # Broadcast the new lock_version to all users in the channel - # so they can update their latestSnapshotLockVersion in SessionContextStore - broadcast_from!(socket, "workflow_saved", %{ - latest_snapshot_lock_version: workflow.lock_version, - workflow: workflow - }) + case authorize_content_edit(socket) do + :ok -> + session_pid = socket.assigns.session_pid + user = socket.assigns.current_user - # The workflow now has a DB row, so this channel is no longer editing a - # brand-new (:new) workflow. No client rejoin happens after a first save, - # so we must self-promote the cached kind + struct here; otherwise - # request_versions / get_context keep short-circuiting to empty for the - # rest of this session (until a full page refresh re-joins as :existing). - socket = assign(socket, workflow: workflow, workflow_kind: :existing) + defer_reply(socket, :save_workflow_reply, fn -> + Session.save_workflow(session_pid, user) + end) - {:reply, - {:ok, - %{ - saved_at: workflow.updated_at, - lock_version: workflow.lock_version, - workflow: workflow - }}, socket} - else - error -> workflow_error_reply(socket, error) + error -> + {:reply, workflow_error_reply(error), socket} end end @@ -407,7 +380,7 @@ defmodule LightningWeb.WorkflowChannel do {:reply, {:ok, %{sandboxes: sandboxes}}, socket} error -> - workflow_error_reply(socket, error) + {:reply, workflow_error_reply(error), socket} end end @@ -448,7 +421,7 @@ defmodule LightningWeb.WorkflowChannel do dataclip_id: starting_dataclip_id }}, socket} else - error -> workflow_error_reply(socket, error) + error -> {:reply, workflow_error_reply(error), socket} end end @@ -538,8 +511,8 @@ defmodule LightningWeb.WorkflowChannel do {:reply, {:ok, %{lock_version: restored.lock_version}}, socket} else - nil -> workflow_error_reply(socket, {:error, :version_not_found}) - error -> workflow_error_reply(socket, error) + nil -> {:reply, workflow_error_reply({:error, :version_not_found}), socket} + error -> {:reply, workflow_error_reply(error), socket} end end @@ -559,8 +532,8 @@ defmodule LightningWeb.WorkflowChannel do {:ok, result} <- Projects.promote_workflow(workflow, user) do {:reply, {:ok, result}, socket} else - nil -> workflow_error_reply(socket, {:error, :not_a_sandbox}) - error -> workflow_error_reply(socket, error) + nil -> {:reply, workflow_error_reply({:error, :not_a_sandbox}), socket} + error -> {:reply, workflow_error_reply(error), socket} end end @@ -581,8 +554,8 @@ defmodule LightningWeb.WorkflowChannel do {:ok, _scheduled} <- Sandboxes.schedule_sandbox_deletion(sandbox, user) do {:reply, {:ok, %{parent_project_id: parent.id}}, socket} else - nil -> workflow_error_reply(socket, {:error, :not_a_sandbox}) - error -> workflow_error_reply(socket, error) + nil -> {:reply, workflow_error_reply({:error, :not_a_sandbox}), socket} + error -> {:reply, workflow_error_reply(error), socket} end end @@ -599,47 +572,24 @@ defmodule LightningWeb.WorkflowChannel do @impl true def handle_in("save_and_sync", %{"commit_message" => commit_message}, socket) do - session_pid = socket.assigns.session_pid - user = socket.assigns.current_user - project = socket.assigns.project - - with :ok <- authorize_content_edit(socket), - {:ok, workflow} <- Session.save_workflow(session_pid, user), - repo_connection when not is_nil(repo_connection) <- - VersionControl.get_repo_connection_for_project(project.id), - :ok <- VersionControl.initiate_sync(repo_connection, commit_message) do - broadcast_from!(socket, "workflow_saved", %{ - latest_snapshot_lock_version: workflow.lock_version, - workflow: workflow - }) - - {:reply, - {:ok, - %{ - saved_at: workflow.updated_at, - lock_version: workflow.lock_version, - repo: repo_connection.repo, - workflow: workflow - }}, socket} - else - nil -> - {:reply, - {:error, - %{ - errors: %{base: ["No GitHub connection configured for this project"]}, - type: "github_sync_error" - }}, socket} - - {:error, reason} when is_binary(reason) -> - {:reply, - {:error, - %{ - errors: %{base: [reason]}, - type: "github_sync_error" - }}, socket} + case authorize_content_edit(socket) do + :ok -> + session_pid = socket.assigns.session_pid + user = socket.assigns.current_user + project = socket.assigns.project + + defer_reply(socket, :save_and_sync_reply, fn -> + with {:ok, workflow} <- Session.save_workflow(session_pid, user), + repo_connection when not is_nil(repo_connection) <- + VersionControl.get_repo_connection_for_project(project.id), + :ok <- + VersionControl.initiate_sync(repo_connection, commit_message) do + {:ok, workflow, repo_connection} + end + end) error -> - workflow_error_reply(socket, error) + {:reply, workflow_error_reply(error), socket} end end @@ -657,7 +607,7 @@ defmodule LightningWeb.WorkflowChannel do workflow_id: workflow.id }}, socket} else - error -> workflow_error_reply(socket, error) + error -> {:reply, workflow_error_reply(error), socket} end end @@ -870,7 +820,7 @@ defmodule LightningWeb.WorkflowChannel do {:reply, {:ok, %{template: render_workflow_template(template)}}, socket} else - error -> workflow_error_reply(socket, error) + error -> {:reply, workflow_error_reply(error), socket} end end @@ -966,6 +916,98 @@ defmodule LightningWeb.WorkflowChannel do {:noreply, socket} end + @impl true + def handle_info({:save_workflow_reply, ref, {:ok, workflow}}, socket) do + # Broadcast the new lock_version to all users in the channel so they can + # update their latestSnapshotLockVersion in SessionContextStore. + broadcast_from!(socket, "workflow_saved", %{ + latest_snapshot_lock_version: workflow.lock_version, + workflow: workflow + }) + + reply( + ref, + {:ok, + %{ + saved_at: workflow.updated_at, + lock_version: workflow.lock_version, + workflow: workflow + }} + ) + + # The workflow now has a DB row, so this channel is no longer editing a + # brand-new (:new) workflow. No client rejoin happens after a first save, + # so we must self-promote the cached kind + struct here; otherwise + # request_versions / get_context keep short-circuiting to empty for the + # rest of this session (until a full page refresh re-joins as :existing). + {:noreply, assign(socket, workflow: workflow, workflow_kind: :existing)} + end + + @impl true + def handle_info({:save_workflow_reply, ref, error}, socket) do + reply(ref, workflow_error_reply(error)) + {:noreply, socket} + end + + @impl true + def handle_info( + {:save_and_sync_reply, ref, {:ok, workflow, repo_connection}}, + socket + ) do + broadcast_from!(socket, "workflow_saved", %{ + latest_snapshot_lock_version: workflow.lock_version, + workflow: workflow + }) + + reply( + ref, + {:ok, + %{ + saved_at: workflow.updated_at, + lock_version: workflow.lock_version, + repo: repo_connection.repo, + workflow: workflow + }} + ) + + {:noreply, socket} + end + + @impl true + def handle_info({:save_and_sync_reply, ref, nil}, socket) do + reply( + ref, + {:error, + %{ + errors: %{base: ["No GitHub connection configured for this project"]}, + type: "github_sync_error" + }} + ) + + {:noreply, socket} + end + + @impl true + def handle_info({:save_and_sync_reply, ref, {:error, reason}}, socket) + when is_binary(reason) do + reply( + ref, + {:error, + %{ + errors: %{base: [reason]}, + type: "github_sync_error" + }} + ) + + {:noreply, socket} + end + + @impl true + def handle_info({:save_and_sync_reply, ref, error}, socket) do + reply(ref, workflow_error_reply(error)) + {:noreply, socket} + end + @impl true def handle_info(%{event: "presence_diff", payload: _diff}, socket) do {:noreply, socket} @@ -1232,28 +1274,54 @@ defmodule LightningWeb.WorkflowChannel do defp refresh_lifecycle_from_broadcast(socket, _payload), do: socket + # Unlinked on purpose: a GenServer.call timeout or dead target exits, and + # a linked task would take the channel down. `catch :exit` turns it into + # an error reply instead. defp async_task(socket, event, task_fn) do channel_pid = self() socket_ref = socket_ref(socket) - Task.start_link(fn -> - try do - result = task_fn.() + Task.start(fn -> + result = + try do + {:ok, task_fn.()} + rescue + error -> + Logger.error("Failed to handle #{event}: #{inspect(error)}") + {:error, %{reason: "failed to handle #{event}"}} + catch + :exit, reason -> + Logger.error("Failed to handle #{event}: #{inspect(reason)}") + {:error, %{reason: "failed to handle #{event}"}} + end - send( - channel_pid, - {:async_reply, socket_ref, event, {:ok, result}} - ) - rescue - error -> - Logger.error("Failed to handle #{event}: #{inspect(error)}") - - send( - channel_pid, - {:async_reply, socket_ref, event, - {:error, %{reason: "failed to handle #{event}"}}} - ) - end + send(channel_pid, {:async_reply, socket_ref, event, result}) + end) + + {:noreply, socket} + end + + # As `async_task/3`, but the reply is post-processed by the `handle_info/2` + # clause for `tag`. + defp defer_reply(socket, tag, task_fn) do + channel_pid = self() + ref = socket_ref(socket) + + Task.start(fn -> + result = + try do + task_fn.() + rescue + error -> + Logger.error("Failed to handle #{tag}: #{inspect(error)}") + {:error, :internal_error} + catch + :exit, reason -> + Logger.error("Failed to handle #{tag}: #{inspect(reason)}") + {:error, :internal_error} + end + + send(channel_pid, {tag, ref, result}) end) {:noreply, socket} @@ -1282,21 +1350,15 @@ defmodule LightningWeb.WorkflowChannel do end end - defp with_icon_urls(adaptor) do - Map.put(adaptor, :icon_urls, icon_urls_for(adaptor.name)) - end - - defp icon_urls_for(name) do - case Lightning.Adaptors.icon_meta(name) do - {:ok, meta} -> - %{ - square: LightningWeb.AdaptorIconURL.build(name, meta, :square), - rectangle: LightningWeb.AdaptorIconURL.build(name, meta, :rectangle) - } - - {:error, :not_found} -> - %{square: nil, rectangle: nil} - end + defp with_icon_urls(%Lightning.Adaptors.Package{name: name} = pkg) do + %{ + name: name, + latest_version: pkg.latest_version, + icon_urls: %{ + square: LightningWeb.AdaptorIconURL.build(name, pkg, :square), + rectangle: LightningWeb.AdaptorIconURL.build(name, pkg, :rectangle) + } + } end defp handle_async_event("request_run_steps", socket_ref, reply) do @@ -1645,173 +1707,158 @@ defmodule LightningWeb.WorkflowChannel do end end - # Private helper functions for save_workflow and reset_workflow - - defp workflow_error_reply(socket, {:error, %{type: type, message: message}}) do - {:reply, - {:error, - %{ - errors: %{base: [message]}, - type: type - }}, socket} + # Returns the bare reply payload, not `{:reply, ..., socket}`, so deferred + # replies can use it too. + defp workflow_error_reply({:error, %{type: type, message: message}}) do + {:error, + %{ + errors: %{base: [message]}, + type: type + }} end - defp workflow_error_reply(socket, {:error, :starting_dataclip_invalid_json}) do - starting_dataclip_error(socket, "The input you reviewed is not valid JSON.") + defp workflow_error_reply({:error, :starting_dataclip_invalid_json}) do + starting_dataclip_error("The input you reviewed is not valid JSON.") end - defp workflow_error_reply( - socket, - {:error, :starting_dataclip_not_an_object} - ) do - starting_dataclip_error( - socket, - "The input you reviewed must be a JSON object." - ) + defp workflow_error_reply({:error, :starting_dataclip_not_an_object}) do + starting_dataclip_error("The input you reviewed must be a JSON object.") end - defp workflow_error_reply(socket, {:error, :starting_dataclip_too_large}) do + defp workflow_error_reply({:error, :starting_dataclip_too_large}) do starting_dataclip_error( - socket, "The input you reviewed is too large to copy into a sandbox." ) end - defp workflow_error_reply(socket, {:error, :invalid_starting_dataclip}) do - starting_dataclip_error(socket, "The input you reviewed could not be read.") + defp workflow_error_reply({:error, :invalid_starting_dataclip}) do + starting_dataclip_error("The input you reviewed could not be read.") end - defp workflow_error_reply(socket, {:error, :starting_dataclip_not_found}) do + defp workflow_error_reply({:error, :starting_dataclip_not_found}) do starting_dataclip_error( - socket, "That saved input is no longer available in this project." ) end - defp workflow_error_reply(socket, {:error, :workflow_deleted}) do - {:reply, - {:error, - %{ - errors: %{base: ["This workflow has been deleted"]}, - type: "workflow_deleted" - }}, socket} + defp workflow_error_reply({:error, :workflow_deleted}) do + {:error, + %{ + errors: %{base: ["This workflow has been deleted"]}, + type: "workflow_deleted" + }} end - defp workflow_error_reply(socket, {:error, :deserialization_failed}) do - {:reply, - {:error, - %{ - errors: %{base: ["Failed to extract workflow data from editor"]}, - type: "deserialization_error" - }}, socket} + defp workflow_error_reply({:error, :deserialization_failed}) do + {:error, + %{ + errors: %{base: ["Failed to extract workflow data from editor"]}, + type: "deserialization_error" + }} end - defp workflow_error_reply(socket, {:error, :internal_error}) do - {:reply, - {:error, - %{ - errors: %{base: ["An internal error occurred"]}, - type: "internal_error" - }}, socket} + defp workflow_error_reply({:error, :internal_error}) do + {:error, + %{ + errors: %{base: ["An internal error occurred"]}, + type: "internal_error" + }} end - defp workflow_error_reply(socket, {:error, :nesting_too_deep}) do - {:reply, - {:error, - %{ - errors: %{ - base: ["This project is nested too deeply to create another sandbox"] - }, - type: "nesting_too_deep" - }}, socket} + defp workflow_error_reply({:error, :nesting_too_deep}) do + {:error, + %{ + errors: %{ + base: ["This project is nested too deeply to create another sandbox"] + }, + type: "nesting_too_deep" + }} end - defp workflow_error_reply(socket, {:error, :merge_failed}) do - {:reply, - {:error, - %{ - errors: %{base: ["Could not promote this workflow. Please try again."]}, - type: "merge_error" - }}, socket} + defp workflow_error_reply({:error, :merge_failed}) do + {:error, + %{ + errors: %{base: ["Could not promote this workflow. Please try again."]}, + type: "merge_error" + }} end - defp workflow_error_reply(socket, {:error, :not_a_sandbox}) do - {:reply, - {:error, - %{ - errors: %{ - base: ["This workflow is not in a sandbox and can't be promoted."] - }, - type: "invalid_state" - }}, socket} + defp workflow_error_reply({:error, :not_a_sandbox}) do + {:error, + %{ + errors: %{ + base: ["This workflow is not in a sandbox and can't be promoted."] + }, + type: "invalid_state" + }} end - defp workflow_error_reply(socket, {:error, :snapshot_failed}) do - {:reply, - {:error, - %{ - errors: %{base: ["An internal error occurred"]}, - type: "internal_error" - }}, socket} + defp workflow_error_reply({:error, :snapshot_failed}) do + {:error, + %{ + errors: %{base: ["An internal error occurred"]}, + type: "internal_error" + }} end - defp workflow_error_reply( - socket, - {:error, %Lightning.Extensions.Message{text: text}} - ) do - {:reply, - {:error, - %{ - errors: %{base: [text]}, - type: "limit_error" - }}, socket} + defp workflow_error_reply({:error, :adaptor_catalogue_unavailable}) do + {:error, + %{ + errors: %{ + base: ["The adaptor catalogue is still loading. Try again shortly."] + }, + type: "adaptor_catalogue_unavailable" + }} end - defp workflow_error_reply(socket, {:error, %Ecto.Changeset{} = changeset}) do - {:reply, - {:error, - %{ - errors: format_changeset_errors(changeset), - type: determine_error_type(changeset) - }}, socket} + defp workflow_error_reply({:error, :workflow_moved_on}) do + {:error, + %{ + errors: %{ + base: ["Someone else saved this workflow. Try the restore again."] + }, + type: "workflow_moved_on" + }} end - defp workflow_error_reply(socket, {:error, :workflow_moved_on}) do - {:reply, - {:error, - %{ - errors: %{ - base: ["Someone else saved this workflow. Try the restore again."] - }, - type: "workflow_moved_on" - }}, socket} + defp workflow_error_reply({:error, :version_not_found}) do + {:error, + %{ + errors: %{base: ["That version no longer exists"]}, + type: "version_not_found" + }} end - defp workflow_error_reply(socket, {:error, :version_not_found}) do - {:reply, - {:error, - %{ - errors: %{base: ["That version no longer exists"]}, - type: "version_not_found" - }}, socket} + defp workflow_error_reply({:error, %Lightning.Extensions.Message{text: text}}) do + {:error, + %{ + errors: %{base: [text]}, + type: "limit_error" + }} + end + + defp workflow_error_reply({:error, %Ecto.Changeset{} = changeset}) do + {:error, + %{ + errors: format_changeset_errors(changeset), + type: determine_error_type(changeset) + }} end # Last resort: never let an unexpected error reason crash the channel and drop # the user's socket. Log it and reply with a generic internal error. - defp workflow_error_reply(socket, error) do + defp workflow_error_reply(error) do Logger.warning("Unhandled workflow channel error: #{inspect(error)}") - {:reply, - {:error, - %{ - errors: %{base: ["An internal error occurred"]}, - type: "internal_error" - }}, socket} + {:error, + %{ + errors: %{base: ["An internal error occurred"]}, + type: "internal_error" + }} end - defp starting_dataclip_error(socket, message) do - {:reply, {:error, %{errors: %{base: [message]}, type: "validation_error"}}, - socket} + defp starting_dataclip_error(message) do + {:error, %{errors: %{base: [message]}, type: "validation_error"}} end defp format_changeset_errors(changeset) do @@ -1924,7 +1971,7 @@ defmodule LightningWeb.WorkflowChannel do {:reply, {:ok, %{lock_version: workflow.lock_version, workflow: workflow}}, socket} else - error -> workflow_error_reply(socket, error) + error -> {:reply, workflow_error_reply(error), socket} end end diff --git a/lib/lightning_web/controllers/adaptor_controller.ex b/lib/lightning_web/controllers/adaptor_controller.ex index 8dbd9d588d7..ceb5f15f5af 100644 --- a/lib/lightning_web/controllers/adaptor_controller.ex +++ b/lib/lightning_web/controllers/adaptor_controller.ex @@ -2,18 +2,19 @@ defmodule LightningWeb.AdaptorController do @moduledoc """ Bulk adaptor catalogue for the workflow editor's picker. - Route: `GET /adaptors/catalogue`. A matching `If-None-Match` returns - 304 before the catalogue query or payload build runs — see - `Lightning.Adaptors.catalogue_stamp/0`. + Route: `GET /adaptors/catalogue`. The stamp and the payload it describes + are cached together by `Lightning.Adaptors.Store.catalogue/1`, so a + matching `If-None-Match` answers 304 without touching Postgres, and a + miss on the ETag still serves an already-rendered payload. """ use LightningWeb, :controller alias Lightning.Adaptors - alias LightningWeb.AdaptorIconURL def index(conn, _params) do - etag = etag_for(Adaptors.catalogue_stamp()) + {stamp, entries} = Adaptors.catalogue_with_stamp() + etag = etag_for(stamp) conn = conn @@ -24,23 +25,10 @@ defmodule LightningWeb.AdaptorController do if get_req_header(conn, "if-none-match") == [etag] do send_resp(conn, 304, "") else - json(conn, %{data: Enum.map(Adaptors.catalogue(), &render_entry/1)}) + json(conn, %{data: entries}) end end - defp render_entry(entry) do - %{ - name: entry.name, - latest_version: entry.latest_version, - versions: entry.versions, - repository: entry.repository, - icon_urls: %{ - square: AdaptorIconURL.build(entry.name, entry, :square), - rectangle: AdaptorIconURL.build(entry.name, entry, :rectangle) - } - } - end - defp etag_for({nil, 0}), do: ~s("empty") defp etag_for({%DateTime{} = stamp, count}), diff --git a/lib/lightning_web/live/credential_live/credential_form_component.ex b/lib/lightning_web/live/credential_live/credential_form_component.ex index 6fe68b20293..7da547dd3ae 100644 --- a/lib/lightning_web/live/credential_live/credential_form_component.ex +++ b/lib/lightning_web/live/credential_live/credential_form_component.ex @@ -1193,8 +1193,8 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do |> Enum.sort_by(&String.downcase(elem(&1, 0)), :asc) end - defp adaptor_type_option(%{name: name} = meta) do - {name, name, AdaptorIconURL.build(name, meta, :square), nil} + defp adaptor_type_option(%Adaptors.Package{name: name} = pkg) do + {name, name, AdaptorIconURL.build(name, pkg, :square), nil} end defp list_users do diff --git a/lib/lightning_web/live/maintenance_live/index.ex b/lib/lightning_web/live/maintenance_live/index.ex index e00ab587fe3..ae9959fbe26 100644 --- a/lib/lightning_web/live/maintenance_live/index.ex +++ b/lib/lightning_web/live/maintenance_live/index.ex @@ -3,10 +3,11 @@ defmodule LightningWeb.MaintenanceLive.Index do Superuser-only maintenance page for on-demand operations against `Lightning.Adaptors`. - Exposes two actions: "Refresh Adaptor Registry" (`refresh_now/0`) and - "Refresh Adaptor Icons" (`refresh_icons/0`). Both are fire-and-forget: the - user gets a flash and the actual work happens asynchronously on the leader - node. + Exposes two actions: "Refresh Adaptor Registry" (`refresh/0`) and + "Refresh Adaptor Icons" (`refresh_icons/0`). Neither blocks the LiveView: + the registry refresh is fire-and-forget on the leader node, while the icon + refresh runs under `start_async` (the underlying call can take up to two + minutes) and flashes its result when it completes. """ use LightningWeb, :live_view @@ -33,7 +34,7 @@ defmodule LightningWeb.MaintenanceLive.Index do def handle_event("refresh_adaptors", _params, socket) do if superuser?(socket) do socket = - case Lightning.Adaptors.refresh_now() do + case Lightning.Adaptors.refresh() do :ok -> put_flash(socket, :info, "Adaptor refresh queued.") @@ -52,20 +53,12 @@ defmodule LightningWeb.MaintenanceLive.Index do def handle_event("refresh_icons", _params, socket) do if superuser?(socket) do - socket = - case Lightning.Adaptors.refresh_icons() do - {:ok, %{updated: updated, unchanged: unchanged}} -> - put_flash( - socket, - :info, - "Icon refresh complete — #{updated} updated, #{unchanged} unchanged." - ) - - {:error, reason} -> - put_flash(socket, :error, "Icon refresh failed: #{inspect(reason)}") - end - - {:noreply, socket} + {:noreply, + socket + |> put_flash(:info, "Icon refresh started.") + |> start_async(:refresh_icons, fn -> + Lightning.Adaptors.refresh_icons() + end)} else {:noreply, socket @@ -74,6 +67,29 @@ defmodule LightningWeb.MaintenanceLive.Index do end end + @impl true + def handle_async(:refresh_icons, {:ok, result}, socket) do + socket = + case result do + {:ok, %{updated: updated, unchanged: unchanged}} -> + put_flash( + socket, + :info, + "Icon refresh complete — #{updated} updated, #{unchanged} unchanged." + ) + + {:error, reason} -> + put_flash(socket, :error, "Icon refresh failed: #{inspect(reason)}") + end + + {:noreply, socket} + end + + def handle_async(:refresh_icons, {:exit, reason}, socket) do + {:noreply, + put_flash(socket, :error, "Icon refresh failed: #{inspect(reason)}")} + end + defp superuser?(socket) do Permissions.can?( Users, diff --git a/lib/mix/tasks/lightning.adaptors.dump.ex b/lib/mix/tasks/lightning.adaptors.dump.ex new file mode 100644 index 00000000000..5191592e6ad --- /dev/null +++ b/lib/mix/tasks/lightning.adaptors.dump.ex @@ -0,0 +1,80 @@ +defmodule Mix.Tasks.Lightning.Adaptors.Dump do + @shortdoc "Dump the adaptor catalogue to a JSON snapshot file" + + @moduledoc """ + Write this instance's adaptor catalogue to a JSON file, in the shape + `Lightning.Adaptors.Catalogue.upsert_adaptor/1` accepts — the shape + `mix lightning.adaptors.import` reads back. + + This is the catalogue-to-file leg of mirroring adaptors into an + airgapped environment: hydrate an online instance as usual, dump it + here, carry the file across, and import it on the offline instance. + `mix lightning.adaptors.snapshot` produces the same kind of file by + fetching npm directly, for when there is no populated catalogue to + dump from. + + ## Usage + + mix lightning.adaptors.dump --path snapshot.json + mix lightning.adaptors.dump --path snapshot.json --source local + + `--source` defaults to `npm`. + + Icons are left out. Their bytes live in the on-disk icon cache rather + than the catalogue, so a hash without them would buy the importing + instance nothing; it refetches instead. + """ + + use Mix.Task + + alias Lightning.Adaptors.Catalogue + + @adaptor_fields ~w(name source description homepage repository license + latest_version deprecated schema_data schema_sha256)a + + @version_fields ~w(version integrity tarball_url size_bytes dependencies + peer_dependencies published_at deprecated)a + + @impl Mix.Task + def run(argv) do + Mix.Task.run("app.start") + + {opts, _args} = + OptionParser.parse!(argv, strict: [path: :string, source: :string]) + + path = + opts[:path] || raise "Usage: mix lightning.adaptors.dump --path " + + source = parse_source(opts[:source]) + + records = + source + |> Catalogue.list_adaptors() + |> Enum.map(&dump_record(&1, source)) + + File.write!(path, Jason.encode_to_iodata!(records)) + + Mix.shell().info("Dumped #{length(records)} adaptor(s) to #{path}.") + end + + # ponytail: one version query per adaptor; join them if a catalogue ever + # grows past a few hundred rows. + defp dump_record(adaptor, source) do + versions = + adaptor.name + |> Catalogue.list_versions(source) + |> Enum.map(&(&1 |> Map.from_struct() |> Map.take(@version_fields))) + + adaptor + |> Map.from_struct() + |> Map.take(@adaptor_fields) + |> Map.put(:versions, versions) + end + + defp parse_source(nil), do: :npm + defp parse_source("npm"), do: :npm + defp parse_source("local"), do: :local + + defp parse_source(other), + do: raise("Unknown --source: #{other} (expected npm or local)") +end diff --git a/lib/mix/tasks/seed_adaptors_from_file.ex b/lib/mix/tasks/lightning.adaptors.import.ex similarity index 74% rename from lib/mix/tasks/seed_adaptors_from_file.ex rename to lib/mix/tasks/lightning.adaptors.import.ex index e6c4e80ccf0..bac35b9fdbf 100644 --- a/lib/mix/tasks/seed_adaptors_from_file.ex +++ b/lib/mix/tasks/lightning.adaptors.import.ex @@ -1,18 +1,18 @@ -defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFile do +defmodule Mix.Tasks.Lightning.Adaptors.Import do @shortdoc "Seed the adaptor catalogue from a local JSON snapshot" @moduledoc """ Populate the `adaptors` table from a JSON file, without reaching npm. The file is a JSON array of adaptor records in the shape - `Lightning.Adaptors.Repo.upsert_adaptor/1` accepts — the same shape - `mix lightning.download_adaptor_registry_cache` writes. + `Lightning.Adaptors.Catalogue.upsert_adaptor/1` accepts — the same shape + `mix lightning.adaptors.snapshot` writes. ## Usage - mix lightning.seed_adaptors_from_file --path snapshot.json - mix lightning.seed_adaptors_from_file --path snapshot.json --source local - mix lightning.seed_adaptors_from_file --path snapshot.json --replace + mix lightning.adaptors.import --path snapshot.json + mix lightning.adaptors.import --path snapshot.json --source local + mix lightning.adaptors.import --path snapshot.json --replace `--source` defaults to `npm`. `--replace` deletes every existing row for that source first, so the file becomes the source's entire contents @@ -42,7 +42,7 @@ defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFile do path = opts[:path] || - raise "Usage: mix lightning.seed_adaptors_from_file --path " + raise "Usage: mix lightning.adaptors.import --path " source = parse_source(opts[:source]) diff --git a/lib/mix/tasks/lightning.adaptors.refresh.ex b/lib/mix/tasks/lightning.adaptors.refresh.ex new file mode 100644 index 00000000000..b1f2a13faff --- /dev/null +++ b/lib/mix/tasks/lightning.adaptors.refresh.ex @@ -0,0 +1,83 @@ +defmodule Mix.Tasks.Lightning.Adaptors.Refresh do + @shortdoc "On-demand adaptor metadata refresh" + @moduledoc """ + Trigger an adaptor catalogue refresh from the command line. + + ## Usage + + mix lightning.adaptors.refresh + mix lightning.adaptors.refresh --name @openfn/language-http + + Without `--name`, starts a refresh cycle (or joins one already running) + and waits for it to finish. With `--name`, refetches that one adaptor. + + ## Exit codes + + * `0` - success + * `1` - package name not found + * `2` - any other error, including a listing that returned no adaptors + or a refresh that took longer than 10 minutes + """ + + use Mix.Task + + @await_timeout :timer.minutes(10) + + @impl Mix.Task + def run(argv) do + Mix.Task.run("app.start") + + {opts, _args} = OptionParser.parse!(argv, strict: [name: :string]) + + case opts[:name] do + nil -> refresh_all() + pkg -> refresh_one(pkg) + end + end + + defp refresh_all do + Mix.shell().info("Refreshing adaptors (waiting up to 10 minutes)...") + started = System.monotonic_time(:millisecond) + + case Lightning.Adaptors.refresh(Lightning.Adaptors, + await: true, + timeout: @await_timeout + ) do + {:ok, %{listed: 0}} -> + Mix.shell().error("Refresh completed but the source listed no adaptors.") + exit({:shutdown, 2}) + + {:ok, counts} -> + duration_s = div(System.monotonic_time(:millisecond) - started, 1000) + + Mix.shell().info( + "Refresh complete: listed #{counts.listed}, " <> + "fetched #{counts.fetched}, errors #{counts.errors} " <> + "(#{duration_s}s)." + ) + + {:error, :timeout} -> + Mix.shell().error("Refresh did not complete within 10 minutes.") + exit({:shutdown, 2}) + + {:error, reason} -> + Mix.shell().error("Refresh failed: #{inspect(reason)}") + exit({:shutdown, 2}) + end + end + + defp refresh_one(pkg) do + case Lightning.Adaptors.refresh_package(pkg) do + :ok -> + Mix.shell().info("Adaptors refreshed successfully.") + + {:error, :not_found} -> + Mix.shell().error("Package not found. Check the name and try again.") + exit({:shutdown, 1}) + + {:error, reason} -> + Mix.shell().error("Refresh failed: #{inspect(reason)}") + exit({:shutdown, 2}) + end + end +end diff --git a/lib/mix/tasks/download_adaptor_registry_cache.ex b/lib/mix/tasks/lightning.adaptors.snapshot.ex similarity index 73% rename from lib/mix/tasks/download_adaptor_registry_cache.ex rename to lib/mix/tasks/lightning.adaptors.snapshot.ex index 2ba175149ea..8b2e5cc5470 100644 --- a/lib/mix/tasks/download_adaptor_registry_cache.ex +++ b/lib/mix/tasks/lightning.adaptors.snapshot.ex @@ -1,13 +1,16 @@ -defmodule Mix.Tasks.Lightning.DownloadAdaptorRegistryCache do - @shortdoc "Downloads an adaptor catalogue snapshot for offline seeding" +defmodule Mix.Tasks.Lightning.Adaptors.Snapshot do + @shortdoc "Fetch an adaptor catalogue snapshot from npm for offline seeding" @moduledoc """ - Fetches every `@openfn/language-*` adaptor from npm via + Fetches every `@openfn/language-*` adaptor straight from npm via `Lightning.Adaptors.NPM` and writes the full records to a JSON file, in - the shape `Lightning.Adaptors.Repo.upsert_adaptor/1` accepts. + the shape `Lightning.Adaptors.Catalogue.upsert_adaptor/1` accepts. - The file this writes is what `mix lightning.seed_adaptors_from_file` - reads. + Nothing here touches the catalogue, so this works on an instance with + no database yet — it is the cold-start way to produce a snapshot. + `mix lightning.adaptors.dump` is the equivalent for an instance whose + catalogue is already populated. Either file can be read back by + `mix lightning.adaptors.import`. Use --path to specify the location """ diff --git a/lib/mix/tasks/lightning.refresh_adaptors.ex b/lib/mix/tasks/lightning.refresh_adaptors.ex deleted file mode 100644 index 74707fec9b7..00000000000 --- a/lib/mix/tasks/lightning.refresh_adaptors.ex +++ /dev/null @@ -1,59 +0,0 @@ -defmodule Mix.Tasks.Lightning.RefreshAdaptors do - @shortdoc "On-demand adaptor metadata refresh" - @moduledoc """ - Trigger an immediate adaptor refresh from the command line. - - Use cases: - - * Dev re-scan — force a re-scan after adding local adaptors - * Ops force-pull — pull latest metadata without waiting for the scheduler tick - - ## Usage - - mix lightning.refresh_adaptors - mix lightning.refresh_adaptors --name @openfn/language-http - - The first form calls `Lightning.Adaptors.refresh_now/0`, refreshing all - adaptors. The second form calls `Lightning.Adaptors.refresh_package/1` - to force a single-adaptor refresh, bypassing the ledger diff. - - Both forms block until completion. The Scheduler is wrapped in - `HighlanderPG` and registered globally, so the call routes through - Erlang distribution to whichever node currently holds the lease — the - CLI can be run from any node in the cluster. - - ## Exit codes - - * `0` — success - * `1` — package name not found (possible typo) - * `2` — other error - """ - - use Mix.Task - - @impl Mix.Task - def run(argv) do - Mix.Task.run("app.start") - - {opts, _args} = OptionParser.parse!(argv, strict: [name: :string]) - - result = - case opts[:name] do - nil -> Lightning.Adaptors.refresh_now() - pkg -> Lightning.Adaptors.refresh_package(pkg) - end - - case result do - :ok -> - Mix.shell().info("Adaptors refreshed successfully.") - - {:error, :not_found} -> - Mix.shell().error("Package not found. Check the name and try again.") - exit({:shutdown, 1}) - - {:error, reason} -> - Mix.shell().error("Refresh failed: #{inspect(reason)}") - exit({:shutdown, 2}) - end - end -end diff --git a/test/integration/web_and_worker_test.exs b/test/integration/web_and_worker_test.exs index e8a2c5de729..f3ed97af504 100644 --- a/test/integration/web_and_worker_test.exs +++ b/test/integration/web_and_worker_test.exs @@ -121,8 +121,6 @@ defmodule Lightning.WebAndWorkerTest do @tag :integration @tag timeout: 20_000 test "the whole thing", %{conn: conn, user: user} do - # Seed a concrete version so `PackageName.to_wire/1` resolves - # `@latest` without hitting the live NPM registry. Lightning.AdaptorTestHelpers.seed_adaptor_package( "@openfn/language-http", "3.1.12" diff --git a/test/lightning/adaptor_service_test.exs b/test/lightning/adaptor_service_test.exs new file mode 100644 index 00000000000..41c51d35ae3 --- /dev/null +++ b/test/lightning/adaptor_service_test.exs @@ -0,0 +1,110 @@ +defmodule Lightning.AdaptorServiceTest do + @moduledoc """ + Covers `AdaptorService.known?/1` gating `install/2` on + `Lightning.Adaptors.fetch_adaptor/1`: an empty catalogue waits for one + load, and a name the loaded catalogue lacks refuses the install. + """ + + # set_mox_global: the load runs in a Task owned by the production + # Scheduler. + use Lightning.DataCase, async: false + + import Mox + + alias Lightning.Adaptors.Catalogue + alias Lightning.AdaptorService + + setup :set_mox_global + setup :verify_on_exit! + + setup do + Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() + stub(Lightning.AdaptorService.RepoMock, :list_local, fn _path -> [] end) + + {:ok, agent} = + AdaptorService.start_link( + adaptors_path: "test/tmp/adaptors", + repo: Lightning.AdaptorService.RepoMock + ) + + {:ok, agent: agent} + end + + describe "install/2 refuses a package the catalogue doesn't recognise" do + test "empty catalogue: loads once, then refuses without calling repo.install/2", + %{agent: agent} do + test_pid = self() + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> + send(test_pid, :listed) + {:ok, []} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + assert {:error, :adaptor_not_permitted} = + AdaptorService.install(agent, "@openfn/language-http") + + assert_received :listed + end + + test "populated catalogue without this package: refuses", %{agent: agent} do + {:ok, _} = + Catalogue.upsert_adaptor(%{ + name: "@openfn/language-common", + source: :npm, + latest_version: "1.0.0", + versions: [ + %{ + version: "1.0.0", + integrity: "sha512-abc", + tarball_url: "https://example.com/x-1.0.0.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + ] + }) + + assert {:error, :adaptor_not_permitted} = + AdaptorService.install(agent, "@openfn/language-http") + end + end + + describe "install/2 allows a package the catalogue recognises" do + test "populated catalogue with this package: proceeds to repo.install/2", + %{agent: agent} do + {:ok, _} = + Catalogue.upsert_adaptor(%{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + versions: [ + %{ + version: "1.0.0", + integrity: "sha512-abc", + tarball_url: "https://example.com/x-1.0.0.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + ] + }) + + expect(Lightning.AdaptorService.RepoMock, :install, fn _adaptor, _dir -> + {"", 0} + end) + + stub(Lightning.AdaptorService.RepoMock, :list_local, fn _path -> [] end) + + assert {:ok, _installed} = + AdaptorService.install(agent, "@openfn/language-http") + end + end +end diff --git a/test/lightning/adaptors/repo_adaptor_test.exs b/test/lightning/adaptors/catalogue_adaptor_test.exs similarity index 90% rename from test/lightning/adaptors/repo_adaptor_test.exs rename to test/lightning/adaptors/catalogue_adaptor_test.exs index e77868fa6cd..bae63b6dee5 100644 --- a/test/lightning/adaptors/repo_adaptor_test.exs +++ b/test/lightning/adaptors/catalogue_adaptor_test.exs @@ -1,7 +1,7 @@ -defmodule Lightning.Adaptors.Repo.AdaptorTest do +defmodule Lightning.Adaptors.Catalogue.AdaptorTest do use ExUnit.Case, async: true - alias Lightning.Adaptors.Repo.Adaptor + alias Lightning.Adaptors.Catalogue.Adaptor @valid_attrs %{ name: "@openfn/language-http", @@ -74,6 +74,30 @@ defmodule Lightning.Adaptors.Repo.AdaptorTest do end end + describe "changeset/2 — :name format" do + test "accepts scoped and unscoped names" do + for name <- ["@openfn/language-http", "language-http", "lodash.merge"] do + assert Adaptor.changeset(%Adaptor{}, %{@valid_attrs | name: name}).valid?, + "expected #{inspect(name)} to be valid" + end + end + + test "rejects names with shell metacharacters, whitespace, or a version suffix" do + for name <- [ + "@openfn/language-http@1.0.0", + "evil; rm -rf /", + "name with spaces", + "line\nbreak", + "$(whoami)" + ] do + changeset = Adaptor.changeset(%Adaptor{}, %{@valid_attrs | name: name}) + + refute changeset.valid?, "expected #{inspect(name)} to be invalid" + assert "has invalid format" in errors_on(changeset, :name) + end + end + end + describe "changeset/2 — :source Ecto.Enum cast" do test "round-trips atom :npm" do changeset = diff --git a/test/lightning/adaptors/repo_adaptor_version_test.exs b/test/lightning/adaptors/catalogue_adaptor_version_test.exs similarity index 96% rename from test/lightning/adaptors/repo_adaptor_version_test.exs rename to test/lightning/adaptors/catalogue_adaptor_version_test.exs index b9e9a52167a..713b0a525c8 100644 --- a/test/lightning/adaptors/repo_adaptor_version_test.exs +++ b/test/lightning/adaptors/catalogue_adaptor_version_test.exs @@ -1,7 +1,7 @@ -defmodule Lightning.Adaptors.Repo.AdaptorVersionTest do +defmodule Lightning.Adaptors.Catalogue.AdaptorVersionTest do use ExUnit.Case, async: true - alias Lightning.Adaptors.Repo.AdaptorVersion + alias Lightning.Adaptors.Catalogue.AdaptorVersion @adaptor_id Ecto.UUID.generate() @@ -147,7 +147,7 @@ defmodule Lightning.Adaptors.Repo.AdaptorVersionTest do describe "schema" do test "belongs_to :adaptor uses binary_id" do assoc = AdaptorVersion.__schema__(:association, :adaptor) - assert assoc.related == Lightning.Adaptors.Repo.Adaptor + assert assoc.related == Lightning.Adaptors.Catalogue.Adaptor assert assoc.owner_key == :adaptor_id end diff --git a/test/lightning/adaptors/repo_catalogue_test.exs b/test/lightning/adaptors/catalogue_listing_test.exs similarity index 69% rename from test/lightning/adaptors/repo_catalogue_test.exs rename to test/lightning/adaptors/catalogue_listing_test.exs index 14b5a4f451e..d9d86d3861a 100644 --- a/test/lightning/adaptors/repo_catalogue_test.exs +++ b/test/lightning/adaptors/catalogue_listing_test.exs @@ -1,12 +1,12 @@ -defmodule Lightning.Adaptors.RepoCatalogueTest do +defmodule Lightning.Adaptors.CatalogueListingTest do use Lightning.DataCase, async: true - alias Lightning.Adaptors.Repo, as: AdaptorRepo + alias Lightning.Adaptors.Catalogue describe "catalogue/1" do test "returns name, latest_version, repository, icon fields, and full version list" do {:ok, _adaptor} = - AdaptorRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-http", source: :npm, latest_version: "2.0.0", @@ -21,7 +21,7 @@ defmodule Lightning.Adaptors.RepoCatalogueTest do ] }) - assert [entry] = AdaptorRepo.catalogue(:npm) + assert [entry] = Catalogue.catalogue(:npm) assert entry.name == "@openfn/language-http" assert entry.latest_version == "2.0.0" @@ -33,67 +33,87 @@ defmodule Lightning.Adaptors.RepoCatalogueTest do test "is source-scoped" do {:ok, _} = - AdaptorRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-http", source: :npm, latest_version: "1.0.0", versions: [version_record("1.0.0")] }) - assert AdaptorRepo.catalogue(:local) == [] + assert Catalogue.catalogue(:local) == [] end test "returns an empty list for an adaptor with no versions" do {:ok, _} = - AdaptorRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-http", source: :npm, latest_version: "1.0.0", versions: [] }) - assert [%{versions: []}] = AdaptorRepo.catalogue(:npm) + assert [%{versions: []}] = Catalogue.catalogue(:npm) + end + + test "omits the excluded adaptors" do + for name <- [ + "@openfn/language-devtools", + "@openfn/language-template", + "@openfn/language-fhir-jembi", + "@openfn/language-collections", + "@openfn/language-http" + ] do + {:ok, _} = + Catalogue.upsert_adaptor(%{ + name: name, + source: :npm, + latest_version: "1.0.0", + versions: [version_record("1.0.0")] + }) + end + + assert [%{name: "@openfn/language-http"}] = Catalogue.catalogue(:npm) end end describe "catalogue_stamp/1" do test "returns a nil timestamp and zero count when the source has no rows" do - assert AdaptorRepo.catalogue_stamp(:npm) == {nil, 0} + assert Catalogue.catalogue_stamp(:npm) == {nil, 0} end test "reflects the adaptor row's updated_at when there are no versions" do {:ok, adaptor} = - AdaptorRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-http", source: :npm, latest_version: "1.0.0", versions: [] }) - assert {stamp, 0} = AdaptorRepo.catalogue_stamp(:npm) + assert {stamp, 0} = Catalogue.catalogue_stamp(:npm) assert stamp == adaptor.updated_at end test "advances when a new version is published, without touching the adaptor row" do {:ok, _adaptor} = - AdaptorRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-http", source: :npm, latest_version: "1.0.0", versions: [version_record("1.0.0")] }) - {before_stamp, _count} = AdaptorRepo.catalogue_stamp(:npm) + {before_stamp, _count} = Catalogue.catalogue_stamp(:npm) {:ok, _adaptor} = - AdaptorRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-http", source: :npm, latest_version: "1.0.0", versions: [version_record("1.0.0"), version_record("1.1.0")] }) - {after_stamp, count} = AdaptorRepo.catalogue_stamp(:npm) + {after_stamp, count} = Catalogue.catalogue_stamp(:npm) assert DateTime.after?(after_stamp, before_stamp) assert count == 2 @@ -101,7 +121,7 @@ defmodule Lightning.Adaptors.RepoCatalogueTest do test "changes when a version is removed from an adaptor that doesn't hold the current max" do {:ok, _b} = - AdaptorRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-b", source: :npm, latest_version: "1.0.0", @@ -109,24 +129,24 @@ defmodule Lightning.Adaptors.RepoCatalogueTest do }) {:ok, _a} = - AdaptorRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-a", source: :npm, latest_version: "1.0.0", versions: [version_record("1.0.0")] }) - before_stamp = AdaptorRepo.catalogue_stamp(:npm) + before_stamp = Catalogue.catalogue_stamp(:npm) {:ok, _b} = - AdaptorRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-b", source: :npm, latest_version: "1.0.0", versions: [] }) - assert AdaptorRepo.catalogue_stamp(:npm) != before_stamp + assert Catalogue.catalogue_stamp(:npm) != before_stamp end end diff --git a/test/lightning/adaptors/repo_test.exs b/test/lightning/adaptors/catalogue_test.exs similarity index 62% rename from test/lightning/adaptors/repo_test.exs rename to test/lightning/adaptors/catalogue_test.exs index ee41cf514db..53d97448581 100644 --- a/test/lightning/adaptors/repo_test.exs +++ b/test/lightning/adaptors/catalogue_test.exs @@ -1,9 +1,9 @@ -defmodule Lightning.Adaptors.RepoTest do +defmodule Lightning.Adaptors.CatalogueTest do use Lightning.DataCase, async: true - alias Lightning.Adaptors.Repo, as: AdaptorRepo - alias Lightning.Adaptors.Repo.Adaptor - alias Lightning.Adaptors.Repo.AdaptorVersion + alias Lightning.Adaptors.Catalogue + alias Lightning.Adaptors.Catalogue.Adaptor + alias Lightning.Adaptors.Catalogue.AdaptorVersion describe "upsert_adaptor/1 — initial insert" do test "inserts the adaptor row and its versions in one transaction" do @@ -12,7 +12,7 @@ defmodule Lightning.Adaptors.RepoTest do versions: [version_record("1.0.0"), version_record("1.1.0")] ) - assert {:ok, %Adaptor{} = adaptor} = AdaptorRepo.upsert_adaptor(record) + assert {:ok, %Adaptor{} = adaptor} = Catalogue.upsert_adaptor(record) assert adaptor.name == "@openfn/language-http" assert adaptor.source == :npm @@ -20,7 +20,7 @@ defmodule Lightning.Adaptors.RepoTest do assert %DateTime{} = adaptor.checked_at assert %DateTime{} = adaptor.updated_at - versions = AdaptorRepo.list_versions(adaptor.name, :npm) + versions = Catalogue.list_versions(adaptor.name, :npm) assert versions |> Enum.map(& &1.version) |> Enum.sort() == [ "1.0.0", @@ -33,18 +33,51 @@ defmodule Lightning.Adaptors.RepoTest do test "accepts a record with no versions" do record = adaptor_record(versions: []) - assert {:ok, %Adaptor{} = adaptor} = AdaptorRepo.upsert_adaptor(record) - assert AdaptorRepo.list_versions(adaptor.name, :npm) == [] + assert {:ok, %Adaptor{} = adaptor} = Catalogue.upsert_adaptor(record) + assert Catalogue.list_versions(adaptor.name, :npm) == [] + end + + test "accepts a fully string-keyed record, as read from a JSON snapshot" do + record = %{ + "name" => "@openfn/language-salesforce", + "source" => "npm", + "latest_version" => "3.0.0", + "description" => "Salesforce adaptor", + "checked_at" => DateTime.utc_now(), + "versions" => [ + %{ + "version" => "3.0.0", + "integrity" => "sha512-3.0.0", + "tarball_url" => "https://example.com/x/-/x-3.0.0.tgz", + "size_bytes" => 1024, + "dependencies" => %{"axios" => "^1.0.0"}, + "peer_dependencies" => %{}, + "deprecated" => false + } + ] + } + + assert {:ok, %Adaptor{} = adaptor} = Catalogue.upsert_adaptor(record) + + assert adaptor.name == "@openfn/language-salesforce" + assert adaptor.source == :npm + assert adaptor.latest_version == "3.0.0" + assert adaptor.description == "Salesforce adaptor" + + assert [version] = Catalogue.list_versions(adaptor.name, :npm) + assert version.version == "3.0.0" + assert version.adaptor_id == adaptor.id + assert version.dependencies == %{"axios" => "^1.0.0"} end end describe "upsert_adaptor/1 — idempotency (§12.2)" do test "re-upserting the same record advances :checked_at but not :updated_at" do - {:ok, first} = AdaptorRepo.upsert_adaptor(adaptor_record()) + {:ok, first} = Catalogue.upsert_adaptor(adaptor_record()) Process.sleep(5) - {:ok, second} = AdaptorRepo.upsert_adaptor(adaptor_record()) + {:ok, second} = Catalogue.upsert_adaptor(adaptor_record()) assert second.id == first.id assert second.updated_at == first.updated_at @@ -54,24 +87,24 @@ defmodule Lightning.Adaptors.RepoTest do describe "upsert_adaptor/1 — diff-aware :updated_at" do test "changing :latest_version bumps :updated_at" do - {:ok, first} = AdaptorRepo.upsert_adaptor(adaptor_record()) + {:ok, first} = Catalogue.upsert_adaptor(adaptor_record()) Process.sleep(5) {:ok, second} = - AdaptorRepo.upsert_adaptor(adaptor_record(latest_version: "1.1.0")) + Catalogue.upsert_adaptor(adaptor_record(latest_version: "1.1.0")) assert second.latest_version == "1.1.0" assert DateTime.compare(second.updated_at, first.updated_at) == :gt end test "changing :description bumps :updated_at" do - {:ok, first} = AdaptorRepo.upsert_adaptor(adaptor_record()) + {:ok, first} = Catalogue.upsert_adaptor(adaptor_record()) Process.sleep(5) {:ok, second} = - AdaptorRepo.upsert_adaptor(adaptor_record(description: "new copy")) + Catalogue.upsert_adaptor(adaptor_record(description: "new copy")) assert second.description == "new copy" assert DateTime.compare(second.updated_at, first.updated_at) == :gt @@ -81,18 +114,18 @@ defmodule Lightning.Adaptors.RepoTest do describe "upsert_adaptor/1 — version row replacement (§12.2)" do test "replaces version rows atomically" do {:ok, _adaptor} = - AdaptorRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( versions: [version_record("1.0.0"), version_record("1.1.0")] ) ) - assert AdaptorRepo.list_versions("@openfn/language-http", :npm) + assert Catalogue.list_versions("@openfn/language-http", :npm) |> Enum.map(& &1.version) |> Enum.sort() == ["1.0.0", "1.1.0"] {:ok, _adaptor} = - AdaptorRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( versions: [ version_record("1.1.0"), @@ -102,26 +135,26 @@ defmodule Lightning.Adaptors.RepoTest do ) ) - assert AdaptorRepo.list_versions("@openfn/language-http", :npm) + assert Catalogue.list_versions("@openfn/language-http", :npm) |> Enum.map(& &1.version) |> Enum.sort() == ["1.1.0", "1.2.0", "2.0.0"] end test "shrinking the version set drops the missing rows" do {:ok, _} = - AdaptorRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( versions: [version_record("1.0.0"), version_record("1.1.0")] ) ) {:ok, _} = - AdaptorRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record(versions: [version_record("1.1.0")]) ) assert [%AdaptorVersion{version: "1.1.0"}] = - AdaptorRepo.list_versions("@openfn/language-http", :npm) + Catalogue.list_versions("@openfn/language-http", :npm) end test "persists version-row payload fields verbatim" do @@ -136,10 +169,10 @@ defmodule Lightning.Adaptors.RepoTest do deprecated: false } - {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(versions: [payload])) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(versions: [payload])) assert [version] = - AdaptorRepo.list_versions("@openfn/language-http", :npm) + Catalogue.list_versions("@openfn/language-http", :npm) assert version.integrity == payload.integrity assert version.tarball_url == payload.tarball_url @@ -154,10 +187,10 @@ defmodule Lightning.Adaptors.RepoTest do describe "upsert_adaptor/1 — source isolation" do test "the same name can coexist across sources" do {:ok, npm_row} = - AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) + Catalogue.upsert_adaptor(adaptor_record(source: :npm)) {:ok, local_row} = - AdaptorRepo.upsert_adaptor(adaptor_record(source: :local)) + Catalogue.upsert_adaptor(adaptor_record(source: :local)) assert npm_row.id != local_row.id assert npm_row.source == :npm @@ -167,35 +200,35 @@ defmodule Lightning.Adaptors.RepoTest do describe "touch_checked_at/2 (§12.2)" do test "advances :checked_at and leaves :updated_at alone" do - {:ok, original} = AdaptorRepo.upsert_adaptor(adaptor_record()) + {:ok, original} = Catalogue.upsert_adaptor(adaptor_record()) Process.sleep(5) - assert :ok = AdaptorRepo.touch_checked_at(original.name, :npm) + assert :ok = Catalogue.touch_checked_at(original.name, :npm) - reloaded = AdaptorRepo.get_adaptor(original.name, :npm) + reloaded = Catalogue.get_adaptor(original.name, :npm) assert DateTime.compare(reloaded.checked_at, original.checked_at) == :gt assert reloaded.updated_at == original.updated_at end test "is a no-op for an unknown (name, source) — does not require loading the row" do - assert :ok = AdaptorRepo.touch_checked_at("@openfn/never-existed", :npm) - assert AdaptorRepo.get_adaptor("@openfn/never-existed", :npm) == nil + assert :ok = Catalogue.touch_checked_at("@openfn/never-existed", :npm) + assert Catalogue.get_adaptor("@openfn/never-existed", :npm) == nil end test "is source-scoped" do {:ok, npm_row} = - AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) + Catalogue.upsert_adaptor(adaptor_record(source: :npm)) {:ok, local_row} = - AdaptorRepo.upsert_adaptor(adaptor_record(source: :local)) + Catalogue.upsert_adaptor(adaptor_record(source: :local)) Process.sleep(5) - :ok = AdaptorRepo.touch_checked_at(npm_row.name, :npm) + :ok = Catalogue.touch_checked_at(npm_row.name, :npm) - reloaded_npm = AdaptorRepo.get_adaptor(npm_row.name, :npm) - reloaded_local = AdaptorRepo.get_adaptor(local_row.name, :local) + reloaded_npm = Catalogue.get_adaptor(npm_row.name, :npm) + reloaded_local = Catalogue.get_adaptor(local_row.name, :local) assert DateTime.compare(reloaded_npm.checked_at, npm_row.checked_at) == :gt assert reloaded_local.checked_at == local_row.checked_at @@ -209,45 +242,45 @@ defmodule Lightning.Adaptors.RepoTest do seed_adaptor(name: "@openfn/a", checked_at: DateTime.add(base, -300)) seed_adaptor(name: "@openfn/b", checked_at: newest) - assert AdaptorRepo.max_checked_at(:npm) == newest + assert Catalogue.max_checked_at(:npm) == newest end test "returns nil when the source has no rows" do seed_adaptor(name: "@openfn/a", source: :npm) - assert AdaptorRepo.max_checked_at(:local) == nil + assert Catalogue.max_checked_at(:local) == nil end end describe "get_adaptor/2" do test "returns the matching adaptor" do - {:ok, inserted} = AdaptorRepo.upsert_adaptor(adaptor_record()) - reloaded = AdaptorRepo.get_adaptor(inserted.name, :npm) + {:ok, inserted} = Catalogue.upsert_adaptor(adaptor_record()) + reloaded = Catalogue.get_adaptor(inserted.name, :npm) assert %Adaptor{} = reloaded assert reloaded.id == inserted.id end test "returns nil when not found" do - assert AdaptorRepo.get_adaptor("@openfn/never-existed", :npm) == nil + assert Catalogue.get_adaptor("@openfn/never-existed", :npm) == nil end test "is source-scoped" do - {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) - assert AdaptorRepo.get_adaptor("@openfn/language-http", :local) == nil + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(source: :npm)) + assert Catalogue.get_adaptor("@openfn/language-http", :local) == nil end end describe "list_package_metas/1" do test "returns the lean projection without heavy JSONB columns" do {:ok, _} = - AdaptorRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( description: "yep", schema_data: %{"big" => "json", "nested" => %{"more" => "stuff"}} ) ) - assert [meta] = AdaptorRepo.list_package_metas(:npm) + assert [meta] = Catalogue.list_package_metas(:npm) assert meta.name == "@openfn/language-http" assert meta.latest_version == "1.0.0" @@ -260,33 +293,48 @@ defmodule Lightning.Adaptors.RepoTest do end test "filters by source" do - {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) - {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :local)) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(source: :npm)) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(source: :local)) + + assert [%{name: "@openfn/language-http"}] = + Catalogue.list_package_metas(:npm) assert [%{name: "@openfn/language-http"}] = - AdaptorRepo.list_package_metas(:npm) + Catalogue.list_package_metas(:local) + end + + test "omits the excluded adaptors" do + for name <- [ + "@openfn/language-devtools", + "@openfn/language-template", + "@openfn/language-fhir-jembi", + "@openfn/language-collections", + "@openfn/language-http" + ] do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(name: name)) + end assert [%{name: "@openfn/language-http"}] = - AdaptorRepo.list_package_metas(:local) + Catalogue.list_package_metas(:npm) end end describe "list_adaptors/1" do test "returns full structs filtered by source" do - {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :npm)) - {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(source: :local)) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(source: :npm)) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(source: :local)) - assert [%Adaptor{source: :npm}] = AdaptorRepo.list_adaptors(:npm) - assert [%Adaptor{source: :local}] = AdaptorRepo.list_adaptors(:local) + assert [%Adaptor{source: :npm}] = Catalogue.list_adaptors(:npm) + assert [%Adaptor{source: :local}] = Catalogue.list_adaptors(:local) end end describe "list_missing_icons/1" do test "returns rows where either icon shape sha256 is nil" do - {:ok, _} = AdaptorRepo.upsert_adaptor(adaptor_record(name: "@openfn/a")) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/a")) {:ok, _} = - AdaptorRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: "@openfn/b", icon_square_ext: "png", @@ -295,7 +343,7 @@ defmodule Lightning.Adaptors.RepoTest do ) {:ok, _} = - AdaptorRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: "@openfn/c", icon_square_ext: "png", @@ -306,7 +354,7 @@ defmodule Lightning.Adaptors.RepoTest do ) names = - AdaptorRepo.list_missing_icons(:npm) + Catalogue.list_missing_icons(:npm) |> Enum.map(& &1.name) |> Enum.sort() @@ -315,28 +363,28 @@ defmodule Lightning.Adaptors.RepoTest do test "is source-scoped" do {:ok, _} = - AdaptorRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record(name: "@openfn/x", source: :local) ) - assert AdaptorRepo.list_missing_icons(:npm) == [] - assert [%{name: "@openfn/x"}] = AdaptorRepo.list_missing_icons(:local) + assert Catalogue.list_missing_icons(:npm) == [] + assert [%{name: "@openfn/x"}] = Catalogue.list_missing_icons(:local) end end describe "update_icons/3" do test "writes only icon columns and bumps :updated_at" do - {:ok, before} = AdaptorRepo.upsert_adaptor(adaptor_record()) + {:ok, before} = Catalogue.upsert_adaptor(adaptor_record()) Process.sleep(5) sha = :crypto.hash(:sha256, "PNG") assert {1, nil} = - AdaptorRepo.update_icons(before.name, :npm, %{ + Catalogue.update_icons(before.name, :npm, %{ icon_square_ext: "png", icon_square_sha256: sha }) - after_row = AdaptorRepo.get_adaptor(before.name, :npm) + after_row = Catalogue.get_adaptor(before.name, :npm) assert after_row.icon_square_ext == "png" assert after_row.icon_square_sha256 == sha @@ -345,31 +393,31 @@ defmodule Lightning.Adaptors.RepoTest do end test "ignores keys outside the icon set" do - {:ok, before} = AdaptorRepo.upsert_adaptor(adaptor_record()) + {:ok, before} = Catalogue.upsert_adaptor(adaptor_record()) - AdaptorRepo.update_icons(before.name, :npm, %{ + Catalogue.update_icons(before.name, :npm, %{ latest_version: "9.9.9", icon_square_ext: "svg", icon_square_sha256: :crypto.hash(:sha256, "S") }) - after_row = AdaptorRepo.get_adaptor(before.name, :npm) + after_row = Catalogue.get_adaptor(before.name, :npm) assert after_row.latest_version == before.latest_version assert after_row.icon_square_ext == "svg" end test "writes icon etag columns alongside ext/sha256" do - {:ok, before} = AdaptorRepo.upsert_adaptor(adaptor_record()) + {:ok, before} = Catalogue.upsert_adaptor(adaptor_record()) sha = :crypto.hash(:sha256, "PNG") assert {1, nil} = - AdaptorRepo.update_icons(before.name, :npm, %{ + Catalogue.update_icons(before.name, :npm, %{ icon_square_ext: "png", icon_square_sha256: sha, icon_square_etag: ~s("abc123") }) - after_row = AdaptorRepo.get_adaptor(before.name, :npm) + after_row = Catalogue.get_adaptor(before.name, :npm) assert %{ icon_square_ext: "png", @@ -381,18 +429,18 @@ defmodule Lightning.Adaptors.RepoTest do test "leaves version rows untouched" do {:ok, before} = - AdaptorRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( versions: [version_record("1.0.0"), version_record("2.0.0")] ) ) - AdaptorRepo.update_icons(before.name, :npm, %{ + Catalogue.update_icons(before.name, :npm, %{ icon_square_ext: "png", icon_square_sha256: :crypto.hash(:sha256, "P") }) - versions = AdaptorRepo.list_versions(before.name, :npm) + versions = Catalogue.list_versions(before.name, :npm) assert length(versions) == 2 end end diff --git a/test/lightning/adaptors/config_test.exs b/test/lightning/adaptors/config_test.exs index 7a3af965c44..382a2dd23fb 100644 --- a/test/lightning/adaptors/config_test.exs +++ b/test/lightning/adaptors/config_test.exs @@ -5,19 +5,33 @@ defmodule Lightning.Adaptors.ConfigTest do @parent_key Lightning.Adaptors - describe "current_source/0" do - test "returns :local when strategy is Lightning.Adaptors.Local" do - put_parent(:strategy, Lightning.Adaptors.Local) + describe "source_for/1" do + test "returns :local for Lightning.Adaptors.Local" do + assert Config.source_for(Lightning.Adaptors.Local) == :local + end + + test "returns :npm for Lightning.Adaptors.NPM" do + assert Config.source_for(Lightning.Adaptors.NPM) == :npm + end + + test "raises for an unmapped strategy module with no declared source" do + assert_raise ArgumentError, ~r/has no catalogue source/, fn -> + Config.source_for(SomeOther.Strategy) + end + end + + test "uses the :source declared under a third-party strategy's own key" do + put_strategy_opts(SomeOther.Strategy, source: :local) - assert Config.current_source() == :local + assert Config.source_for(SomeOther.Strategy) == :local end - test "returns :npm for any other strategy module" do - put_parent(:strategy, Lightning.Adaptors.NPM) - assert Config.current_source() == :npm + test "raises when the declared :source isn't :npm or :local" do + put_strategy_opts(SomeOther.Strategy, source: :bogus) - put_parent(:strategy, SomeOther.Strategy) - assert Config.current_source() == :npm + assert_raise ArgumentError, ~r/has no catalogue source/, fn -> + Config.source_for(SomeOther.Strategy) + end end end diff --git a/test/lightning/adaptors/invalidator_test.exs b/test/lightning/adaptors/invalidator_test.exs index f0a096b8265..6417e1f28d9 100644 --- a/test/lightning/adaptors/invalidator_test.exs +++ b/test/lightning/adaptors/invalidator_test.exs @@ -25,7 +25,7 @@ defmodule Lightning.Adaptors.InvalidatorTest do end describe "handle_info/2 - {:changed, name, source}" do - test "evicts all four matching cache keys on broadcast", %{ + test "evicts all five matching cache keys on broadcast", %{ sup: sup, cache: cache, inv_name: inv_name @@ -44,6 +44,7 @@ defmodule Lightning.Adaptors.InvalidatorTest do ) Cachex.put!(cache, {:packages, source}, {:ok, [%{name: name}]}) + Cachex.put!(cache, {:catalogue, source}, {:ok, {{nil, 0}, []}}) Phoenix.PubSub.broadcast!( Lightning.PubSub, @@ -57,6 +58,7 @@ defmodule Lightning.Adaptors.InvalidatorTest do assert {:ok, nil} = Cachex.get(cache, {:versions, name, source}) assert {:ok, nil} = Cachex.get(cache, {:icon_meta, name, source}) assert {:ok, nil} = Cachex.get(cache, {:packages, source}) + assert {:ok, nil} = Cachex.get(cache, {:catalogue, source}) end test "does not evict name-scoped keys for a different adaptor", %{ diff --git a/test/lightning/adaptors/node_monitor_test.exs b/test/lightning/adaptors/node_monitor_test.exs index e4b0132340e..1e8fa6b47c3 100644 --- a/test/lightning/adaptors/node_monitor_test.exs +++ b/test/lightning/adaptors/node_monitor_test.exs @@ -3,7 +3,7 @@ defmodule Lightning.Adaptors.NodeMonitorTest do import Mox - alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor setup :verify_on_exit! @@ -52,7 +52,7 @@ defmodule Lightning.Adaptors.NodeMonitorTest do :unreachable end) - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) source = AdaptorsSupervisor.source(sup) send(nm_name, {:nodeup, :node@host, %{node_type: :visible}}) @@ -80,7 +80,7 @@ defmodule Lightning.Adaptors.NodeMonitorTest do {:ok, %{"kept" => true}} ) - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) send(nm_name, {:nodeup, :node@host, %{node_type: :visible}}) :sys.get_state(nm_name) @@ -94,7 +94,7 @@ defmodule Lightning.Adaptors.NodeMonitorTest do cache: cache, nm_name: nm_name } do - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) send(nm_name, {:nodeup, :node@host, %{node_type: :visible}}) :sys.get_state(nm_name) diff --git a/test/lightning/adaptors/package_name_test.exs b/test/lightning/adaptors/package_name_test.exs index 7f01aacc2b6..19958ab9140 100644 --- a/test/lightning/adaptors/package_name_test.exs +++ b/test/lightning/adaptors/package_name_test.exs @@ -1,7 +1,5 @@ defmodule Lightning.Adaptors.PackageNameTest do - use Lightning.DataCase, async: false - - import Lightning.Factories + use ExUnit.Case, async: true alias Lightning.Adaptors.PackageName @@ -39,7 +37,7 @@ defmodule Lightning.Adaptors.PackageNameTest do end end - describe "to_wire/1" do + describe "to_wire/2" do test "passes through concrete semver unchanged" do assert PackageName.to_wire("@openfn/language-common@1.6.2") == "@openfn/language-common@1.6.2" @@ -50,24 +48,36 @@ defmodule Lightning.Adaptors.PackageNameTest do end test "preserves @local literal regardless of source" do - assert PackageName.to_wire("@openfn/language-common@local") == - "@openfn/language-common@local" + assert PackageName.to_wire("@openfn/language-common@local", + source: :npm + ) == "@openfn/language-common@local" end - test "resolves @latest to the concrete latest_version from Adaptors.Repo" do - insert(:adaptor, - name: "@openfn/language-common", - source: :npm, - latest_version: "9.9.9" - ) + test "substitutes the caller-resolved version for @latest" do + assert PackageName.to_wire("@openfn/language-common@latest", + source: :npm, + latest: "9.9.9" + ) == "@openfn/language-common@9.9.9" + end - assert PackageName.to_wire("@openfn/language-common@latest") == - "@openfn/language-common@9.9.9" + test "requires a resolved version for @latest under a non-local source" do + assert_raise KeyError, fn -> + PackageName.to_wire("@openfn/never-existed@latest", source: :npm) + end end - test "falls back to @latest literal when adaptor is unknown" do - assert PackageName.to_wire("@openfn/never-existed@latest") == - "@openfn/never-existed@latest" + test "forces @local under a :local source, whatever the spec says" do + assert PackageName.to_wire("@openfn/language-common@1.6.2", + source: :local + ) == "@openfn/language-common@local" + + assert PackageName.to_wire("@openfn/language-common@latest", + source: :local + ) == "@openfn/language-common@local" + + assert PackageName.to_wire("@openfn/language-common", + source: :local + ) == "@openfn/language-common@local" end end end diff --git a/test/lightning/adaptors/readiness_test.exs b/test/lightning/adaptors/readiness_test.exs new file mode 100644 index 00000000000..45a58d46cde --- /dev/null +++ b/test/lightning/adaptors/readiness_test.exs @@ -0,0 +1,304 @@ +defmodule Lightning.Adaptors.ReadinessTest do + @moduledoc """ + `fetch_adaptor/2` and `ensure_loaded/1` against an isolated supervisor: + a populated catalogue never contacts the Scheduler, an empty one waits + for exactly one coalesced load, and every failure mode of that wait maps + to its error atom. + """ + + use Lightning.DataCase, async: true + + import Eventually + import Mox + + alias Lightning.Adaptors + alias Lightning.Adaptors.Catalogue + alias Lightning.Adaptors.Scheduler + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + setup :verify_on_exit! + + # Without the built-in Scheduler, unarranged contact fails with + # `:unavailable` rather than a Mox error in a process the test does not own. + setup do + sup = :"readiness_test_#{System.unique_integer([:positive])}" + + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + ) + + :ok = + Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) + + {:ok, sup: sup} + end + + defp adaptor_record(overrides \\ []) do + overrides = Map.new(overrides) + + %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + description: nil, + homepage: nil, + repository: nil, + license: nil, + deprecated: false, + schema_data: nil, + schema_sha256: nil, + versions: [ + %{ + version: "1.0.0", + integrity: "sha512-abc", + tarball_url: "https://example.com/x-1.0.0.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + ] + } + |> Map.merge(overrides) + end + + # Tasks the Scheduler spawns inherit its `$callers`, so allowing the + # Scheduler covers them. + defp start_scheduler(sup) do + pid = + start_supervised!({ + Scheduler, + name: AdaptorsSupervisor.global_scheduler_name(sup), + sup: sup, + lock_key: AdaptorsSupervisor.lock_key(sup), + cache: AdaptorsSupervisor.cache_name(sup), + tasks: AdaptorsSupervisor.tasks_name(sup), + source_topic: AdaptorsSupervisor.source_topic(sup) + }) + + Ecto.Adapters.SQL.Sandbox.allow(Lightning.Repo, self(), pid) + Mox.allow(Lightning.Adaptors.StrategyMock, self(), pid) + pid + end + + defp expect_one_load(records) do + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> + {:ok, Enum.map(records, &Map.take(&1, [:name, :latest_version]))} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + length(records), + fn name -> {:ok, Enum.find(records, &(&1.name == name))} end + ) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + end + + describe "fetch_adaptor/2 on a populated catalogue" do + test "serves a known adaptor from the cache without contacting the Scheduler", + %{sup: sup} do + {:ok, _} = + Catalogue.upsert_adaptor(adaptor_record(latest_version: "2.0.0")) + + assert {:ok, %Adaptors.Package{latest_version: "2.0.0", source: :npm}} = + Adaptors.fetch_adaptor(sup, "@openfn/language-http") + + source = AdaptorsSupervisor.source(sup) + + assert {:ok, {:ok, [%{name: "@openfn/language-http"}]}} = + Cachex.get( + AdaptorsSupervisor.cache_name(sup), + {:packages, source} + ) + + assert {:ok, %Adaptors.Package{latest_version: "2.0.0"}} = + Adaptors.fetch_adaptor(sup, "@openfn/language-http") + end + + test "finds a row the cached list does not have yet", %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + assert {:ok, _} = Adaptors.fetch_adaptor(sup, "@openfn/language-http") + + {:ok, _} = + Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/language-x")) + + assert {:ok, %Adaptors.Package{name: "@openfn/language-x"}} = + Adaptors.fetch_adaptor(sup, "@openfn/language-x") + end + + test "returns {:error, :not_found} for an absent name without contacting the Scheduler", + %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert {:error, :not_found} = + Adaptors.fetch_adaptor(sup, "@openfn/never-existed") + + assert Adaptors.get_adaptor(sup, "@openfn/never-existed") == nil + end + + test "answers for the given supervisor's source, not the default one" do + local_sup = :"readiness_local_#{System.unique_integer([:positive])}" + + start_supervised!( + Supervisor.child_spec( + {AdaptorsSupervisor, + name: local_sup, strategy: Lightning.Adaptors.Local}, + id: local_sup + ) + ) + + :ok = + Supervisor.terminate_child( + local_sup, + AdaptorsSupervisor.highlander_name(local_sup) + ) + + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert {:error, :unavailable} = + Adaptors.fetch_adaptor(local_sup, "@openfn/language-http") + + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(source: :local)) + + assert {:ok, %Adaptors.Package{source: :local}} = + Adaptors.fetch_adaptor(local_sup, "@openfn/language-http") + end + end + + describe "fetch_adaptor/2 on an empty catalogue" do + test "waits for one coalesced load shared by concurrent callers", %{sup: sup} do + test_pid = self() + record = adaptor_record() + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> + send(test_pid, :listed) + Process.sleep(50) + {:ok, [Map.take(record, [:name, :latest_version])]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn _ -> + {:ok, record} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + start_scheduler(sup) + + task_a = + Task.async(fn -> + Adaptors.fetch_adaptor(sup, "@openfn/language-http") + end) + + assert_receive :listed, 2000 + + task_b = + Task.async(fn -> + Adaptors.fetch_adaptor(sup, "@openfn/language-http") + end) + + assert {:ok, %Adaptors.Package{latest_version: "1.0.0"}} = + Task.await(task_a, 5_000) + + assert {:ok, %Adaptors.Package{latest_version: "1.0.0"}} = + Task.await(task_b, 5_000) + end + + test "returns {:error, :not_found} when the load does not list the name", + %{sup: sup} do + expect_one_load([adaptor_record()]) + start_scheduler(sup) + + assert {:error, :not_found} = + Adaptors.fetch_adaptor(sup, "@openfn/never-existed") + end + + test "returns {:error, :not_ready} when the load leaves the catalogue empty", + %{sup: sup} do + expect_one_load([]) + start_scheduler(sup) + + assert {:error, :not_ready} = + Adaptors.fetch_adaptor(sup, "@openfn/language-http") + end + + test "returns {:error, :unavailable} when no Scheduler is reachable", + %{sup: sup} do + assert {:error, :unavailable} = + Adaptors.fetch_adaptor(sup, "@openfn/language-http") + end + + test "returns {:error, :timeout} on a slow load, and the late result still lands", + %{sup: sup} do + Mimic.stub(Lightning.Adaptors.Config, :first_load_timeout, fn -> 50 end) + + record = adaptor_record() + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> + Process.sleep(300) + {:ok, [Map.take(record, [:name, :latest_version])]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn _ -> + {:ok, record} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + pid = start_scheduler(sup) + + assert {:error, :timeout} = + Adaptors.fetch_adaptor(sup, "@openfn/language-http") + + assert Process.alive?(pid) + + assert_eventually( + match?( + {:ok, %Adaptors.Package{}}, + Adaptors.fetch_adaptor(sup, "@openfn/language-http") + ), + 2000 + ) + + assert Process.alive?(pid) + end + end + + describe "ensure_loaded/1" do + test "returns :ok immediately when rows exist, without contacting the Scheduler", + %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + assert :ok = Adaptors.ensure_loaded(sup) + end + + test "loads once, then answers from data even with the Scheduler gone", + %{sup: sup} do + expect_one_load([adaptor_record()]) + pid = start_scheduler(sup) + + assert :ok = Adaptors.ensure_loaded(sup) + + stop_supervised!(Scheduler) + refute Process.alive?(pid) + + assert :ok = Adaptors.ensure_loaded(sup) + end + + test "maps a failed wait to its error atom", %{sup: sup} do + assert {:error, :unavailable} = Adaptors.ensure_loaded(sup) + + expect_one_load([]) + start_scheduler(sup) + + assert {:error, :not_ready} = Adaptors.ensure_loaded(sup) + end + end +end diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index ae807d27ef8..77b64c1a29b 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -4,9 +4,10 @@ defmodule Lightning.Adaptors.SchedulerTest do # 2. set_mox_global is safe only when tests run serially use Lightning.DataCase, async: false + import Eventually import Mox - alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Scheduler alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor @@ -202,7 +203,7 @@ defmodule Lightning.Adaptors.SchedulerTest do source = AdaptorsSupervisor.source(sup) source_topic = AdaptorsSupervisor.source_topic(sup) - {:ok, existing} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, existing} = Catalogue.upsert_adaptor(adaptor_record()) checked_at_before = existing.checked_at expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> @@ -227,7 +228,7 @@ defmodule Lightning.Adaptors.SchedulerTest do # Allow the spawned task to complete before asserting no broadcast. refute_receive {:changed, _, _}, 200 - row = AdaptorsRepo.get_adaptor("@openfn/language-http", source) + row = Catalogue.get_adaptor("@openfn/language-http", source) assert DateTime.compare(row.checked_at, checked_at_before) == :gt assert row.latest_version == "1.0.0" end @@ -237,7 +238,7 @@ defmodule Lightning.Adaptors.SchedulerTest do source = AdaptorsSupervisor.source(sup) source_topic = AdaptorsSupervisor.source_topic(sup) - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> send(test_pid, :list_adaptors_called) @@ -263,7 +264,7 @@ defmodule Lightning.Adaptors.SchedulerTest do assert_receive :list_adaptors_called, 2000 assert_receive {:changed, "@openfn/language-http", ^source}, 2000 - row = AdaptorsRepo.get_adaptor("@openfn/language-http", source) + row = Catalogue.get_adaptor("@openfn/language-http", source) assert row.latest_version == "2.0.0" end @@ -288,7 +289,7 @@ defmodule Lightning.Adaptors.SchedulerTest do start_scheduler(sup) assert_receive {:changed, "@openfn/language-new", ^source}, 2000 - assert AdaptorsRepo.get_adaptor("@openfn/language-new", source) != nil + assert Catalogue.get_adaptor("@openfn/language-new", source) != nil end test "list_adaptors error: no DB writes, no broadcasts", %{sup: sup} do @@ -361,12 +362,69 @@ defmodule Lightning.Adaptors.SchedulerTest do assert_receive :tick_ran, 2000 sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + # Let the init tick's cycle clear so refresh_now starts a new one + # instead of coalescing. + {:global, gname} = sched_name + pid = :global.whereis_name(gname) + assert_eventually(:sys.get_state(pid).refresh == nil, 2000) + assert :ok = Scheduler.refresh_now(sched_name) assert_receive :tick_ran, 2000 end end + # Rows are seeded before the Scheduler starts so no init tick fires and + # the Mox counts stay exact. + describe "await_refresh/2 result" do + test "carries the cycle's counts on success, with per-adaptor failures as errors", + %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> + {:ok, + [ + %{name: "@openfn/language-http", latest_version: "1.0.0"}, + %{name: "@openfn/language-new", latest_version: "2.0.0"}, + %{name: "@openfn/language-bad", latest_version: "1.0.0"} + ]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 2, fn + "@openfn/language-new" -> + {:ok, + adaptor_record( + name: "@openfn/language-new", + latest_version: "2.0.0" + )} + + "@openfn/language-bad" -> + {:error, :upstream_5xx} + end) + + start_scheduler(sup) + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:ok, %{listed: 3, changed: 1, fetched: 1, errors: 1}} = + Scheduler.await_refresh(sched_name, 5_000) + end + + test "returns the upstream listing failure to waiters", %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> + {:error, :upstream_down} + end) + + start_scheduler(sup) + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:error, :upstream_down} = + Scheduler.await_refresh(sched_name, 5_000) + end + end + describe "icons pipeline" do test "writes icon bytes to disk and stamps ext+sha256 on the row", %{ sup: sup @@ -399,7 +457,7 @@ defmodule Lightning.Adaptors.SchedulerTest do assert_receive {:changed, "@openfn/language-http", ^source}, 2000 - row = AdaptorsRepo.get_adaptor("@openfn/language-http", source) + row = Catalogue.get_adaptor("@openfn/language-http", source) assert row.icon_square_ext == "png" assert row.icon_square_sha256 == sha assert row.icon_rectangle_ext == nil @@ -439,7 +497,7 @@ defmodule Lightning.Adaptors.SchedulerTest do assert_receive {:changed, "@openfn/language-http", ^source}, 2000 - row = AdaptorsRepo.get_adaptor("@openfn/language-http", source) + row = Catalogue.get_adaptor("@openfn/language-http", source) assert row != nil assert row.icon_square_ext == nil assert row.icon_square_sha256 == nil @@ -452,9 +510,7 @@ defmodule Lightning.Adaptors.SchedulerTest do # (so the diff path will :touch instead of :fetch). Without # self-heal this row would stay iconless forever. {:ok, _} = - AdaptorsRepo.upsert_adaptor( - adaptor_record(name: "@openfn/language-stale") - ) + Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/language-stale")) bytes = "STALE_ICON" sha = :crypto.hash(:sha256, bytes) @@ -487,7 +543,7 @@ defmodule Lightning.Adaptors.SchedulerTest do assert_receive {:changed, "@openfn/language-stale", ^source}, 2000 - row = AdaptorsRepo.get_adaptor("@openfn/language-stale", source) + row = Catalogue.get_adaptor("@openfn/language-stale", source) assert row.icon_square_ext == "png" assert row.icon_square_sha256 == sha @@ -534,7 +590,7 @@ defmodule Lightning.Adaptors.SchedulerTest do assert :ok = Scheduler.refresh_package(sched_name, "@openfn/language-http") assert_receive {:changed, "@openfn/language-http", ^source}, 2000 - assert AdaptorsRepo.get_adaptor("@openfn/language-http", source) != nil + assert Catalogue.get_adaptor("@openfn/language-http", source) != nil end test "returns error tuple when fetch_adaptor fails", %{sup: sup} do @@ -597,20 +653,67 @@ defmodule Lightning.Adaptors.SchedulerTest do end describe "refresh_icons/1" do + test "an in-flight icon refresh does not block the Scheduler loop", %{ + sup: sup + } do + test_pid = self() + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, []} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, 1, fn _opts -> + send(test_pid, {:icons_started, self()}) + + receive do + :finish_icons -> {:ok, %{}} + end + end) + + start_scheduler(sup, interval: 0) + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + icons_task = Task.async(fn -> Scheduler.refresh_icons(sched_name) end) + assert_receive {:icons_started, icons_pid}, 2000 + + # Completes while the icon fetch is parked, so that fetch is not on + # the GenServer loop. + assert {:ok, %{listed: 0}} = Scheduler.await_refresh(sched_name, 5_000) + + send(icons_pid, :finish_icons) + + assert {:ok, %{updated: 0, unchanged: 0}} = + Task.await(icons_task, 5_000) + end + + test "a crash in the icon refresh task replies an error instead of killing the Scheduler", + %{sup: sup} do + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, 1, fn _opts -> + raise "boom" + end) + + start_scheduler(sup, interval: 0) + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:error, {:refresh_failed, _reason}} = + Scheduler.refresh_icons(sched_name) + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> {:ok, []} end) + assert {:ok, _counts} = Scheduler.await_refresh(sched_name, 2_000) + end + test "updates rows whose shape sha256 differs from the fetched icon", %{ sup: sup } do source = AdaptorsSupervisor.source(sup) {:ok, _} = - AdaptorsRepo.upsert_adaptor( - adaptor_record(name: "@openfn/language-empty") - ) + Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/language-empty")) old_sha = :crypto.hash(:sha256, "OLD") {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: "@openfn/language-current", icon_square_ext: "png", @@ -642,11 +745,11 @@ defmodule Lightning.Adaptors.SchedulerTest do assert {:ok, %{updated: 2, unchanged: 0}} = Scheduler.refresh_icons(sched_name) - empty = AdaptorsRepo.get_adaptor("@openfn/language-empty", source) + empty = Catalogue.get_adaptor("@openfn/language-empty", source) assert empty.icon_square_ext == "png" assert empty.icon_square_sha256 == new_sha - current = AdaptorsRepo.get_adaptor("@openfn/language-current", source) + current = Catalogue.get_adaptor("@openfn/language-current", source) assert current.icon_square_sha256 == new_sha for name <- ["@openfn/language-empty", "@openfn/language-current"] do @@ -662,7 +765,7 @@ defmodule Lightning.Adaptors.SchedulerTest do etag = ~s("prior-etag-1") {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: "@openfn/language-same", icon_square_ext: "png", @@ -686,7 +789,7 @@ defmodule Lightning.Adaptors.SchedulerTest do assert {:ok, %{updated: 0, unchanged: 1}} = Scheduler.refresh_icons(sched_name) - row = AdaptorsRepo.get_adaptor("@openfn/language-same", source) + row = Catalogue.get_adaptor("@openfn/language-same", source) assert row.icon_square_sha256 == sha assert row.icon_square_etag == etag end @@ -700,7 +803,7 @@ defmodule Lightning.Adaptors.SchedulerTest do new_etag = ~s("etag-B") {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: "@openfn/language-rotated", icon_square_ext: "png", @@ -730,7 +833,7 @@ defmodule Lightning.Adaptors.SchedulerTest do assert {:ok, %{updated: 1, unchanged: 0}} = Scheduler.refresh_icons(sched_name) - row = AdaptorsRepo.get_adaptor("@openfn/language-rotated", source) + row = Catalogue.get_adaptor("@openfn/language-rotated", source) assert row.icon_square_sha256 == new_sha assert row.icon_square_etag == new_etag @@ -756,7 +859,7 @@ defmodule Lightning.Adaptors.SchedulerTest do # Two rows: one returns 200 with etag: nil (NPM-style), the other # returns 200 with the :etag key entirely absent (Local-style). {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: "@openfn/language-nil-etag", icon_square_ext: "png", @@ -766,7 +869,7 @@ defmodule Lightning.Adaptors.SchedulerTest do ) {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: "@openfn/language-no-etag-key", icon_square_ext: "png", @@ -799,11 +902,11 @@ defmodule Lightning.Adaptors.SchedulerTest do assert {:ok, %{updated: 2, unchanged: 0}} = Scheduler.refresh_icons(sched_name) - row_a = AdaptorsRepo.get_adaptor("@openfn/language-nil-etag", source) + row_a = Catalogue.get_adaptor("@openfn/language-nil-etag", source) assert row_a.icon_square_sha256 == new_sha_a assert row_a.icon_square_etag == prior_etag - row_b = AdaptorsRepo.get_adaptor("@openfn/language-no-etag-key", source) + row_b = Catalogue.get_adaptor("@openfn/language-no-etag-key", source) assert row_b.icon_square_sha256 == new_sha_b assert row_b.icon_square_etag == prior_etag @@ -826,7 +929,7 @@ defmodule Lightning.Adaptors.SchedulerTest do current_etag = ~s("etag-current") {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: "@openfn/language-stale-etag", icon_square_ext: "png", @@ -836,7 +939,7 @@ defmodule Lightning.Adaptors.SchedulerTest do ) {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: "@openfn/language-current-etag", icon_square_ext: "png", @@ -871,12 +974,12 @@ defmodule Lightning.Adaptors.SchedulerTest do assert {:ok, %{updated: 1, unchanged: 1}} = Scheduler.refresh_icons(sched_name) - stale_row = AdaptorsRepo.get_adaptor("@openfn/language-stale-etag", source) + stale_row = Catalogue.get_adaptor("@openfn/language-stale-etag", source) assert stale_row.icon_square_sha256 == stale_new_sha assert stale_row.icon_square_etag == stale_new_etag current_row = - AdaptorsRepo.get_adaptor("@openfn/language-current-etag", source) + Catalogue.get_adaptor("@openfn/language-current-etag", source) assert current_row.icon_square_sha256 == current_sha assert current_row.icon_square_etag == current_etag diff --git a/test/lightning/adaptors/seed_test.exs b/test/lightning/adaptors/seed_test.exs new file mode 100644 index 00000000000..38bf84e5142 --- /dev/null +++ b/test/lightning/adaptors/seed_test.exs @@ -0,0 +1,132 @@ +defmodule Lightning.Adaptors.SeedTest do + use Lightning.DataCase, async: true + + alias Lightning.Adaptors.Seed + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + @moduletag :tmp_dir + + setup %{tmp_dir: tmp_dir} do + sup = :"seed_test_#{System.unique_integer([:positive])}" + + start_supervised!( + {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + ) + + :ok = + Phoenix.PubSub.subscribe( + Lightning.PubSub, + AdaptorsSupervisor.source_topic(sup) + ) + + {:ok, + sup: sup, + source: AdaptorsSupervisor.source(sup), + cache: AdaptorsSupervisor.cache_name(sup), + tmp_dir: tmp_dir} + end + + defp write_snapshot(tmp_dir, records) do + path = + Path.join(tmp_dir, "snapshot-#{System.unique_integer([:positive])}.json") + + File.write!(path, Jason.encode_to_iodata!(records)) + path + end + + defp record(name, opts \\ []) do + %{ + name: name, + latest_version: Keyword.get(opts, :latest_version, "1.0.0"), + versions: Keyword.get(opts, :versions, []) + } + end + + describe "seed_from_file/2" do + test "broadcasts {:changed, name, source} for every seeded name", %{ + sup: sup, + source: source, + tmp_dir: tmp_dir + } do + path = + write_snapshot(tmp_dir, [ + record("@openfn/language-http"), + record("@openfn/language-dhis2") + ]) + + assert {:ok, 2} = Seed.seed_from_file(path, sup: sup) + + assert_receive {:changed, "@openfn/language-http", ^source} + assert_receive {:changed, "@openfn/language-dhis2", ^source} + end + + test "replace: true also broadcasts for names the wipe removed", %{ + sup: sup, + source: source, + tmp_dir: tmp_dir + } do + insert(:adaptor, name: "@openfn/language-stale", source: source) + + path = write_snapshot(tmp_dir, [record("@openfn/language-http")]) + + assert {:ok, 1} = Seed.seed_from_file(path, sup: sup, replace: true) + + assert_receive {:changed, "@openfn/language-http", ^source} + assert_receive {:changed, "@openfn/language-stale", ^source} + end + + test "replace: true broadcasts for a removed name even when the catalogue listing excludes it", + %{ + sup: sup, + source: source, + tmp_dir: tmp_dir + } do + insert(:adaptor, name: "@openfn/language-collections", source: source) + + path = write_snapshot(tmp_dir, [record("@openfn/language-http")]) + + assert {:ok, 1} = Seed.seed_from_file(path, sup: sup, replace: true) + + assert_receive {:changed, "@openfn/language-http", ^source} + assert_receive {:changed, "@openfn/language-collections", ^source} + end + + test "a rolled-back seed broadcasts nothing", %{ + sup: sup, + tmp_dir: tmp_dir + } do + insert(:adaptor, name: "@openfn/language-stale", source: :npm) + + path = + write_snapshot(tmp_dir, [ + record("@openfn/language-http"), + # No `latest_version`, which the Adaptor changeset requires, so + # this raises inside the transaction. + %{name: "@openfn/language-broken", versions: []} + ]) + + assert_raise ArgumentError, fn -> + Seed.seed_from_file(path, sup: sup, replace: true) + end + + refute_receive {:changed, _, _} + end + + test "the instance's Invalidator drops the cached catalogue", %{ + sup: sup, + source: source, + cache: cache, + tmp_dir: tmp_dir + } do + Cachex.put!(cache, {:catalogue, source}, {:ok, {{nil, 0}, []}}) + + path = write_snapshot(tmp_dir, [record("@openfn/language-http")]) + + assert {:ok, 1} = Seed.seed_from_file(path, sup: sup) + + :sys.get_state(AdaptorsSupervisor.invalidator_name(sup)) + + assert {:ok, nil} = Cachex.get(cache, {:catalogue, source}) + end + end +end diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index 02f565aa867..dad3b61da74 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -3,9 +3,10 @@ defmodule Lightning.Adaptors.StoreTest do import Mox - alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Store alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + alias LightningWeb.AdaptorIconURL setup :verify_on_exit! @@ -45,7 +46,7 @@ defmodule Lightning.Adaptors.StoreTest do assert {:ok, ~s({"type":"object"})} = Store.schema(sup, "@openfn/language-http") - assert AdaptorsRepo.get_adaptor("@openfn/language-http", source) == nil + assert Catalogue.get_adaptor("@openfn/language-http", source) == nil end test "cache miss + DB hit returns DB value without calling Strategy", %{ @@ -56,7 +57,7 @@ defmodule Lightning.Adaptors.StoreTest do end) {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record(schema_data: ~s({"type":"object"})) ) @@ -64,13 +65,15 @@ defmodule Lightning.Adaptors.StoreTest do Store.schema(sup, "@openfn/language-http") end - test "cache miss + DB miss calls Strategy once, upserts to DB, caches result", + test "known adaptor with missing schema calls Strategy once, upserts to DB, caches result", %{ sup: sup, cache: cache } do source = AdaptorsSupervisor.source(sup) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) + expect( Lightning.Adaptors.StrategyMock, :fetch_adaptor, @@ -84,16 +87,33 @@ defmodule Lightning.Adaptors.StoreTest do Store.schema(sup, "@openfn/language-http") assert %{schema_data: ~s({"type":"object"})} = - AdaptorsRepo.get_adaptor("@openfn/language-http", source) + Catalogue.get_adaptor("@openfn/language-http", source) assert {:ok, {:ok, ~s({"type":"object"})}} = Cachex.get(cache, {:schema, "@openfn/language-http", source}) end + test "unknown adaptor returns {:error, :not_found} without calling Strategy or minting a row", + %{sup: sup, cache: cache} do + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + source = AdaptorsSupervisor.source(sup) + + assert {:error, :not_found} = Store.schema(sup, "@openfn/never-existed") + assert Catalogue.get_adaptor("@openfn/never-existed", source) == nil + + assert {:ok, nil} = + Cachex.get(cache, {:schema, "@openfn/never-existed", source}) + end + test "three concurrent calls coalesce to one Strategy call", %{sup: sup} do name = "@openfn/language-http" test_pid = self() + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> # Brief sleep so the other two tasks queue up in Cachex's courier. Process.sleep(30) @@ -128,6 +148,8 @@ defmodule Lightning.Adaptors.StoreTest do } do source = AdaptorsSupervisor.source(sup) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn _ -> {:error, :upstream_error} end) @@ -160,7 +182,7 @@ defmodule Lightning.Adaptors.StoreTest do ordered_body = ~s({"a":1,"z":2,"m":3}) {:ok, _} = - AdaptorsRepo.upsert_adaptor(adaptor_record(schema_data: ordered_body)) + Catalogue.upsert_adaptor(adaptor_record(schema_data: ordered_body)) assert {:ok, ^ordered_body} = Store.schema(sup, "@openfn/language-http") end @@ -174,7 +196,7 @@ defmodule Lightning.Adaptors.StoreTest do end) {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( versions: [version_record("1.0.0"), version_record("1.1.0")] ) @@ -186,12 +208,15 @@ defmodule Lightning.Adaptors.StoreTest do assert Enum.all?(versions, &Map.has_key?(&1, :deprecated)) end - test "cache miss + DB miss calls Strategy and caches projected versions", %{ - sup: sup, - cache: cache - } do + test "known adaptor with no version rows calls Strategy and caches projected versions", + %{ + sup: sup, + cache: cache + } do source = AdaptorsSupervisor.source(sup) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(versions: [])) + expect( Lightning.Adaptors.StrategyMock, :fetch_adaptor, @@ -211,6 +236,45 @@ defmodule Lightning.Adaptors.StoreTest do Cachex.get(cache, {:versions, "@openfn/language-http", source}) assert length(cached_versions) == 2 + + for cached <- cached_versions do + assert Map.keys(cached) |> Enum.sort() == + [:deprecated, :integrity, :published_at, :size_bytes, :version] + end + end + + test "a fetched record whose name differs from the requested name is refused", + %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(versions: [])) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> + {:ok, adaptor_record(name: "@openfn/language-impostor")} + end + ) + + assert {:error, {:name_mismatch, "@openfn/language-impostor"}} = + Store.versions(sup, "@openfn/language-http") + + assert Catalogue.get_adaptor("@openfn/language-impostor", source) == nil + assert Catalogue.list_versions("@openfn/language-http", source) == [] + end + + test "unknown adaptor returns {:error, :not_found} without calling Strategy or minting a row", + %{sup: sup} do + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + source = AdaptorsSupervisor.source(sup) + + assert {:error, :not_found} = Store.versions(sup, "@openfn/never-existed") + assert Catalogue.get_adaptor("@openfn/never-existed", source) == nil end end @@ -233,7 +297,7 @@ defmodule Lightning.Adaptors.StoreTest do sup: sup, cache: cache } do - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) assert {:ok, [pkg]} = Store.packages(sup) assert pkg.name == "@openfn/language-http" @@ -241,6 +305,129 @@ defmodule Lightning.Adaptors.StoreTest do source = AdaptorsSupervisor.source(sup) assert {:ok, {:ok, [_]}} = Cachex.get(cache, {:packages, source}) end + + test "the catalogue's excluded adaptors never reach the cache", %{ + sup: sup, + cache: cache + } do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record(name: "@openfn/language-collections") + ) + + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert {:ok, [%{name: "@openfn/language-http"}]} = Store.packages(sup) + + source = AdaptorsSupervisor.source(sup) + + assert {:ok, {:ok, [%{name: "@openfn/language-http"}]}} = + Cachex.get(cache, {:packages, source}) + end + end + + describe "catalogue/1" do + test "empty DB returns an empty payload but does NOT cache it", %{ + sup: sup, + cache: cache + } do + assert {:ok, {{nil, 0}, []}} = Store.catalogue(sup) + + source = AdaptorsSupervisor.source(sup) + assert {:ok, nil} = Cachex.get(cache, {:catalogue, source}) + end + + test "caches the stamp and the rendered payload as one entry", %{ + sup: sup, + cache: cache + } do + square_sha = :crypto.hash(:sha256, "square") + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + repository: "https://github.com/openfn/language-http", + icon_square_ext: "png", + icon_square_sha256: square_sha + ) + ) + + assert {:ok, {{%DateTime{}, 1}, [entry]}} = Store.catalogue(sup) + + assert entry == %{ + name: "@openfn/language-http", + latest_version: "1.0.0", + versions: ["1.0.0"], + repository: "https://github.com/openfn/language-http", + icon_urls: %{ + square: + AdaptorIconURL.build( + "@openfn/language-http", + %{icon_square_ext: "png", icon_square_sha256: square_sha}, + :square + ), + rectangle: nil + } + } + + source = AdaptorsSupervisor.source(sup) + + assert {:ok, {:ok, {{%DateTime{}, 1}, [^entry]}}} = + Cachex.get(cache, {:catalogue, source}) + end + + test "a second call is served from cache, without re-reading the projection", + %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert {:ok, first} = Store.catalogue(sup) + + {:ok, _} = + Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/language-late")) + + assert {:ok, ^first} = Store.catalogue(sup) + end + + test "local-source entries render latest_version and versions as \"local\", not the real on-disk semver" do + local_sup = :"store_test_local_#{System.unique_integer([:positive])}" + + start_supervised!( + Supervisor.child_spec( + {AdaptorsSupervisor, + name: local_sup, strategy: Lightning.Adaptors.Local}, + id: local_sup + ) + ) + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + source: :local, + latest_version: "1.4.2", + versions: [version_record("1.4.2")] + ) + ) + + assert {:ok, {_stamp, [entry]}} = Store.catalogue(local_sup) + + assert entry.latest_version == "local" + assert entry.versions == ["local"] + end + + test "npm-source entries keep the real semver untouched", %{sup: sup} do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + latest_version: "1.4.2", + versions: [version_record("1.4.2")] + ) + ) + + assert {:ok, {_stamp, [entry]}} = Store.catalogue(sup) + + assert entry.latest_version == "1.4.2" + assert entry.versions == ["1.4.2"] + end end describe "icon/3" do @@ -258,7 +445,7 @@ defmodule Lightning.Adaptors.StoreTest do name = unique_name("disk-hit") {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: name, icon_square_ext: "png", @@ -291,7 +478,7 @@ defmodule Lightning.Adaptors.StoreTest do name = unique_name("disk-miss") {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: name, icon_square_ext: "png", @@ -320,7 +507,7 @@ defmodule Lightning.Adaptors.StoreTest do name = unique_name("err") {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: name, icon_square_ext: "png", @@ -345,7 +532,7 @@ defmodule Lightning.Adaptors.StoreTest do name = unique_name("coalesce") {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: name, icon_square_ext: "png", @@ -381,7 +568,7 @@ defmodule Lightning.Adaptors.StoreTest do name_b = unique_name("parB") {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: name_a, icon_square_ext: "png", @@ -390,7 +577,7 @@ defmodule Lightning.Adaptors.StoreTest do ) {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( name: name_b, icon_square_ext: "png", @@ -454,7 +641,7 @@ defmodule Lightning.Adaptors.StoreTest do cache: cache } do {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( icon_square_ext: "svg", icon_square_sha256: :crypto.hash(:sha256, "fake-svg-bytes") @@ -474,11 +661,11 @@ defmodule Lightning.Adaptors.StoreTest do end describe "warm_from_repo/1" do - test "populates {:packages, source} and {:icon_meta, name, source} keys", %{ + test "populates the {:packages}, {:icon_meta} and {:catalogue} keys", %{ sup: sup, cache: cache } do - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) assert :ok = Store.warm_from_repo(sup) @@ -492,6 +679,22 @@ defmodule Lightning.Adaptors.StoreTest do assert Map.has_key?(icon_meta, :icon_square_ext) assert Map.has_key?(icon_meta, :icon_rectangle_ext) + + assert {:ok, {:ok, {{%DateTime{}, 1}, [entry]}}} = + Cachex.get(cache, {:catalogue, source}) + + assert entry.name == "@openfn/language-http" + assert entry.icon_urls == %{square: nil, rectangle: nil} + end + + test "leaves {:catalogue, source} uncached when the catalogue is empty", %{ + sup: sup, + cache: cache + } do + assert :ok = Store.warm_from_repo(sup) + + source = AdaptorsSupervisor.source(sup) + assert {:ok, nil} = Cachex.get(cache, {:catalogue, source}) end test "overwrites existing keys without clearing unrelated ones", %{ @@ -506,7 +709,7 @@ defmodule Lightning.Adaptors.StoreTest do {:ok, %{"kept" => true}} ) - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) assert :ok = Store.warm_from_repo(sup) assert {:ok, {:ok, %{"kept" => true}}} = diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index 737a73d26a1..2100067ce8b 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -1,10 +1,11 @@ defmodule Lightning.AdaptorsTest do use Lightning.DataCase, async: false + import Eventually import Mox alias Lightning.Adaptors - alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Scheduler alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor @@ -18,6 +19,8 @@ defmodule Lightning.AdaptorsTest do {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} ) + Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() + {:ok, sup: sup} end @@ -95,10 +98,13 @@ defmodule Lightning.AdaptorsTest do {:error, :unreachable} end) - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert {:ok, [%Adaptors.Package{} = pkg]} = + Adaptors.packages(sup) - assert {:ok, [pkg]} = Adaptors.packages(sup) assert pkg.name == "@openfn/language-http" + assert pkg.source == :npm end test "returns {:ok, []} when DB is empty", %{sup: sup} do @@ -113,30 +119,8 @@ defmodule Lightning.AdaptorsTest do # `Lightning.Adaptors.StrategyMock` per `config/test.exs`. Both forms # resolve to `Store.packages(Lightning.Adaptors)`; equality is always # guaranteed regardless of cache state. - assert Adaptors.packages() == Adaptors.packages(Lightning.Adaptors) - end - end - - describe "versions/2" do - test "delegates to Store.versions/2 and returns version list", %{sup: sup} do - stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> - {:error, :unreachable} - end) - - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) - - assert {:ok, [v]} = Adaptors.versions(sup, "@openfn/language-http") - assert v.version == "1.0.0" - end - - test "returns {:error, _} for unknown adaptor when strategy unavailable", %{ - sup: sup - } do - stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> - {:error, :not_found} - end) - - assert {:error, _} = Adaptors.versions(sup, "@openfn/does-not-exist") + assert Adaptors.packages() == + Adaptors.packages(Lightning.Adaptors) end end @@ -147,7 +131,7 @@ defmodule Lightning.AdaptorsTest do end) {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record(schema_data: ~s({"type":"object"})) ) @@ -163,47 +147,17 @@ defmodule Lightning.AdaptorsTest do ordered_body = ~s({"a":1,"z":2,"m":3}) {:ok, _} = - AdaptorsRepo.upsert_adaptor(adaptor_record(schema_data: ordered_body)) + Catalogue.upsert_adaptor(adaptor_record(schema_data: ordered_body)) assert {:ok, ^ordered_body} = Adaptors.schema(sup, "@openfn/language-http") end end - describe "resolve_version/2" do - test "\"latest\" resolves from DB and returns latest_version" do - {:ok, _} = - AdaptorsRepo.upsert_adaptor(adaptor_record(latest_version: "2.3.4")) - - assert {:ok, "2.3.4"} = - Adaptors.resolve_version("@openfn/language-http", "latest") - end - - test "\"local\" resolves from DB and returns latest_version" do - {:ok, _} = - AdaptorsRepo.upsert_adaptor(adaptor_record(latest_version: "1.5.0")) - - assert {:ok, "1.5.0"} = - Adaptors.resolve_version("@openfn/language-http", "local") - end - - test "\"latest\" returns {:error, :not_found} when adaptor absent from DB" do - assert {:error, :not_found} = - Adaptors.resolve_version("@openfn/does-not-exist", "latest") - end - - test "concrete semver passes through without any DB lookup" do - # No adaptor in DB: if a lookup occurred the result would be :not_found. - # Pass-through means we get {:ok, version} regardless. - assert {:ok, "3.0.0"} = - Adaptors.resolve_version("@openfn/language-http", "3.0.0") - end - end - describe "get_adaptor/1" do test "returns a Package for an adaptor in the active source" do {:ok, _} = - AdaptorsRepo.upsert_adaptor(adaptor_record(latest_version: "4.1.0")) + Catalogue.upsert_adaptor(adaptor_record(latest_version: "4.1.0")) assert %Adaptors.Package{ name: "@openfn/language-http", @@ -213,34 +167,63 @@ defmodule Lightning.AdaptorsTest do end test "returns nil for an adaptor absent from the catalogue" do - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record()) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) assert Adaptors.get_adaptor("@openfn/never-existed") == nil end - test "returns nil when the catalogue is empty" do + test "returns nil when the catalogue is empty, without triggering a refresh" do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + flunk("get_adaptor/1 must not trigger a load") + end) + assert Adaptors.get_adaptor("@openfn/language-http") == nil end + test "still resolves a name the catalogue listing excludes" do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record(name: "@openfn/language-collections") + ) + + assert %Adaptors.Package{name: "@openfn/language-collections"} = + Adaptors.get_adaptor("@openfn/language-collections") + + assert {:ok, %Adaptors.Package{name: "@openfn/language-collections"}} = + Adaptors.fetch_adaptor("@openfn/language-collections") + end + test "returns nil for a row under a different source than the active one" do - {:ok, _} = AdaptorsRepo.upsert_adaptor(adaptor_record(source: :local)) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(source: :local)) assert Adaptors.get_adaptor("@openfn/language-http") == nil end end describe "to_wire/1" do - test "delegates to PackageName.to_wire/1" do + test "resolves @latest against the catalogue and passes semver through" do {:ok, _} = - AdaptorsRepo.upsert_adaptor(adaptor_record(latest_version: "2.0.0")) + Catalogue.upsert_adaptor(adaptor_record(latest_version: "2.0.0")) assert Adaptors.to_wire("@openfn/language-http@latest") == - "@openfn/language-http@2.0.0" + {:ok, "@openfn/language-http@2.0.0"} assert Adaptors.to_wire("@openfn/language-http@1.0.0") == - "@openfn/language-http@1.0.0" + {:ok, "@openfn/language-http@1.0.0"} + + assert Adaptors.to_wire(nil) == {:ok, ""} + end + + test "returns the lookup error for an unresolvable @latest" do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) - assert Adaptors.to_wire(nil) == "" + assert Adaptors.to_wire("@openfn/never-existed@latest") == + {:error, :not_found} + end + + test "preserves the @local literal" do + assert Adaptors.to_wire("@openfn/language-http@local") == + {:ok, "@openfn/language-http@local"} end end @@ -300,7 +283,7 @@ defmodule Lightning.AdaptorsTest do end end - describe "refresh_now/1" do + describe "refresh/1" do test "delegates to Scheduler.refresh_now via global_scheduler_name/1", %{ sup: sup } do @@ -309,17 +292,43 @@ defmodule Lightning.AdaptorsTest do # list_adaptors is called by the background Task that :tick spawns. # With an empty DB the scheduler fires an init-tick immediately, so # we must stub before start_scheduler and drain that first tick before - # calling refresh_now (which triggers a second tick). + # calling refresh (which triggers a second tick). stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> send(test_pid, :tick_ran) {:ok, []} end) + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + start_scheduler(sup) assert_receive :tick_ran, 2000 - assert :ok = Adaptors.refresh_now(sup) + # Let the init tick's cycle clear so refresh starts a new one instead + # of coalescing. + {:global, gname} = AdaptorsSupervisor.global_scheduler_name(sup) + pid = :global.whereis_name(gname) + assert_eventually(:sys.get_state(pid).refresh == nil, 2000) + + assert :ok = Adaptors.refresh(sup) assert_receive :tick_ran, 2000 + assert_eventually(:sys.get_state(pid).refresh == nil, 2000) + end + + test "await: true returns the awaited cycle's counts", %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, []} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + start_scheduler(sup) + + assert {:ok, %{listed: 0, changed: 0, fetched: 0, errors: 0}} = + Adaptors.refresh(sup, await: true) end end @@ -327,18 +336,48 @@ defmodule Lightning.AdaptorsTest do test "delegates to Scheduler.refresh_package via global_scheduler_name/1", %{ sup: sup } do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> {:ok, []} end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _name -> {:ok, adaptor_record(latest_version: "2.0.0")} end) - start_scheduler(sup) + pid = start_scheduler(sup) + + assert :ok = + Adaptors.refresh_package(sup, "@openfn/language-http") + + assert_eventually(:sys.get_state(pid).refresh == nil, 2000) + end + + test "returns {:error, :unavailable} when no Scheduler is running", %{ + sup: sup + } do + :ok = + Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) - assert :ok = Adaptors.refresh_package(sup, "@openfn/language-http") + assert {:error, :unavailable} = + Adaptors.refresh_package(sup, "@openfn/language-http") + end + end + + describe "refresh_icons/1" do + test "returns {:error, :unavailable} when no Scheduler is running", %{ + sup: sup + } do + :ok = + Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) + + assert {:error, :unavailable} = Adaptors.refresh_icons(sup) end end describe "icon_meta/1,2" do - test "icon_meta is @doc false for all arities" do + test "icon_meta is documented (it has callers in lightning_web)" do {:docs_v1, _, :elixir, _, _, _, docs} = Code.fetch_docs(Lightning.Adaptors) icon_meta_docs = @@ -350,7 +389,7 @@ defmodule Lightning.AdaptorsTest do refute Enum.empty?(icon_meta_docs) Enum.each(icon_meta_docs, fn doc -> - assert {{:function, :icon_meta, _}, _, _, :hidden, _} = doc + assert {{:function, :icon_meta, _}, _, _, %{"en" => _}, _} = doc end) end @@ -358,14 +397,16 @@ defmodule Lightning.AdaptorsTest do sup: sup } do {:ok, _} = - AdaptorsRepo.upsert_adaptor( + Catalogue.upsert_adaptor( adaptor_record( icon_square_ext: "svg", icon_square_sha256: :crypto.hash(:sha256, "fake-svg-bytes") ) ) - assert {:ok, meta} = Adaptors.icon_meta(sup, "@openfn/language-http") + assert {:ok, meta} = + Adaptors.icon_meta(sup, "@openfn/language-http") + assert meta.icon_square_ext == "svg" end diff --git a/test/lightning/ai_assistant/ai_assistant_test.exs b/test/lightning/ai_assistant/ai_assistant_test.exs index 99816ed8c5a..4edadd9bf17 100644 --- a/test/lightning/ai_assistant/ai_assistant_test.exs +++ b/test/lightning/ai_assistant/ai_assistant_test.exs @@ -11,6 +11,8 @@ defmodule Lightning.AiAssistantTest do user = insert(:user) project = insert(:project, project_users: [%{user: user, role: :owner}]) workflow = insert(:simple_workflow, project: project) + insert(:adaptor, name: "@openfn/language-common") + insert(:adaptor, name: "@openfn/language-http") [user: user, project: project, workflow: workflow] end @@ -450,7 +452,7 @@ defmodule Lightning.AiAssistantTest do assert session.user_id == user.id assert session.expression == job_1.body - assert session.adaptor == + assert {:ok, session.adaptor} == Lightning.Adaptors.to_wire(job_1.adaptor) assert length(session.messages) == 1 @@ -978,7 +980,7 @@ defmodule Lightning.AiAssistantTest do assert updated_session.expression == expression - assert updated_session.adaptor == + assert {:ok, updated_session.adaptor} == Lightning.Adaptors.to_wire(adaptor) end end @@ -1102,7 +1104,7 @@ defmodule Lightning.AiAssistantTest do assert enriched.expression == job.body - assert enriched.adaptor == + assert {:ok, enriched.adaptor} == Lightning.Adaptors.to_wire(job.adaptor) end @@ -1268,7 +1270,7 @@ defmodule Lightning.AiAssistantTest do # Verify the job body and adaptor are set correctly assert enriched.expression == "console.log('test');" - assert enriched.adaptor == + assert {:ok, enriched.adaptor} == Lightning.Adaptors.to_wire("@openfn/language-http@latest") end diff --git a/test/lightning/ai_assistant/unsaved_job_test.exs b/test/lightning/ai_assistant/unsaved_job_test.exs index 17d5ef94c89..1fd95acfc98 100644 --- a/test/lightning/ai_assistant/unsaved_job_test.exs +++ b/test/lightning/ai_assistant/unsaved_job_test.exs @@ -68,7 +68,7 @@ defmodule Lightning.AiAssistant.UnsavedJobTest do enriched_session = AiAssistant.enrich_session_with_job_context(session) assert enriched_session.expression == "fn(state => state);" - # PackageName.to_wire resolves "latest" to a versioned adaptor + assert String.starts_with?( enriched_session.adaptor, "@openfn/language-http" diff --git a/test/lightning/collaboration/no_change_snapshot_test.exs b/test/lightning/collaboration/no_change_snapshot_test.exs index 61a065d72e8..7f3d213952b 100644 --- a/test/lightning/collaboration/no_change_snapshot_test.exs +++ b/test/lightning/collaboration/no_change_snapshot_test.exs @@ -23,6 +23,8 @@ defmodule Lightning.Collaboration.NoChangeSnapshotTest do # granted access by the owner-anchored startup hook via `owner: self()`. Mox.stub(LightningMock, :broadcast, fn _topic, _message -> :ok end) + Lightning.AdaptorTestHelpers.seed_ready_catalogue() + instance = start_collaboration_instance() user = insert(:user) diff --git a/test/lightning/collaboration/session_readiness_test.exs b/test/lightning/collaboration/session_readiness_test.exs new file mode 100644 index 00000000000..fcf256425b3 --- /dev/null +++ b/test/lightning/collaboration/session_readiness_test.exs @@ -0,0 +1,170 @@ +defmodule Lightning.Collaboration.SessionReadinessTest do + @moduledoc """ + `Session.save_workflow/2`'s first-load wait runs off the Session's own + mailbox, so a concurrent call into the same process is never stalled + behind it. + """ + + # set_mox_global: the strategy call runs in a Task owned by the production + # Scheduler. + use Lightning.DataCase, async: false + + import Lightning.Factories + import Lightning.CollaborationHelpers + import Mox + + alias Lightning.Collaboration.DocumentSupervisor + alias Lightning.Collaboration.Registry + alias Lightning.Collaboration.Session + + setup :set_mox_global + setup :verify_on_exit! + + setup do + Mox.stub(LightningMock, :broadcast, fn _topic, _message -> :ok end) + + instance = start_collaboration_instance() + user = insert(:user) + workflow = insert(:workflow, name: "Original Name") + document_name = "workflow:#{workflow.id}" + + start_supervised!( + {DocumentSupervisor, + workflow: workflow, + document_name: document_name, + registry: instance.registry, + pg_scope: instance.pg_scope, + owner: self(), + auto_exit: false, + name: Registry.via(instance.registry, {:doc_supervisor, document_name})} + ) + + session_pid = + start_supervised!( + {Session, + workflow: workflow, + user: user, + document_name: document_name, + registry: instance.registry, + pg_scope: instance.pg_scope, + name: + Registry.via(instance.registry, {:session, document_name, user.id, 1})} + ) + + allow_collaboration_process(session_pid) + + %{ + instance: instance, + session: session_pid, + user: user, + workflow: workflow, + document_name: document_name + } + end + + defp adaptor_record(overrides \\ []) do + overrides = Map.new(overrides) + + %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + description: nil, + homepage: nil, + repository: nil, + license: nil, + deprecated: false, + schema_data: nil, + schema_sha256: nil, + versions: [ + %{ + version: "1.0.0", + integrity: "sha512-abc", + tarball_url: "https://example.com/x-1.0.0.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + ] + } + |> Map.merge(overrides) + end + + test "does not stall a concurrent call into the same session while waiting, and resolves via GenServer.reply on success", + %{session: session, user: user} do + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + Process.sleep(150) + + {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + fn "@openfn/language-http" -> {:ok, adaptor_record()} end + ) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + save_task = Task.async(fn -> Session.save_workflow(session, user) end) + + # The Session must still answer while the save is waiting. + Process.sleep(30) + assert %Yex.Doc{} = Session.get_doc(session) + + assert {:ok, saved_workflow} = Task.await(save_task, 5_000) + assert saved_workflow.id != nil + end + + test "replies {:error, :adaptor_catalogue_unavailable} when the wait fails, without stalling a concurrent call", + %{session: session, user: user} do + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + Process.sleep(100) + {:error, :unreachable} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + save_task = Task.async(fn -> Session.save_workflow(session, user) end) + + Process.sleep(20) + assert %Yex.Doc{} = Session.get_doc(session) + + assert {:error, :adaptor_catalogue_unavailable} = + Task.await(save_task, 5_000) + end + + describe "readiness wait crash safety" do + setup context do + Mimic.copy(Lightning.Adaptors) + Mimic.set_mimic_global(context) + :ok + end + + test "a raise inside the readiness wait replies :adaptor_catalogue_unavailable", + %{session: session, user: user} do + Mimic.stub(Lightning.Adaptors, :ensure_loaded, fn -> raise "boom" end) + + save_task = Task.async(fn -> Session.save_workflow(session, user) end) + + assert {:error, :adaptor_catalogue_unavailable} = + Task.await(save_task, 5_000) + end + + test "an exit inside the readiness wait replies :adaptor_catalogue_unavailable", + %{session: session, user: user} do + Mimic.stub(Lightning.Adaptors, :ensure_loaded, fn -> exit(:boom) end) + + save_task = Task.async(fn -> Session.save_workflow(session, user) end) + + assert {:error, :adaptor_catalogue_unavailable} = + Task.await(save_task, 5_000) + end + end +end diff --git a/test/lightning/collaboration/session_test.exs b/test/lightning/collaboration/session_test.exs index 6881fd44253..ef3d9302add 100644 --- a/test/lightning/collaboration/session_test.exs +++ b/test/lightning/collaboration/session_test.exs @@ -2,6 +2,7 @@ defmodule Lightning.SessionTest do use Lightning.DataCase, async: true import Eventually + import Lightning.AdaptorTestHelpers import Lightning.Factories import Lightning.CollaborationHelpers import Mox @@ -25,6 +26,9 @@ defmodule Lightning.SessionTest do setup do instance = start_collaboration_instance() user = insert(:user) + + seed_ready_catalogue() + {:ok, instance: instance, user: user} end diff --git a/test/lightning/projects/provisioner_test.exs b/test/lightning/projects/provisioner_test.exs index d360e9e7f5e..ab07b0433cb 100644 --- a/test/lightning/projects/provisioner_test.exs +++ b/test/lightning/projects/provisioner_test.exs @@ -269,6 +269,34 @@ defmodule Lightning.Projects.ProvisionerTest do end end + describe "import_document/2 adaptor validation" do + test "allows the import when the adaptor is known" do + user = insert(:user) + insert(:adaptor, name: "@openfn/language-foo") + %{body: body} = valid_document() + + body = + Map.update!(body, "workflows", fn workflows -> + Enum.map(workflows, fn workflow -> + Map.update!(workflow, "jobs", fn [first_job | rest] -> + [ + Map.put(first_job, "adaptor", "@openfn/language-foo@1.0.0") + | rest + ] + end) + end) + end) + + Mox.stub( + Lightning.Extensions.MockUsageLimiter, + :limit_action, + fn _action, _context -> :ok end + ) + + assert {:ok, _project} = Provisioner.import_document(nil, user, body) + end + end + describe "import_document/2 with a new project" do test "finds a credential named in a different normal form" do user = insert(:user) diff --git a/test/lightning/workflows/job_test.exs b/test/lightning/workflows/job_test.exs index cc7e5cad0d1..c5861dd6ffb 100644 --- a/test/lightning/workflows/job_test.exs +++ b/test/lightning/workflows/job_test.exs @@ -517,9 +517,23 @@ defmodule Lightning.Workflows.JobTest do end) end - test "rejects an unrecognised adaptor whether or not the catalogue has rows for the active source" do - # `job.adaptor` reaches the worker's install step unfiltered, so an - # empty catalogue must permit nothing — not even an `@openfn/` name. + test "accepts an adaptor the catalogue listing excludes" do + insert(:adaptor, name: "@openfn/language-collections") + + errors = + Job.changeset(%Job{}, %{ + name: "job", + body: "fn(state => state)", + adaptor: "@openfn/language-collections@1.0.0" + }) + |> errors_on() + + refute errors[:adaptor] + end + + test "a never-loaded catalogue refuses the adaptor as not ready, then rejects it once loaded" do + # With no expectations the production Scheduler's load fails, as it + # would with npm unreachable. params = %{ name: "job", body: "fn(state => state)", @@ -527,7 +541,7 @@ defmodule Lightning.Workflows.JobTest do } assert Job.changeset(%Job{}, params) |> errors_on() |> Map.get(:adaptor) == - ["is not a recognised adaptor"] + ["adaptor catalogue is not ready yet, try again shortly"] insert(:adaptor, name: "@openfn/language-http") @@ -541,7 +555,7 @@ defmodule Lightning.Workflows.JobTest do # The registry membership check only runs on an otherwise-valid changeset, # so name and body are supplied here. [ - "@openfn/language-foo@1.0.0", + "@openfn/language-never-seeded@1.0.0", "@evilcorp/language-http@1.0.0", "common@1.0.0" ] diff --git a/test/lightning/workflows_test.exs b/test/lightning/workflows_test.exs index 18562ccbac4..13911a962bd 100644 --- a/test/lightning/workflows_test.exs +++ b/test/lightning/workflows_test.exs @@ -1468,6 +1468,62 @@ defmodule Lightning.WorkflowsTest do end end + describe "save_workflow/3 adaptor validation" do + test "allows a job adaptor change to a known adaptor" do + user = insert(:user) + project = insert(:project) + insert(:adaptor, name: "@openfn/language-common") + + changeset = + Lightning.Workflows.Workflow.changeset( + %Lightning.Workflows.Workflow{}, + %{ + name: "ungated", + project_id: project.id, + jobs: [ + %{name: "job", body: "fn()", adaptor: "@openfn/language-common"} + ] + } + ) + + assert {:ok, _workflow} = Workflows.save_workflow(changeset, user) + end + + test "refuses an adaptor the catalogue does not list" do + user = insert(:user) + project = insert(:project) + insert(:adaptor, name: "@openfn/language-common") + + changeset = + Lightning.Workflows.Workflow.changeset( + %Lightning.Workflows.Workflow{}, + %{ + name: "gated", + project_id: project.id, + jobs: [ + %{name: "job", body: "fn()", adaptor: "@openfn/language-evil"} + ] + } + ) + + assert {:error, %Ecto.Changeset{} = cs} = + Workflows.save_workflow(changeset, user) + + assert %{jobs: [%{adaptor: ["is not a recognised adaptor"]}]} = + errors_on(cs) + end + + test "a save without adaptor changes does not consult the catalogue" do + user = insert(:user) + workflow = insert(:workflow) + + changeset = + Lightning.Workflows.Workflow.changeset(workflow, %{name: "renamed"}) + + assert {:ok, _workflow} = Workflows.save_workflow(changeset, user) + end + end + describe "save_workflow/3 rescue" do setup do Mimic.copy(Lightning.WorkflowVersions) diff --git a/test/lightning_web/channels/run_channel_test.exs b/test/lightning_web/channels/run_channel_test.exs index 87c8c827c81..071f432a05d 100644 --- a/test/lightning_web/channels/run_channel_test.exs +++ b/test/lightning_web/channels/run_channel_test.exs @@ -247,18 +247,6 @@ defmodule LightningWeb.RunChannelTest do setup :set_google_credential setup :create_socket_and_run - # `@latest` resolves via a direct `Repo.get_adaptor/2` read, so it's - # safe to seed here even though this file runs async: true. - setup do - insert(:adaptor, - name: "@openfn/language-common", - source: :npm, - latest_version: "1.6.2" - ) - - :ok - end - test "fetch:plan success", %{ socket: socket, run: run, @@ -323,6 +311,33 @@ defmodule LightningWeb.RunChannelTest do } end + test "fetch:plan replies with an error when a job adaptor cannot be resolved", + %{project: project} = context do + insert(:adaptor, name: "@openfn/language-readiness-fixture") + + trigger = build(:trigger, type: :webhook, enabled: true) + job = build(:job, adaptor: "@openfn/language-never-published-zzz@latest") + + workflow = + %{triggers: [trigger]} = + build(:workflow, project: project) + |> with_trigger(trigger) + |> with_job(job) + |> with_edge({trigger, job}, %{condition_type: :always}) + |> insert() + + {:ok, snapshot} = Workflows.Snapshot.create(workflow) + + %{socket: socket} = + context + |> Map.merge(%{workflow: workflow, trigger: trigger, snapshot: snapshot}) + |> merge_setups([:create_run, :create_socket, :join_run_channel]) + + ref = push(socket, "fetch:plan", %{}) + + assert_reply ref, :error, %{reason: "adaptor_not_found"} + end + @tag project_retention_policy: :erase_all test "fetch:plan for project with erase_all retention setting", %{ credential: credential, @@ -2793,6 +2808,7 @@ defmodule LightningWeb.RunChannelTest do job = build(:job, + adaptor: "@openfn/language-common@1.6.2", body: ~s[fn(state => { return {...state, extra: "data"} })], project_credential: %{credential: credential, project: project} ) diff --git a/test/lightning_web/channels/run_with_options_test.exs b/test/lightning_web/channels/run_with_options_test.exs index e64d091d4fd..ab45b06e83c 100644 --- a/test/lightning_web/channels/run_with_options_test.exs +++ b/test/lightning_web/channels/run_with_options_test.exs @@ -10,13 +10,9 @@ defmodule LightningWeb.RunWithOptionsTest do describe "rendering a run" do setup do - # Clear the production Adaptors.Supervisor Cachex so each test's seeded - # rows are visible (Cachex persists across DB-sandbox boundaries). cache = Lightning.Adaptors.Supervisor.cache_name(Lightning.Adaptors) Cachex.clear(cache) - # Seed @openfn/language-common so `@latest` resolves to a concrete - # semver via `Lightning.Adaptors.PackageName.to_wire/1`. insert(:adaptor, name: "@openfn/language-common", source: :npm, @@ -77,10 +73,8 @@ defmodule LightningWeb.RunWithOptionsTest do run = Runs.get_for_worker(run.id) - assert RunWithOptions.render(run) - |> Jason.encode!() - |> Jason.decode!() == - expected_result + assert {:ok, plan} = RunWithOptions.render(run) + assert plan |> Jason.encode!() |> Jason.decode!() == expected_result {:ok, workflow} = workflow @@ -130,31 +124,11 @@ defmodule LightningWeb.RunWithOptionsTest do } } - assert RunWithOptions.render(run) - |> Jason.encode!() - |> Jason.decode!() == - expected_result + assert {:ok, plan} = RunWithOptions.render(run) + assert plan |> Jason.encode!() |> Jason.decode!() == expected_result end - test "renders adaptors with @local when :local strategy source is active" do - prev = Application.get_env(:lightning, Lightning.Adaptors, []) - - Application.put_env( - :lightning, - Lightning.Adaptors, - Keyword.put(prev, :strategy, Lightning.Adaptors.Local) - ) - - on_exit(fn -> - Application.put_env(:lightning, Lightning.Adaptors, prev) - end) - - insert(:adaptor, - name: "@openfn/language-common", - source: :local, - latest_version: "local" - ) - + test "returns the adaptor lookup error when a job's @latest cannot be resolved" do user = insert(:user) {:ok, %{triggers: [trigger], jobs: [job]} = workflow} = @@ -163,28 +137,18 @@ defmodule LightningWeb.RunWithOptionsTest do |> Workflows.save_workflow(user) %{runs: [run]} = - work_order_for(trigger, - workflow: workflow, - dataclip: insert(:dataclip) - ) + work_order_for(trigger, workflow: workflow, dataclip: insert(:dataclip)) |> insert() - expected_result = - %{ - "jobs" => [ - %{ - "adaptor" => "@openfn/language-common@local", - "body" => job.body, - "credential_id" => nil, - "id" => job.id, - "name" => job.name - } - ] - } + run = Runs.get_for_worker(run.id) + + snapshot_job = + Enum.find(run.snapshot.jobs, &(&1.id == job.id)) + |> Map.put(:adaptor, "@openfn/language-never-published@latest") - result = run.id |> Runs.get_for_worker() |> RunWithOptions.render() + run = put_in(run.snapshot.jobs, [snapshot_job]) - assert expected_result["jobs"] == result["jobs"] + assert {:error, :not_found} = RunWithOptions.render(run) end end diff --git a/test/lightning_web/channels/workflow_channel_test.exs b/test/lightning_web/channels/workflow_channel_test.exs index 0e1d7b64136..ec6c957b8e9 100644 --- a/test/lightning_web/channels/workflow_channel_test.exs +++ b/test/lightning_web/channels/workflow_channel_test.exs @@ -1,6 +1,7 @@ defmodule LightningWeb.WorkflowChannelTest do use LightningWeb.ChannelCase + import Lightning.AdaptorTestHelpers import Lightning.CollaborationHelpers import Lightning.Factories import Lightning.ProjectsHelpers @@ -20,6 +21,8 @@ defmodule LightningWeb.WorkflowChannelTest do # Stub the broadcast calls that save_workflow makes Mox.stub(LightningMock, :broadcast, fn _topic, _message -> :ok end) + seed_ready_catalogue() + user = insert(:user) project = insert(:project, project_users: [%{user: user, role: :owner}]) workflow = insert(:workflow, project: project) @@ -2701,13 +2704,9 @@ defmodule LightningWeb.WorkflowChannelTest do describe "request_adaptors and request_credentials" do setup do - # The production Adaptors.Supervisor's Cachex persists across tests; - # clear it so each test's seeded Adaptors.Repo rows are visible. cache = Lightning.Adaptors.Supervisor.cache_name(Lightning.Adaptors) Cachex.clear(cache) - # Seed Adaptors.Repo rows so packages/0 returns a non-empty list. - # Individual tests insert additional rows for icon-meta assertions. insert(:adaptor, name: "@openfn/language-salesforce", source: :npm) insert(:adaptor, name: "@openfn/language-http", source: :npm) :ok @@ -3250,6 +3249,25 @@ defmodule LightningWeb.WorkflowChannelTest do assert saved.lock_version == lock_version end + test "replies an error and keeps the channel alive when the session process is dead", + %{socket: socket} do + # A dead session makes the call exit rather than raise; the reply must + # still arrive. + session_pid = socket.assigns.session_pid + ref_mon = Process.monitor(session_pid) + Process.exit(session_pid, :kill) + assert_receive {:DOWN, ^ref_mon, :process, ^session_pid, :killed} + + ref = push(socket, "save_workflow", %{}) + + assert_reply ref, :error, %{ + errors: %{base: ["An internal error occurred"]}, + type: "internal_error" + } + + assert Process.alive?(socket.channel_pid) + end + test "returns validation errors", %{socket: socket} do # Set invalid data in Y.Doc (blank name) session_pid = socket.assigns.session_pid @@ -3344,6 +3362,76 @@ defmodule LightningWeb.WorkflowChannelTest do "Snapshot Collision" end + test "handles an adaptor catalogue that is not ready", %{ + socket: socket, + workflow: workflow + } do + # Global mode: the refresh runs in a Task owned by the production + # Scheduler. + Lightning.Adaptors.Catalogue.delete_all_for_source(:npm) + Mox.set_mox_global(Lightning.Adaptors.StrategyMock) + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:error, :unreachable} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + session_pid = socket.assigns.session_pid + doc = Lightning.Collaboration.Session.get_doc(session_pid) + workflow_map = Yex.Doc.get_map(doc, "workflow") + + Yex.Doc.transaction(doc, "test_update", fn -> + Yex.Map.set(workflow_map, "name", "Blocked By Catalogue") + end) + + ref = push(socket, "save_workflow", %{}) + + assert_reply ref, :error, %{ + errors: %{ + base: ["The adaptor catalogue is still loading. Try again shortly."] + }, + type: "adaptor_catalogue_unavailable" + } + + refute Lightning.Workflows.get_workflow!(workflow.id).name == + "Blocked By Catalogue" + end + + test "does not block other channel traffic while the save is pending", %{ + socket: socket + } do + # Slow enough that a synchronous handle_in would still be blocked when + # the second push is asserted. + Lightning.Adaptors.Catalogue.delete_all_for_source(:npm) + Mox.set_mox_global(Lightning.Adaptors.StrategyMock) + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + Process.sleep(300) + {:error, :unreachable} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + save_ref = push(socket, "save_workflow", %{}) + + name_ref = + push(socket, "validate_workflow_name", %{ + "workflow" => %{"name" => "Another Name"} + }) + + assert_reply name_ref, :ok, _payload, 100 + + assert_reply save_ref, + :error, + %{type: "adaptor_catalogue_unavailable"}, + 2000 + end + test "handles deleted workflow", %{socket: socket, workflow: workflow} do # Delete the workflow Lightning.Repo.update!( diff --git a/test/lightning_web/controllers/adaptor_controller_test.exs b/test/lightning_web/controllers/adaptor_controller_test.exs index cf80fd6d141..fd9298acc5a 100644 --- a/test/lightning_web/controllers/adaptor_controller_test.exs +++ b/test/lightning_web/controllers/adaptor_controller_test.exs @@ -3,11 +3,16 @@ defmodule LightningWeb.AdaptorControllerTest do import Lightning.Factories - alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.AdaptorTestHelpers + alias Lightning.Adaptors.Catalogue alias LightningWeb.AdaptorIconURL describe "GET /adaptors/catalogue" do + # The production cache outlives the SQL sandbox, so an entry another + # test committed would otherwise be served here. setup %{conn: conn} do + AdaptorTestHelpers.clear_global_adaptors_cache() + %{conn: log_in_user(conn, insert(:user))} end @@ -16,7 +21,7 @@ defmodule LightningWeb.AdaptorControllerTest do square_sha = :crypto.hash(:sha256, "square") {:ok, _adaptor} = - AdaptorsRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-http", source: :npm, latest_version: "2.0.0", @@ -52,7 +57,7 @@ defmodule LightningWeb.AdaptorControllerTest do conn: conn } do {:ok, _adaptor} = - AdaptorsRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-http", source: :npm, latest_version: "1.0.0", @@ -75,7 +80,7 @@ defmodule LightningWeb.AdaptorControllerTest do conn: conn } do {:ok, _b} = - AdaptorsRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-b", source: :npm, latest_version: "1.0.0", @@ -83,7 +88,7 @@ defmodule LightningWeb.AdaptorControllerTest do }) {:ok, _a} = - AdaptorsRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-a", source: :npm, latest_version: "1.0.0", @@ -94,13 +99,17 @@ defmodule LightningWeb.AdaptorControllerTest do [first_etag] = get_resp_header(first, "etag") {:ok, _b} = - AdaptorsRepo.upsert_adaptor(%{ + Catalogue.upsert_adaptor(%{ name: "@openfn/language-b", source: :npm, latest_version: "1.0.0", versions: [] }) + # A bare `upsert_adaptor/1` broadcasts nothing, so nothing evicts the + # cached stamp; the Scheduler and Seed are what announce a change. + AdaptorTestHelpers.clear_global_adaptors_cache() + second = get(conn, ~p"/adaptors/catalogue") [second_etag] = get_resp_header(second, "etag") diff --git a/test/lightning_web/controllers/adaptor_icon_controller_test.exs b/test/lightning_web/controllers/adaptor_icon_controller_test.exs index 79c3cb46ff5..bf7712f689c 100644 --- a/test/lightning_web/controllers/adaptor_icon_controller_test.exs +++ b/test/lightning_web/controllers/adaptor_icon_controller_test.exs @@ -5,7 +5,7 @@ defmodule LightningWeb.AdaptorIconControllerTest do import Mox alias Lightning.Adaptors.IconCache - alias Lightning.Adaptors.Repo, as: AdaptorsRepo + alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor alias LightningWeb.AdaptorIconController alias LightningWeb.AdaptorIconURL @@ -41,7 +41,7 @@ defmodule LightningWeb.AdaptorIconControllerTest do overrides ) - {:ok, _adaptor} = AdaptorsRepo.upsert_adaptor(attrs) + {:ok, _adaptor} = Catalogue.upsert_adaptor(attrs) end defp write_icon(name, shape, ext, bytes) do diff --git a/test/lightning_web/live/credential_live_test.exs b/test/lightning_web/live/credential_live_test.exs index 8b89058d4e6..227d1a7d0d3 100644 --- a/test/lightning_web/live/credential_live_test.exs +++ b/test/lightning_web/live/credential_live_test.exs @@ -43,8 +43,6 @@ defmodule LightningWeb.CredentialLiveTest do setup :register_and_log_in_user setup :create_project_for_current_user - # `Credentials.get_schema/1` reads through `Lightning.Adaptors.Repo`; - # seed schemas there so it doesn't fall through to the Strategy mock. setup do Lightning.AdaptorTestHelpers.seed_all_credential_schemas() :ok diff --git a/test/lightning_web/live/maintenance_live/index_test.exs b/test/lightning_web/live/maintenance_live/index_test.exs index 68c31bac8a3..d1d38e8802d 100644 --- a/test/lightning_web/live/maintenance_live/index_test.exs +++ b/test/lightning_web/live/maintenance_live/index_test.exs @@ -57,11 +57,36 @@ defmodule LightningWeb.MaintenanceLive.IndexTest do assert has_element?(live, "#refresh-icons-button") end - test "clicking the icons button reports the refresh result", %{conn: conn} do + test "clicking the icons button starts the refresh and reports the result asynchronously", + %{conn: conn} do stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> {:ok, %{}} end) + {:ok, live, _html} = + live(conn, ~p"/settings/maintenance", on_error: :raise) + + html = + live + |> element("#refresh-icons-button") + |> render_click() + + assert html =~ "Icon refresh started." + + render_async(live) + + assert has_element?( + live, + "p[role=alert][phx-value-key=info]", + "Icon refresh complete" + ) + end + + test "the icons button reports a failed refresh", %{conn: conn} do + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:error, :boom} + end) + {:ok, live, _html} = live(conn, ~p"/settings/maintenance", on_error: :raise) @@ -69,10 +94,12 @@ defmodule LightningWeb.MaintenanceLive.IndexTest do |> element("#refresh-icons-button") |> render_click() + render_async(live) + assert has_element?( live, - "p[role=alert][phx-value-key=info]", - "Icon refresh complete" + "p[role=alert][phx-value-key=error]", + "Icon refresh failed" ) end end diff --git a/test/lightning_web/live/project_live_test.exs b/test/lightning_web/live/project_live_test.exs index 35a90def6db..bea72e682af 100644 --- a/test/lightning_web/live/project_live_test.exs +++ b/test/lightning_web/live/project_live_test.exs @@ -881,9 +881,6 @@ defmodule LightningWeb.ProjectLiveTest do setup :create_project_for_current_user setup do - # Credential creation flows render the JsonSchemaBodyComponent, which - # calls `Credentials.get_schema/1` and so reads through - # `Lightning.Adaptors.Repo`. Lightning.AdaptorTestHelpers.seed_credential_schema("http") :ok end diff --git a/test/lightning_web/live/workflow_live/collaborate_test.exs b/test/lightning_web/live/workflow_live/collaborate_test.exs index 4ea41f8a096..1a16c64c42e 100644 --- a/test/lightning_web/live/workflow_live/collaborate_test.exs +++ b/test/lightning_web/live/workflow_live/collaborate_test.exs @@ -1028,9 +1028,6 @@ defmodule LightningWeb.WorkflowLive.CollaborateTest do end describe "credential modal interactions" do - # `Credentials.get_schema/1` reads through `Lightning.Adaptors.Repo`; - # seed the `http` fixture so the JsonSchemaBodyComponent renders - # without raising. setup do Lightning.AdaptorTestHelpers.seed_credential_schema("http") :ok diff --git a/test/mix/tasks/gen_workflow_hash_test.exs b/test/mix/tasks/gen_workflow_hash_test.exs index 65bd87c31f0..2517a800ab2 100644 --- a/test/mix/tasks/gen_workflow_hash_test.exs +++ b/test/mix/tasks/gen_workflow_hash_test.exs @@ -6,6 +6,13 @@ defmodule Mix.Tasks.Lightning.GenWorkflowHashTest do alias Lightning.WorkflowVersions alias Mix.Tasks.Lightning.GenWorkflowHash + # The task changes the global Logger level; restore it or capture_log goes + # quiet for the rest of the suite. + setup do + level = Logger.level() + on_exit(fn -> Logger.configure(level: level) end) + end + defp run(args) do capture_io(fn -> GenWorkflowHash.run(args) end) |> String.trim() end diff --git a/test/mix/tasks/lightning.adaptors.dump_test.exs b/test/mix/tasks/lightning.adaptors.dump_test.exs new file mode 100644 index 00000000000..00a82daee52 --- /dev/null +++ b/test/mix/tasks/lightning.adaptors.dump_test.exs @@ -0,0 +1,209 @@ +defmodule Mix.Tasks.Lightning.Adaptors.DumpTest do + use Lightning.DataCase + + import ExUnit.CaptureIO + + alias Lightning.Adaptors.Catalogue + alias Mix.Tasks.Lightning.Adaptors.Dump + + @moduletag :tmp_dir + + defp http_record(source) do + %{ + name: "@openfn/language-http", + source: source, + description: "HTTP adaptor", + homepage: "https://openfn.org", + repository: "git+https://github.com/OpenFn/adaptors.git", + license: "LGPL-3.0", + latest_version: "2.1.0", + deprecated: false, + schema_data: ~s({"type":"object"}), + schema_sha256: "sha256-schema-http", + versions: [ + %{ + version: "2.0.0", + integrity: "sha512-two-oh", + tarball_url: "https://example.com/http-2.0.0.tgz", + size_bytes: 11_111, + dependencies: %{"axios" => "^1.4.0"}, + peer_dependencies: %{}, + published_at: ~U[2024-01-01 00:00:00.000000Z], + deprecated: true + }, + %{ + version: "2.1.0", + integrity: "sha512-two-one", + tarball_url: "https://example.com/http-2.1.0.tgz", + size_bytes: 12_345, + dependencies: %{"axios" => "^1.5.0"}, + peer_dependencies: %{"@openfn/language-common" => "^2.0.0"}, + published_at: ~U[2024-06-01 12:00:00.000000Z], + deprecated: false + } + ] + } + end + + defp common_record(source) do + %{ + name: "@openfn/language-common", + source: source, + description: "Common helpers", + latest_version: "1.2.0", + deprecated: false, + versions: [ + %{version: "1.1.0", integrity: "sha512-one-one"}, + %{version: "1.2.0", integrity: "sha512-one-two"} + ] + } + end + + defp dump(args) do + capture_io(fn -> Dump.run(args) end) + end + + defp read_dump(path) do + path |> File.read!() |> Jason.decode!() + end + + @compared_adaptor_fields ~w(name source description homepage repository + license latest_version deprecated schema_data + schema_sha256)a + + @compared_version_fields ~w(version integrity tarball_url size_bytes + dependencies peer_dependencies published_at + deprecated)a + + # Every version row is stamped with the same `inserted_at`, so + # `list_versions/2`'s ordering is not stable enough to compare on. + defp comparable(name, source) do + adaptor = + name |> Catalogue.get_adaptor(source) |> Map.take(@compared_adaptor_fields) + + versions = + name + |> Catalogue.list_versions(source) + |> Enum.map(&Map.take(&1, @compared_version_fields)) + |> Enum.sort_by(& &1.version) + + {adaptor, versions} + end + + describe "run/1" do + setup do + {:ok, _} = Catalogue.upsert_adaptor(http_record(:npm)) + {:ok, _} = Catalogue.upsert_adaptor(common_record(:npm)) + :ok + end + + test "writes every adaptor and its versions to --path", %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "dump.json") + refute File.exists?(path) + + dump(["--path", path]) + + records = read_dump(path) + assert length(records) == 2 + + http = Enum.find(records, &(&1["name"] == "@openfn/language-http")) + + assert http["source"] == "npm" + assert http["latest_version"] == "2.1.0" + assert http["description"] == "HTTP adaptor" + assert http["schema_data"] == ~s({"type":"object"}) + + assert http["versions"] |> Enum.map(& &1["version"]) |> Enum.sort() == + ["2.0.0", "2.1.0"] + + two_one = Enum.find(http["versions"], &(&1["version"] == "2.1.0")) + assert two_one["integrity"] == "sha512-two-one" + assert two_one["size_bytes"] == 12_345 + assert two_one["dependencies"] == %{"axios" => "^1.5.0"} + end + + test "omits row identity and timestamps the importer regenerates", %{ + tmp_dir: tmp_dir + } do + path = Path.join(tmp_dir, "dump.json") + dump(["--path", path]) + + [record | _] = read_dump(path) + + refute Map.has_key?(record, "id") + refute Map.has_key?(record, "inserted_at") + refute Map.has_key?(record, "updated_at") + refute Map.has_key?(record, "checked_at") + + [version | _] = record["versions"] + + refute Map.has_key?(version, "id") + refute Map.has_key?(version, "adaptor_id") + refute Map.has_key?(version, "inserted_at") + end + + test "round-trips back through lightning.adaptors.import", %{ + tmp_dir: tmp_dir + } do + path = Path.join(tmp_dir, "dump.json") + dump(["--path", path]) + + names = ~w(@openfn/language-common @openfn/language-http) + before = Enum.map(names, &comparable(&1, :npm)) + + Catalogue.delete_all_for_source(:npm) + assert Catalogue.list_adaptors(:npm) == [] + + {:ok, 2} = + Lightning.Adaptors.seed_from_file(path, source: :npm, replace: true) + + assert Enum.map(names, &comparable(&1, :npm)) == before + end + + test "includes adaptors the catalogue listing excludes", %{ + tmp_dir: tmp_dir + } do + {:ok, _} = + Catalogue.upsert_adaptor(%{ + name: "@openfn/language-collections", + source: :npm, + latest_version: "1.0.0", + versions: [%{version: "1.0.0"}] + }) + + path = Path.join(tmp_dir, "dump.json") + dump(["--path", path]) + + names = read_dump(path) |> Enum.map(& &1["name"]) + assert "@openfn/language-collections" in names + end + + test "--source local dumps only local rows", %{tmp_dir: tmp_dir} do + {:ok, _} = + Catalogue.upsert_adaptor(%{ + name: "@openfn/language-local-only", + source: :local, + latest_version: "0.1.0", + versions: [%{version: "0.1.0"}] + }) + + path = Path.join(tmp_dir, "dump.json") + dump(["--path", path, "--source", "local"]) + + assert [%{"name" => "@openfn/language-local-only", "source" => "local"}] = + read_dump(path) + end + + test "raises without --path" do + assert_raise RuntimeError, ~r/--path/, fn -> dump([]) end + end + + test "raises on an unknown --source", %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "dump.json") + + assert_raise RuntimeError, ~r/Unknown --source/, fn -> + dump(["--path", path, "--source", "nope"]) + end + end + end +end diff --git a/test/mix/tasks/seed_adaptors_from_file_test.exs b/test/mix/tasks/lightning.adaptors.import_test.exs similarity index 73% rename from test/mix/tasks/seed_adaptors_from_file_test.exs rename to test/mix/tasks/lightning.adaptors.import_test.exs index c82dfe4113d..88b1be9fe35 100644 --- a/test/mix/tasks/seed_adaptors_from_file_test.exs +++ b/test/mix/tasks/lightning.adaptors.import_test.exs @@ -1,10 +1,10 @@ -defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFileTest do +defmodule Mix.Tasks.Lightning.Adaptors.ImportTest do use Lightning.DataCase import ExUnit.CaptureIO - alias Lightning.Adaptors.Repo, as: AdaptorsRepo - alias Mix.Tasks.Lightning.SeedAdaptorsFromFile + alias Lightning.Adaptors.Catalogue + alias Mix.Tasks.Lightning.Adaptors.Import @moduletag :tmp_dir @@ -28,13 +28,13 @@ defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFileTest do ]) capture_io(fn -> - SeedAdaptorsFromFile.run(["--path", path]) + Import.run(["--path", path]) end) assert %{latest_version: "2.1.0"} = - AdaptorsRepo.get_adaptor("@openfn/language-http", :npm) + Catalogue.get_adaptor("@openfn/language-http", :npm) - assert length(AdaptorsRepo.list_versions("@openfn/language-http", :npm)) == + assert length(Catalogue.list_versions("@openfn/language-http", :npm)) == 2 end @@ -51,13 +51,13 @@ defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFileTest do ]) capture_io(fn -> - SeedAdaptorsFromFile.run(["--path", path, "--source", "local"]) + Import.run(["--path", path, "--source", "local"]) end) - assert AdaptorsRepo.get_adaptor("@openfn/language-common", :npm) == nil + assert Catalogue.get_adaptor("@openfn/language-common", :npm) == nil assert %{source: :local} = - AdaptorsRepo.get_adaptor("@openfn/language-common", :local) + Catalogue.get_adaptor("@openfn/language-common", :local) end test "--replace deletes existing rows for the source before seeding", %{ @@ -71,11 +71,11 @@ defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFileTest do ]) capture_io(fn -> - SeedAdaptorsFromFile.run(["--path", path, "--replace"]) + Import.run(["--path", path, "--replace"]) end) - assert AdaptorsRepo.get_adaptor("@openfn/language-stale", :npm) == nil - assert AdaptorsRepo.get_adaptor("@openfn/language-http", :npm) != nil + assert Catalogue.get_adaptor("@openfn/language-stale", :npm) == nil + assert Catalogue.get_adaptor("@openfn/language-http", :npm) != nil end test "--replace rolls back the delete when a later record fails to upsert", @@ -96,18 +96,18 @@ defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFileTest do assert_raise ArgumentError, fn -> capture_io(fn -> - SeedAdaptorsFromFile.run(["--path", path, "--replace"]) + Import.run(["--path", path, "--replace"]) end) end - assert AdaptorsRepo.get_adaptor("@openfn/language-stale", :npm) != nil - assert AdaptorsRepo.get_adaptor("@openfn/language-http", :npm) == nil + assert Catalogue.get_adaptor("@openfn/language-stale", :npm) != nil + assert Catalogue.get_adaptor("@openfn/language-http", :npm) == nil end test "round-trips a snapshot in the shape the download task emits", %{ tmp_dir: tmp_dir } do - # Same shape `mix lightning.download_adaptor_registry_cache` writes: + # Same shape `mix lightning.adaptors.snapshot` writes: # an atom-keyed adaptor_record (see Lightning.Adaptors.Strategy) plus # :source, run through Jason.encode_to_iodata!/1. record = %{ @@ -139,11 +139,11 @@ defmodule Mix.Tasks.Lightning.SeedAdaptorsFromFileTest do File.write!(path, Jason.encode_to_iodata!([record])) capture_io(fn -> - SeedAdaptorsFromFile.run(["--path", path]) + Import.run(["--path", path]) end) assert [%{name: "@openfn/language-http", versions: ["2.1.0"]}] = - Lightning.Adaptors.catalogue() + Catalogue.catalogue(:npm) end end end diff --git a/test/mix/tasks/lightning.adaptors.refresh_test.exs b/test/mix/tasks/lightning.adaptors.refresh_test.exs new file mode 100644 index 00000000000..1800d9711d7 --- /dev/null +++ b/test/mix/tasks/lightning.adaptors.refresh_test.exs @@ -0,0 +1,116 @@ +defmodule Mix.Tasks.Lightning.Adaptors.RefreshTest do + use ExUnit.Case, async: false + use Mimic + + setup_all do + Mimic.copy(Lightning.Adaptors) + :ok + end + + setup do + Mix.shell(Mix.Shell.Process) + on_exit(fn -> Mix.shell(Mix.Shell.IO) end) + :ok + end + + describe "bare invocation" do + test "calls refresh/2 with await: true and exits 0 on {:ok, counts}, reporting them" do + stub(Lightning.Adaptors, :refresh, fn Lightning.Adaptors, opts -> + assert Keyword.fetch!(opts, :await) == true + assert Keyword.fetch!(opts, :timeout) == :timer.minutes(10) + {:ok, %{listed: 109, changed: 4, fetched: 4, errors: 1}} + end) + + Mix.Tasks.Lightning.Adaptors.Refresh.run([]) + assert_received {:mix_shell, :info, [_]} + assert_received {:mix_shell, :info, [msg]} + assert msg =~ "listed 109" + assert msg =~ "fetched 4" + assert msg =~ "errors 1" + end + + test "exits 2 when the cycle succeeds but the source listed no adaptors" do + stub(Lightning.Adaptors, :refresh, fn _sup, _opts -> + {:ok, %{listed: 0, changed: 0, fetched: 0, errors: 0}} + end) + + assert catch_exit(Mix.Tasks.Lightning.Adaptors.Refresh.run([])) == + {:shutdown, 2} + + assert_received {:mix_shell, :error, [_]} + end + + test "exits 2 on {:error, :timeout}" do + stub(Lightning.Adaptors, :refresh, fn _sup, _opts -> {:error, :timeout} end) + + assert catch_exit(Mix.Tasks.Lightning.Adaptors.Refresh.run([])) == + {:shutdown, 2} + + assert_received {:mix_shell, :error, [_]} + end + + test "exits 2 on other error" do + stub(Lightning.Adaptors, :refresh, fn _sup, _opts -> + {:error, :network_down} + end) + + assert catch_exit(Mix.Tasks.Lightning.Adaptors.Refresh.run([])) == + {:shutdown, 2} + + assert_received {:mix_shell, :error, [_]} + end + end + + describe "--name flag" do + test "dispatches to refresh_package/1 with the exact package string" do + pkg = "@openfn/language-http" + stub(Lightning.Adaptors, :refresh_package, fn ^pkg -> :ok end) + Mix.Tasks.Lightning.Adaptors.Refresh.run(["--name", pkg]) + assert_received {:mix_shell, :info, [_]} + end + + test "exits 1 on {:error, :not_found}" do + stub(Lightning.Adaptors, :refresh_package, fn _pkg -> + {:error, :not_found} + end) + + assert catch_exit( + Mix.Tasks.Lightning.Adaptors.Refresh.run([ + "--name", + "@openfn/language-http" + ]) + ) == {:shutdown, 1} + + assert_received {:mix_shell, :error, [_]} + end + + test "exits 2 on other error" do + stub(Lightning.Adaptors, :refresh_package, fn _pkg -> + {:error, :timeout} + end) + + assert catch_exit( + Mix.Tasks.Lightning.Adaptors.Refresh.run([ + "--name", + "@openfn/language-http" + ]) + ) == {:shutdown, 2} + + assert_received {:mix_shell, :error, [_]} + end + end + + describe "rejected flags" do + test "raises on unknown --strategy flag" do + assert_raise OptionParser.ParseError, fn -> + Mix.Tasks.Lightning.Adaptors.Refresh.run(["--strategy", "local"]) + end + end + + test "raises on unknown --source flag" do + assert_raise OptionParser.ParseError, fn -> + Mix.Tasks.Lightning.Adaptors.Refresh.run(["--source", "local"]) + end + end + end +end diff --git a/test/lightning/download_adaptor_registry_test.exs b/test/mix/tasks/lightning.adaptors.snapshot_test.exs similarity index 91% rename from test/lightning/download_adaptor_registry_test.exs rename to test/mix/tasks/lightning.adaptors.snapshot_test.exs index bfebd7a7f87..9686649a002 100644 --- a/test/lightning/download_adaptor_registry_test.exs +++ b/test/mix/tasks/lightning.adaptors.snapshot_test.exs @@ -1,9 +1,9 @@ -defmodule Lightning.DownloadAdaptorRegistryCacheTest do +defmodule Mix.Tasks.Lightning.Adaptors.SnapshotTest do use ExUnit.Case, async: false import ExUnit.CaptureIO - alias Mix.Tasks.Lightning.DownloadAdaptorRegistryCache + alias Mix.Tasks.Lightning.Adaptors.Snapshot @package "@openfn/language-http" @latest_version "2.1.0" @@ -42,7 +42,7 @@ defmodule Lightning.DownloadAdaptorRegistryCacheTest do %{registry: registry, jsdelivr: jsdelivr} end - describe "download_adaptor_registry_cache mix task" do + describe "lightning.adaptors.snapshot mix task" do @describetag :tmp_dir test "does not write file when no adaptors are found", %{ tmp_dir: tmp_dir, @@ -56,7 +56,7 @@ defmodule Lightning.DownloadAdaptorRegistryCacheTest do refute File.exists?(file_path) capture_io(fn -> - DownloadAdaptorRegistryCache.run(["--path", file_path]) + Snapshot.run(["--path", file_path]) end) refute File.exists?(file_path) @@ -90,7 +90,7 @@ defmodule Lightning.DownloadAdaptorRegistryCacheTest do refute File.exists?(file_path) capture_io(fn -> - DownloadAdaptorRegistryCache.run(["--path", file_path]) + Snapshot.run(["--path", file_path]) end) assert [record] = diff --git a/test/mix/tasks/lightning.refresh_adaptors_test.exs b/test/mix/tasks/lightning.refresh_adaptors_test.exs deleted file mode 100644 index ae46ebc60a2..00000000000 --- a/test/mix/tasks/lightning.refresh_adaptors_test.exs +++ /dev/null @@ -1,85 +0,0 @@ -defmodule Mix.Tasks.Lightning.RefreshAdaptorsTest do - use ExUnit.Case, async: false - use Mimic - - setup_all do - Mimic.copy(Lightning.Adaptors) - :ok - end - - setup do - Mix.shell(Mix.Shell.Process) - on_exit(fn -> Mix.shell(Mix.Shell.IO) end) - :ok - end - - describe "bare invocation" do - test "calls refresh_now/0 and exits 0 on :ok" do - stub(Lightning.Adaptors, :refresh_now, fn -> :ok end) - Mix.Tasks.Lightning.RefreshAdaptors.run([]) - assert_received {:mix_shell, :info, [_]} - end - - test "exits 2 on other error" do - stub(Lightning.Adaptors, :refresh_now, fn -> {:error, :network_down} end) - - assert catch_exit(Mix.Tasks.Lightning.RefreshAdaptors.run([])) == - {:shutdown, 2} - - assert_received {:mix_shell, :error, [_]} - end - end - - describe "--name flag" do - test "dispatches to refresh_package/1 with the exact package string" do - pkg = "@openfn/language-http" - stub(Lightning.Adaptors, :refresh_package, fn ^pkg -> :ok end) - Mix.Tasks.Lightning.RefreshAdaptors.run(["--name", pkg]) - assert_received {:mix_shell, :info, [_]} - end - - test "exits 1 on {:error, :not_found}" do - stub(Lightning.Adaptors, :refresh_package, fn _pkg -> - {:error, :not_found} - end) - - assert catch_exit( - Mix.Tasks.Lightning.RefreshAdaptors.run([ - "--name", - "@openfn/language-http" - ]) - ) == {:shutdown, 1} - - assert_received {:mix_shell, :error, [_]} - end - - test "exits 2 on other error" do - stub(Lightning.Adaptors, :refresh_package, fn _pkg -> - {:error, :timeout} - end) - - assert catch_exit( - Mix.Tasks.Lightning.RefreshAdaptors.run([ - "--name", - "@openfn/language-http" - ]) - ) == {:shutdown, 2} - - assert_received {:mix_shell, :error, [_]} - end - end - - describe "rejected flags" do - test "raises on unknown --strategy flag" do - assert_raise OptionParser.ParseError, fn -> - Mix.Tasks.Lightning.RefreshAdaptors.run(["--strategy", "local"]) - end - end - - test "raises on unknown --source flag" do - assert_raise OptionParser.ParseError, fn -> - Mix.Tasks.Lightning.RefreshAdaptors.run(["--source", "local"]) - end - end - end -end diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex index 8ef8e0b5d2f..7ec94d79a63 100644 --- a/test/support/adaptor_test_helpers.ex +++ b/test/support/adaptor_test_helpers.ex @@ -1,21 +1,10 @@ defmodule Lightning.AdaptorTestHelpers do @moduledoc """ - Seeds `Lightning.Adaptors.Repo` and clears the global - `Lightning.Adaptors.Supervisor` Cachex. + Seeds `Lightning.Adaptors.Catalogue` rows and manages the production + `Lightning.Adaptors` cache for tests that read through it. - The production `Lightning.Adaptors` supervisor starts with the - application and is shared across the test suite: its Cachex persists - across the `Ecto.Adapters.SQL.Sandbox` boundary, so tests that seed - rows via the `:adaptor` factory (`insert(:adaptor, attrs)`) must - clear the cache to make them visible to Cachex-backed facade reads - (`packages/1`, `schema/2`, `versions/2`, `icon/3`, via `Store`). - `get_adaptor/1` and `resolve_version/2` read `Repo` directly with no - Cachex in the path, so seeding for those (see `ensure_adaptor/1` - below) doesn't need a cache clear. - - For tests that run their own isolated supervisor instead of the - production one, see `test/lightning/adaptors_test.exs` and - `test/lightning/adaptors/store_test.exs`. + The production cache outlives the SQL sandbox, so a test that seeds + rows and reads them through the cache must clear it first. """ import Lightning.Factories @@ -23,42 +12,50 @@ defmodule Lightning.AdaptorTestHelpers do alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor @doc """ - Clear the production `Lightning.Adaptors` Cachex so subsequent reads - fall back through the DB. + Seeds a throwaway adaptor row so the catalogue counts as loaded and + saves do not wait on the production Scheduler. """ - @spec clear_global_adaptors_cache() :: :ok - def clear_global_adaptors_cache do - cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) - Cachex.clear(cache) + @spec seed_ready_catalogue() :: :ok + def seed_ready_catalogue do + {:ok, _} = + Lightning.Adaptors.Catalogue.upsert_adaptor(%{ + name: "@openfn/language-readiness-fixture", + source: :npm, + latest_version: "1.0.0", + description: nil, + homepage: nil, + repository: nil, + license: nil, + deprecated: false, + schema_data: nil, + schema_sha256: nil, + versions: [] + }) + :ok end @doc """ - Insert one `Adaptors.Repo.Adaptor` row using the factory and clear - the global Cachex so it's immediately visible to facade reads. - - `attrs` is forwarded to the `:adaptor` factory verbatim. + Clears the production `Lightning.Adaptors` cache. """ - @spec seed_adaptor(keyword() | map()) :: Lightning.Adaptors.Repo.Adaptor.t() - def seed_adaptor(attrs \\ []) do - row = insert(:adaptor, attrs) - clear_global_adaptors_cache() - row + @spec clear_global_adaptors_cache() :: :ok + def clear_global_adaptors_cache do + cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) + Cachex.clear(cache) + :ok end @doc """ - Seed the catalogue row an adaptor spec needs to clear - `Lightning.Workflows.Job`'s validation, unless it's already there. - - Deliberately leaves the Cachex alone: the changeset check reads the DB - directly, and clearing a cache shared with other async tests would - disturb them. + Seeds the catalogue row an adaptor spec needs to pass + `Lightning.Workflows.Job` validation, unless it is already there. """ @spec ensure_adaptor(String.t()) :: :ok def ensure_adaptor(spec) when is_binary(spec) do case Lightning.Adaptors.parse_spec(spec) do {name, _version} when is_binary(name) -> - if is_nil(Lightning.Adaptors.get_adaptor(name)), + source = AdaptorsSupervisor.source(Lightning.Adaptors) + + if is_nil(Lightning.Adaptors.Catalogue.get_adaptor(name, source)), do: insert(:adaptor, name: name) :ok @@ -69,18 +66,14 @@ defmodule Lightning.AdaptorTestHelpers do end @doc """ - Seed a credential schema row keyed by short name (e.g. `"postgresql"`), + Seeds a credential schema row keyed by short name (e.g. `"postgresql"`), reading the JSON body from `test/fixtures/schemas/.json`. - - `Credentials.get_schema/1` reads schemas from the adaptor registry, - not directly from disk, so tests exercising it must seed them here. """ @spec seed_credential_schema(String.t()) :: - Lightning.Adaptors.Repo.Adaptor.t() + Lightning.Adaptors.Catalogue.Adaptor.t() def seed_credential_schema(short_name) when is_binary(short_name) do - # Keep the raw JSON binary (not a decoded map) so - # `Lightning.Credentials.Schema.new/2` can decode it downstream with - # `Jason.decode!(_, objects: :ordered_objects)` and preserve field order. + # Raw JSON binary, not a decoded map: `Credentials.Schema.new/2` decodes + # it with ordered objects. schema_body = Path.join(["test", "fixtures", "schemas", "#{short_name}.json"]) |> File.read!() @@ -88,9 +81,8 @@ defmodule Lightning.AdaptorTestHelpers do row = insert(:adaptor, name: short_name, source: :npm, schema_data: schema_body) - # Cachex's fallback runs in the Courier process — it can't see the - # test-owned sandbox connection. Pre-populate the cache so reads - # never need to fall through to a DB lookup from the Courier. + # Cachex fills run in its Courier process, which cannot see the sandbox + # connection, so populate the cache directly. cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) source = AdaptorsSupervisor.source(Lightning.Adaptors) Cachex.put(cache, {:schema, short_name, source}, {:ok, schema_body}) @@ -99,10 +91,7 @@ defmodule Lightning.AdaptorTestHelpers do end @doc """ - Seed every credential schema present in `test/fixtures/schemas/`. - - Use from a `setup` block in tests that exercise multiple credential - types (e.g. `LightningWeb.CredentialLiveTest`). + Seeds every credential schema present in `test/fixtures/schemas/`. """ @spec seed_all_credential_schemas() :: :ok def seed_all_credential_schemas do @@ -134,22 +123,17 @@ defmodule Lightning.AdaptorTestHelpers do end @doc """ - Seed an `@openfn/*` adaptor package with a concrete `latest_version` - so `Lightning.Adaptors.PackageName.to_wire/1` resolves `@latest` - correctly. + Seeds an adaptor with one version so `@latest` resolves to it. """ - @spec seed_adaptor_package(String.t(), String.t() | [String.t()]) :: - Lightning.Adaptors.Repo.Adaptor.t() - def seed_adaptor_package(name, versions) - when is_binary(name) and is_list(versions) do - # The first version in the list is treated as latest. - [latest | _] = versions - + @spec seed_adaptor_package(String.t(), String.t()) :: + Lightning.Adaptors.Catalogue.Adaptor.t() + def seed_adaptor_package(name, latest_version) + when is_binary(name) and is_binary(latest_version) do {:ok, row} = - Lightning.Adaptors.Repo.upsert_adaptor(%{ + Lightning.Adaptors.Catalogue.upsert_adaptor(%{ name: name, source: :npm, - latest_version: latest, + latest_version: latest_version, description: nil, homepage: nil, repository: nil, @@ -157,137 +141,20 @@ defmodule Lightning.AdaptorTestHelpers do deprecated: false, schema_data: nil, schema_sha256: nil, - versions: - Enum.map(versions, fn v -> - %{ - version: v, - integrity: "sha512-#{v}", - tarball_url: "https://example.com/x-#{v}.tgz", - size_bytes: 1024, - dependencies: %{}, - peer_dependencies: %{}, - published_at: nil, - deprecated: false - } - end) - }) - - row - end - - def seed_adaptor_package(name, latest_version) - when is_binary(name) and is_binary(latest_version) do - seed_adaptor_package(name, [latest_version]) - end - - @doc """ - Build a record matching `t:Lightning.Adaptors.Strategy.adaptor_record/0` - for use in `Mox.stub`/`Mox.expect` setups. - """ - @spec build_strategy_adaptor_record(String.t(), String.t()) :: map() - def build_strategy_adaptor_record(name, latest_version) do - %{ - name: name, - source: :npm, - latest_version: latest_version, - description: nil, - homepage: nil, - repository: nil, - license: nil, - deprecated: false, - schema_data: nil, - schema_sha256: nil, - versions: [ - %{ - version: latest_version, - integrity: "sha512-#{latest_version}", - tarball_url: "https://example.com/x-#{latest_version}.tgz", - size_bytes: 1024, - dependencies: %{}, - peer_dependencies: %{}, - published_at: nil, - deprecated: false - } - ] - } - end - - @doc """ - Pre-populate the production `Lightning.Adaptors` supervisor's Cachex - with a packages map, so async tests get seeded data without the - Cachex Courier process falling through to a DB query it can't see - (the sandboxed connection is invisible to it). - - Returns the supervisor source atom for convenience. - """ - @spec warm_packages_cache([map()]) :: :npm | :local - def warm_packages_cache(metas) when is_list(metas) do - cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) - source = AdaptorsSupervisor.source(Lightning.Adaptors) - - Cachex.put(cache, {:packages, source}, {:ok, metas}) - - source - end - - @doc """ - Bulk-seed the common `@openfn/*` packages, with the versions - expected by tests across the suite. - """ - @spec seed_common_packages() :: :ok - def seed_common_packages do - packages = [ - {"@openfn/language-common", ["1.6.2", "1.2.22", "1.1.0"]}, - {"@openfn/language-http", ["3.1.12", "2.0.0", "1.0.0"]}, - {"@openfn/language-postgresql", ["3.2.0", "2.0.0", "1.0.0"]}, - {"@openfn/language-dhis2", ["3.0.4", "2.0.0", "1.0.0"]}, - {"@openfn/language-salesforce", ["3.0.0", "2.0.0", "1.0.0"]}, - {"@openfn/language-godata", ["2.0.0", "1.0.0"]}, - {"@openfn/language-googlesheets", ["2.0.0", "1.0.0"]} - ] - - Enum.each(packages, fn {name, versions} -> - seed_adaptor_package(name, versions) - end) - - # Warm Cachex for the production supervisor so reads from any - # process (including the LiveView's caller chain) see the seeded - # data without falling through to the Cachex Courier's - # sandbox-blind DB query. - cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) - source = AdaptorsSupervisor.source(Lightning.Adaptors) - - metas = - Enum.map(packages, fn {name, [latest | _]} -> - %{ - name: name, - latest_version: latest, - description: nil, - deprecated: false, - icon_square_ext: nil, - icon_rectangle_ext: nil, - icon_square_sha256: nil, - icon_rectangle_sha256: nil - } - end) - - Cachex.put(cache, {:packages, source}, {:ok, metas}) - - Enum.each(packages, fn {name, versions} -> - version_metas = - Enum.map(versions, fn v -> + versions: [ %{ - version: v, - integrity: "sha512-#{v}", + version: latest_version, + integrity: "sha512-#{latest_version}", + tarball_url: "https://example.com/x-#{latest_version}.tgz", size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, published_at: nil, deprecated: false } - end) - - Cachex.put(cache, {:versions, name, source}, {:ok, version_metas}) - end) + ] + }) - :ok + row end end diff --git a/test/support/factories.ex b/test/support/factories.ex index 73946274845..0d363445b21 100644 --- a/test/support/factories.ex +++ b/test/support/factories.ex @@ -5,7 +5,7 @@ defmodule Lightning.Factories do alias Lightning.Workflows.Snapshot def adaptor_factory do - %Lightning.Adaptors.Repo.Adaptor{ + %Lightning.Adaptors.Catalogue.Adaptor{ name: sequence(:adaptor_name, &"@openfn/language-test-#{&1}"), source: :npm, latest_version: "1.0.0", diff --git a/test/test_helper.exs b/test/test_helper.exs index 9f0c8398d3c..790779e12bb 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -7,11 +7,16 @@ Mox.defmock(Lightning.Tesla.Mock, for: Tesla.Adapter) Mox.defmock(Lightning.Adaptors.StrategyMock, for: Lightning.Adaptors.Strategy) +Mox.defmock(Lightning.AdaptorService.RepoMock, + for: Lightning.AdaptorService.Repo +) + :ok = Application.ensure_started(:ex_machina) Mimic.copy(:hackney) Mimic.copy(File) Mimic.copy(IO) +Mimic.copy(Lightning.Adaptors.Config) Mimic.copy(Lightning.FailureEmail) Mimic.copy(Lightning.Projects.Provisioner) Mimic.copy(Lightning.MetadataService) diff --git a/tooling/adaptor_cache/README.md b/tooling/adaptor_cache/README.md index 4cfe1043c48..2705626bcba 100644 --- a/tooling/adaptor_cache/README.md +++ b/tooling/adaptor_cache/README.md @@ -67,7 +67,7 @@ bin/adaptor_cache --help # full usage ``` With the cache up and the three vars exported, run -`mix lightning.refresh_adaptors` as usual. The first run records the cache; +`mix lightning.adaptors.refresh` as usual. The first run records the cache; every run after that is local, with no network needed at all. ### Reading `bin/adaptor_cache logs` @@ -119,7 +119,7 @@ Either form updates the packument _and_ the search response's `latest_version` together in one call — `scheduler.ex`'s change-detection compares the search response against the DB to decide whether to bother fetching the packument at all, so updating only one is a silent no-op. Run -`mix lightning.refresh_adaptors` (or reopen the picker) afterwards to see it +`mix lightning.adaptors.refresh` (or reopen the picker) afterwards to see it take effect. ## Scenarios diff --git a/tooling/adaptor_cache/lib/cli.ex b/tooling/adaptor_cache/lib/cli.ex index e6ec3f847ea..c34047891d6 100644 --- a/tooling/adaptor_cache/lib/cli.ex +++ b/tooling/adaptor_cache/lib/cli.ex @@ -300,7 +300,7 @@ defmodule AdaptorCache.Cli do export ADAPTORS_NPM_JSDELIVR_URL=#{base}/jsdelivr export ADAPTORS_NPM_GITHUB_URL=#{base}/github - Then: mix lightning.refresh_adaptors + Then: mix lightning.adaptors.refresh Watch it work: bin/adaptor_cache logs Prove it works: bin/adaptor_cache check """) From fbff7a522a4585e14129a84c66f428bf33ca098e Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Wed, 2 Sep 2026 16:41:24 +0200 Subject: [PATCH 07/37] Add the ADAPTORS.md guide, carry icon metadata for airgapped mirroring - Comment pass across the adaptors branch: fix stale/narration/inaccurate comments - Icon metadata carried through adaptor dump/import for airgapped mirroring - ADAPTORS.md guide added, adaptor docs elsewhere trimmed - Lightning.Adaptors.refresh/1 no longer crashes on a bare keyword list - Adaptor catalogue seeded in the real-worker integration tests - Two inaccurate warning-timing claims in ADAPTORS.md tightened --- .claude/rules/adaptors-docs.md | 11 +- .claude/rules/adaptors-otp.md | 14 +- .dialyzer_ignore.exs | 1 - .env.example | 31 +- .gitignore | 1 - ADAPTORS-CONTEXT.md | 417 ++++++++++++++++++ ADAPTORS.md | 118 +++++ DEPLOYMENT.md | 8 +- RUNNINGLOCAL.md | 98 +--- .../components/AdaptorIcon.tsx | 7 +- .../collaborative-editor/hooks/useChannel.ts | 1 + .../components/AdaptorIcon.test.tsx | 6 +- .../components/MiniMapNode.test.tsx | 2 +- .../test/workflow-diagram/nodes/Job.test.tsx | 4 +- bin/e2e.d/manager | 4 +- config/config.exs | 2 +- config/dev.exs | 2 - config/prod.exs | 4 - config/test.exs | 4 - lib/lightning/adaptor_registry.ex | 19 - lib/lightning/adaptor_service.ex | 3 + lib/lightning/adaptors.ex | 2 + lib/lightning/adaptors/icon_cache.ex | 6 +- lib/lightning/adaptors/local.ex | 9 +- lib/lightning/adaptors/npm/github.ex | 16 +- lib/lightning/adaptors/npm/registry.ex | 6 +- lib/lightning/adaptors/scheduler.ex | 19 +- lib/lightning/adaptors/seed.ex | 17 +- lib/lightning/adaptors/store.ex | 6 +- lib/lightning/adaptors/strategy.ex | 6 +- lib/lightning/config.ex | 13 - lib/lightning/config/bootstrap.ex | 38 +- lib/lightning/release.ex | 6 +- lib/lightning/runs/handlers.ex | 17 +- lib/lightning/workflows/job.ex | 5 + .../controllers/adaptor_icon_controller.ex | 7 +- lib/mix/tasks/install_adaptor_icons.ex | 101 ----- lib/mix/tasks/install_schemas.ex | 272 ------------ lib/mix/tasks/lightning.adaptors.dump.ex | 24 +- mix.exs | 3 +- test/fixtures/adaptor_registry_cache.json | 232 ---------- test/fixtures/adaptors/dhis2.tar.gz | Bin 24379 -> 0 bytes test/fixtures/adaptors/http.tar.gz | Bin 5641 -> 0 bytes test/fixtures/adaptors/http_dhis2.tar.gz | Bin 29838 -> 0 bytes test/integration/web_and_worker_test.exs | 15 +- test/integration/workflow_edge_cases_test.exs | 17 +- test/lightning/adaptors_test.exs | 14 + test/lightning/config/bootstrap_test.exs | 57 +-- test/lightning/install_adaptor_icons_test.exs | 128 ------ test/lightning/install_schemas_test.exs | 260 ----------- .../tasks/lightning.adaptors.dump_test.exs | 10 +- test/test_helper.exs | 1 - tooling/adaptor_cache/README.md | 6 +- 53 files changed, 761 insertions(+), 1309 deletions(-) create mode 100644 ADAPTORS-CONTEXT.md create mode 100644 ADAPTORS.md delete mode 100644 lib/lightning/adaptor_registry.ex delete mode 100644 lib/mix/tasks/install_adaptor_icons.ex delete mode 100644 lib/mix/tasks/install_schemas.ex delete mode 100644 test/fixtures/adaptor_registry_cache.json delete mode 100644 test/fixtures/adaptors/dhis2.tar.gz delete mode 100644 test/fixtures/adaptors/http.tar.gz delete mode 100644 test/fixtures/adaptors/http_dhis2.tar.gz delete mode 100644 test/lightning/install_adaptor_icons_test.exs delete mode 100644 test/lightning/install_schemas_test.exs diff --git a/.claude/rules/adaptors-docs.md b/.claude/rules/adaptors-docs.md index cd2b1862828..fe9c98d96da 100644 --- a/.claude/rules/adaptors-docs.md +++ b/.claude/rules/adaptors-docs.md @@ -19,13 +19,16 @@ paths: Most of `.context/adaptors/` is archaeology from designs that were abandoned before they shipped. Grep will find it and it reads convincingly. Everything at the top level of that -folder is current, and there are five things: +folder is current, and there are six things: - `README.md` — the entry point, and the shortest thing to read. -- `ATLAS.md` — the architecture in seven diagrams, stamped with the commit it describes. +- `ATLAS.md` — the architecture in eight diagrams, stamped with the commit it describes. Start here to understand the shape of the subsystem. - `REWRITE-2026-05.md` — the canonical spec. Per-callback contracts and the reasoning behind each decision. Grep it, don't read it end to end. +- `BEHAVIOURS.md` — the subsystem's promises, one section per behaviour, each stating what + holds today and what observation would settle it. Read it before treating something as a + gap nobody noticed. - `07-channel-live-update-findings-2026-06-03.md` — two decisions still open, still blocking. - `NOTES.md` — a running log of open questions and irregularities hit while working on the subsystem, newest entry on top. Dated entries, none of them acted on yet. Read it before @@ -42,8 +45,8 @@ Live status is the PR, not the folder: `gh pr view 4801 --json body -q .body`. D reconstruct that checklist anywhere else, and don't infer completion state from the archived phase-A/phase-B PRDs — both phases shipped, so those describe code that already exists. -If you change the subsystem's shape, update `ATLAS.md` and move its commit stamp. The reason -that folder needed archiving is that nobody did this last time. +If you change the subsystem's shape, update `ATLAS.md` and move its commit stamp. A stale +architecture diagram is worse than none, because it misleads with confidence. Process and naming conventions for this subsystem are a separate rule: `.claude/rules/adaptors-otp.md`. diff --git a/.claude/rules/adaptors-otp.md b/.claude/rules/adaptors-otp.md index 51755120bb9..4345c1ad2f4 100644 --- a/.claude/rules/adaptors-otp.md +++ b/.claude/rules/adaptors-otp.md @@ -17,25 +17,25 @@ When adding or changing a process here: - Take `:name` from opts (`Keyword.fetch!(opts, :name)`) and derive any child, cache, topic or lock name from it. Follow the helpers at - `lib/lightning/adaptors/supervisor.ex:159-205`. -- Add it to the fixed child list in `init/1` (`supervisor.ex:101-122`) with its + `lib/lightning/adaptors/supervisor.ex:116-173`. +- Add it to the fixed child list in `init/1` (`supervisor.ex:67-85`) with its collaborators passed in the child spec. Do not add a `Registry`: the child set is fixed and the registered atom already addresses it. - Public functions that talk to a running process lead with the server ref, defaulted: `def refresh(sup \\ @sup, name)`. `start_link` takes `name:` in trailing opts. - The `Scheduler` is a cluster singleton behind `HighlanderPG` and registers under - `global_scheduler_name/1` (`supervisor.ex:196`). `Process.whereis` will not find + `global_scheduler_name/1` (`supervisor.ex:149`). `Process.whereis` will not find it. Known wart, do not copy it: `strategy` and `source` are published to -`:persistent_term` in `init/1` (`supervisor.ex:74-77`) and re-read at call time by -`Scheduler` (`scheduler.ex:156-160`, `:166`, `:194`) and `Store`. New code should +`:persistent_term` in `init/1` (`supervisor.ex:40-43`) and re-read at call time by +`Scheduler` (`scheduler.ex:259`, `:271`, `:313`) and `Store`. New code should take them from process state or the child spec instead. `Scheduler` already does -this correctly for `source` (`scheduler.ex:111`, `:127-135`). +this correctly for `source` (`scheduler.ex:113`, `:132`). In tests, prefer `Mox.allow(StrategyMock, self(), pid)` and keep `async: true`, as -`test/lightning/adaptors/store_test.exs:115` does. `set_mox_global` costs the file +`test/lightning/adaptors/store_test.exs:135` does. `set_mox_global` costs the file its async, and is only justified where the hop graph is genuinely dynamic, as in `highlander_integration_test.exs:27`. diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs index 08e7a01b72c..544f8324bc1 100644 --- a/.dialyzer_ignore.exs +++ b/.dialyzer_ignore.exs @@ -2,7 +2,6 @@ # `task_worker.ex` needed a skip here until `flags: [:no_opaque]` in mix.exs # covered it. Restore it if that flag goes. {"lib/lightning/auth_providers/well_known.ex", :invalid_contract}, - {"lib/mix/tasks/install_schemas.ex", :invalid_contract}, # httpoison 3.0.0 typespec bug, surfaced by hackney 4. Hackney 4 moved to a # process-per-connection design, so a client handle is now a pid where it used diff --git a/.env.example b/.env.example index 4b6b8ca7df9..fed1090c9f2 100644 --- a/.env.example +++ b/.env.example @@ -252,18 +252,11 @@ # `mix lightning.adaptors.dump` (from an existing catalogue) or # `mix lightning.adaptors.snapshot` (straight from npm, no DB needed). # -# Enable local adaptors mode. OPENFN_ADAPTORS_REPO takes one repo path, or a -# comma-separated list to merge several. See RUNNINGLOCAL.md for the details. -# LOCAL_ADAPTORS=true -# OPENFN_ADAPTORS_REPO=/path/to/repo/ -# OPENFN_ADAPTORS_REPO=/path/to/private,/path/to/canonical -# -# The new Lightning.Adaptors subsystem. ADAPTORS_STRATEGY picks which -# strategy serves adaptors: npm (default) or local. LOCAL_ADAPTORS above is a -# deprecated back-compat alias for ADAPTORS_STRATEGY=local, and -# OPENFN_ADAPTORS_REPO above is a deprecated back-compat alias for -# ADAPTORS_LOCAL_REPO below (both log a boot warning). -# ADAPTORS_STRATEGY=local +# Where the adaptor catalogue comes from: npm (default) or local. Local mode +# reads an OpenFn adaptors checkout from ADAPTORS_LOCAL_REPO, which takes one +# path or a comma-separated list; the first checkout that has a package wins. +# See ADAPTORS.md. +# ADAPTORS_STRATEGY=npm # ADAPTORS_LOCAL_REPO=/path/to/repo/ # ADAPTORS_LOCAL_REPO=/path/to/private,/path/to/canonical # @@ -271,14 +264,12 @@ # under the system temp dir. # ADAPTORS_ICONS_PATH=/path/to/icon/cache # -# Lightning.Adaptors.NPM upstream URLs. Leave these unset in production: the -# real npm, jsDelivr and raw.githubusercontent endpoints are the defaults -# baked into the strategy sub-modules themselves (@default_registry_url in -# npm/registry.ex, @default_jsdelivr_url in npm/schema.ex, -# @default_github_url/@default_github_ref in npm/github.ex). Point them at -# the local caching reverse proxy while iterating on the adaptors subsystem, -# so refresh ticks are served from disk and work offline. Start it with -# `bin/adaptor_cache up`; see tooling/adaptor_cache/README.md. +# Upstreams for the npm strategy. Defaults are https://registry.npmjs.org, +# https://cdn.jsdelivr.net and https://raw.githubusercontent.com. Set them to +# use an internal mirror, or point them at the local caching reverse proxy +# while iterating on the adaptors subsystem so refresh ticks are served from +# disk and work offline. Start it with `bin/adaptor_cache up`; see +# tooling/adaptor_cache/README.md. # ADAPTORS_NPM_REGISTRY_URL=http://localhost:4874/npm # ADAPTORS_NPM_JSDELIVR_URL=http://localhost:4874/jsdelivr # ADAPTORS_NPM_GITHUB_URL=http://localhost:4874/github diff --git a/.gitignore b/.gitignore index 529d2a96531..ba16bc618fd 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,6 @@ lightning-*.tar # Ignore adaptor registry cache adaptor_registry_cache.json -!test/fixtures/adaptor_registry_cache.json # Ignore assets that are produced by build tools. /priv/static/images/adaptors/ diff --git a/ADAPTORS-CONTEXT.md b/ADAPTORS-CONTEXT.md new file mode 100644 index 00000000000..91e65f1f991 --- /dev/null +++ b/ADAPTORS-CONTEXT.md @@ -0,0 +1,417 @@ +# Adaptors on the fly: working context + +**Branch:** `adaptors-on-the-fly`, PR [#4801](https://github.com/OpenFn/lightning/pull/4801). The PR targets `release-2.19.0`; the branch itself is based on `main` (see §5 for what that means). +**Worktree:** `/Users/stuart/Sourcecode/lightning/.claude/worktrees/adaptors-on-the-fly`. Run everything from there. +**Last updated:** 2026-09-14 +**Operator reference:** [ADAPTORS.md](ADAPTORS.md) in this repo is the how-to for deployers. This document is the working record for whoever picks the branch up, and it stands on its own. + +Every `file:line` below was checked against the tree on 14 Sep. Read the source at the citation rather than trusting the paraphrase. + +--- + +## 1. What this branch is for + +Lightning used to learn about adaptors once, at Docker build time. The adaptor list, every credential schema and every icon were baked into the image by `mix lightning.install_schemas` and `mix lightning.install_adaptor_icons`. A new adaptor, or a new version of an existing one, meant waiting for a Lightning rebuild and a redeploy before anyone could pick it. + +This branch moves all of that to runtime. Lightning keeps its own catalogue of adaptors in Postgres and refreshes it from npm on a timer, so new adaptors, new versions, icons and credential schemas appear on a running instance without touching the deployment. + +Two things define "working": + +- The adaptor picker matches npm within one refresh interval (an hour by default), without a redeploy. +- An airgapped or offline instance still boots with a catalogue you loaded deliberately, and keeps serving it. + +The second is not an afterthought. Several OpenFn deployments sit behind restricted networks, and this change moves the network requirement from build time to run time: an image build no longer needs npm, jsDelivr or GitHub, but a running instance does. Anywhere the build network had outbound access and the runtime network does not, this breaks unless the operator uses the offline snapshot route. That is why the snapshot/dump/import tooling exists, and why most of the outstanding work in §7 is about behaving sensibly when the upstream cannot be reached. + +--- + +## 2. How the pieces fit + +Everything lives under `lib/lightning/adaptors/`, with `lib/lightning/adaptors.ex` as the public facade. + +**The facade** (`lib/lightning/adaptors.ex`). Every function that talks to a running process takes the supervisor instance as an optional first argument, defaulted from `Config.default_instance()`. The public functions are `packages/1`, `schema/2`, `resolve_name/2`, `icon/3`, `catalogue/1`, `fetch_adaptor/2`, `ensure_loaded/1`, `parse_spec/1`, `valid_format?/1`, `to_wire/2`, `refresh/1,2`, `refresh_package/2`, `refresh_icons/1`, `icon_meta/2`, `subscribe_to_updates/1`, plus two delegates used by the snapshot tooling, `seed_from_file/2` and `dump_to_file/2` (`adaptors.ex:370, 376`). Reads return `{:ok, _}` or `{:error, _}`; `resolve_name/2` and `parse_spec/1` were the last two hiding a failure behind a bare value and were converted in `220f7a4240`, with their callers moved in the same commit. Not everything is a tuple: `valid_format?/1` returns a boolean, and `ensure_loaded/1`, `refresh_package/2`, `subscribe_to_updates/1` and `refresh/2` without `await: true` return a bare `:ok` on success. `ensure_started/1` is on `Lightning.Adaptors.Supervisor` (`supervisor.ex:47`), not the facade. + +Configuration is under `config :lightning, Lightning.Adaptors` with keys `:strategy`, `:refresh_interval` (ms), `:first_load_timeout` (ms), `:cache_timeout_ms` and `:icon_path` (`adaptors.ex:20-30`); defaults live in `Lightning.Adaptors.Config`. Only some of those have environment variables (§4). `first_load_timeout` does not; it is application config only. + +**The store** (`adaptors/store.ex`). A per-node Cachex cache with no TTL, backed by Postgres through `Catalogue`. Reads never write to the catalogue; the scheduler is the only writer. Cache coherence between nodes comes from change broadcasts: `Lightning.Adaptors.Invalidator` subscribes to the source's PubSub topic and drops the affected keys on `{:changed, name, source}`, and `Lightning.Adaptors.NodeMonitor` calls `Store.warm_from_repo/1` when it sees a `:nodeup`, so a node that missed broadcasts during a partition re-warms from Postgres when the other node comes back. `nodedown` is deliberately a no-op. + +**The catalogue** (`adaptors/catalogue.ex`) is the Ecto layer over two tables, `adaptors` and `adaptor_versions`. Read paths for the picker and credential types apply the exclusion list (`@excluded_names`, `catalogue.ex:52-55`) and the `deprecated == false` filter through `active_adaptors/1` (`:336-341`). The resolve paths, `get_adaptor/2` (`:111-113`) and `list_versions/2` (`:119-127`), are deliberately unfiltered so a job already pinned to an excluded or deprecated adaptor keeps validating. Write paths apply no filter at all; whatever the strategy lists gets written. + +**Source strategies.** `Lightning.Adaptors.NPM` (default) and `Lightning.Adaptors.Local`. A strategy provides `list_adaptors/0` plus per-adaptor metadata, schema and icon fetches. npm reads the org listing `GET /-/user/openfn/package` as the authoritative name list and uses npm search only for cheap version lookup, with a per-name packument fallback for names search misses (`npm/registry.ex` moduledoc). The reason for two calls: search omits deprecated packages, and we need to know about those to keep already-pinned jobs working. Change detection in the scheduler diffs each upstream `{name, latest_version}` pair against the rows already in the database (`scheduler.ex:505`, `fetch_if_changed/4`). + +Local mode stores each package's real semver from `package.json` in the database, but the projection served to the picker replaces it with the literal `local` (`store.ex:341-342`), so every adaptor shows version `local` in the UI while `SELECT latest_version FROM adaptors` shows real versions. + +**The scheduler** (`adaptors/scheduler.ex`) is a cluster singleton elected by a Postgres advisory lock (HighlanderPG wraps the scheduler child at `supervisor.ex:124-131`), running a refresh cycle on a timer. Children are supervised `one_for_one` (`supervisor.ex:134`) so a crash in one component does not take the rest down. + +**Naming and instances.** Every child, cache, PubSub topic and lock name derives from the supervisor's `:name` via `Module.concat` or string interpolation (`supervisor.ex:90-93, 178-234`), defaulting to `Config.default_instance()`. That is what lets several instances run in one BEAM, which is how the tests isolate themselves, and how `Supervisor.ensure_started/1` (`:46-55`) can treat `{:error, {:already_started, pid}}` as `{:ok, pid}`. `ensure_started/1` also starts the named Finch pool if Tesla is configured for Finch and nothing is registered under that name yet (`ensure_finch/0`, `:60-69`), which is what the out-of-band paths (mix tasks, `bin/lightning eval`, isolated tests) need. In normal boot the subsystem is an ordinary child of `Lightning.Supervisor` after `Lightning.Repo` (`application.ex:170`). On the out-of-band path it starts inside `Ecto.Migrator.with_repo` (`setup.ex:45-52`) because the scheduler takes its advisory lock as soon as it boots and needs the Repo up first. + +**HTTP.** `GET /adaptors/catalogue` (`adaptor_controller.ex`, `router.ex:121`): session-authenticated, not project-scoped, ETag from a `(stamp, count)` pair, `cache-control: private, no-cache`, `vary: Cookie`, 304 on a matching `If-None-Match`; 503 with `{"error": "adaptor catalogue unavailable"}` on any `{:error, _}`; 200 with `{"data": []}` when the catalogue has loaded and is genuinely empty. `GET /adaptors/icons/:name/:filename` (`adaptor_icon_controller.ex`, `router.ex:70-72`) is public, content-addressed on the first eight characters of the sha, served with `public, max-age=31536000, immutable`, a `default-src 'none'; sandbox` CSP and `nosniff` (`lightning_web/utils.ex:174-187`). A stale sha 302-redirects to the current URL with `Cache-Control: no-store`; an unknown adaptor or removed icon is a 404; a catch-all clause at `:107` returns 404 for anything else. Adaptor names never build a filesystem path directly: `icon_meta/2` looks the name up in the catalogue first, so a traversal attempt is simply `:not_found`. `d65c3e61dc` added the explicit rejection of names that could escape the icon cache directory, and `001161f66e` annotated those checks. + +The editor fetches its adaptor list over HTTP (`assets/js/collaborative-editor/api/adaptors.ts`), not the channel. The channel still has a `request_adaptors` handler (`workflow_channel.ex:109-117`) that collapses `{:error, _}` to an empty list (`:993-997`); nothing in `assets/js` calls it outside tests (see §11). + +**Live update.** `Lightning.Adaptors.ChannelBroadcaster` coalesces a burst of `{:changed, ...}` into one `adaptors_updated` push every 250ms (`channel_broadcaster.ex:19, 62-83`). The channel forwards it (`workflow_channel.ex:726`) and the editor reloads the list itself. `Lightning.Credentials.SchemaReconciler` listens for the same event (§4, migrations). + +### The first-load gate + +A cold instance has an empty catalogue, and a caller reading it must not conclude "no adaptors exist" when the truth is "we have not looked yet". `Store.gated/2` (`store.ex:251-259`) wraps the reads: run the read; if the result is empty and the catalogue has never loaded, wait for a refresh and read once more; otherwise return what came back. "Empty" is `{:error, :not_found}`, `{:ok, []}` or `{:ok, {_stamp, []}}` (`empty?/1`, `:261-264`). Only an empty result pays for the `loaded?` query. + +Five reads go through it (`store.ex:69, 102, 158, 187, 213`): `schema/2`, `icon/3`, `packages/1`, `catalogue/1`, `icon_meta/2`. `fetch_adaptor/2` and `ensure_loaded/1` use the same wait through `first_load/1`. + +"Has it ever loaded" is answered by `loaded?/1` (`store.ex:269-272`), which is true if either: + +- `Catalogue.max_checked_at(source)` is non-nil, meaning at least one row exists for this source. `checked_at` is stamped on every adaptor the cycle saw, changed or not (`Catalogue.upsert_adaptor/1` and `touch_checked_at/2`), so `max_checked_at` is nil only when there are zero rows. A previous boot or a seed import both count. Nothing clears it except `delete_all_for_source/1`. +- `Scheduler.completed?/1` is true. The scheduler sets it when a cycle finishes as `{:ok, %{listed: 0, errors: 0}}` (`scheduler.ex:271-272`) and nothing ever resets it. It is a field in the scheduler's process state (initialised `false` at `:165`), so a scheduler restart loses it and costs one more refresh, not a loop. `completed?/1` answers `false` when the scheduler is unreachable (`:137-142`). A cycle that listed anything at all does not set it; rows answer for those. + +The second clause landed in `5f70ab91d2` to fix a real loop. Before it, a cycle that completed and wrote no rows left `max_checked_at` nil, so every read started another full refresh, forever. There were four ways to reach that state: the upstream listing genuinely empty; every per-adaptor fetch failing; every fetch succeeding and every upsert failing (which `dcbd4dd8c1` fixed separately, below); and local mode pointed at an empty directory. Filters cannot cause it because they only apply on read. The moduledoc at `store.ex:14-23` describes the intended contract: a first-load gate, not a health check. + +The wait is `Config.first_load_timeout/0`, defaulting to 90 seconds (`config.ex:16`). It was 60 and went up in `220f7a4240` because a cold npm listing plus a per-adaptor fetch of the whole org does not reliably finish inside a minute. `Store.await_refresh/1` (`store.ex:284-292`) is a `GenServer.call` to the scheduler with that timeout; the scheduler stashes the caller in `waiters` (`scheduler.ex:351-358`) and replies to everyone when the refresh task finishes (`:261-275`). On a client-side timeout the store returns `{:error, :timeout}`; if the scheduler is not running, `{:error, :unavailable}`. After a refresh resolves without an error, `first_load/1` re-reads `loaded?` rather than trusting the refresh counts, because a failed cycle can land on rows a seed already wrote (`:275-282`). + +The gate lives in `Store` rather than the facade on purpose: reads flow `Adaptors → Store → Catalogue`/`IconCache`, and the blocking belongs at the layer that knows whether it found anything. Putting it in the facade would have meant two competing readiness mechanisms, which is also why `get_schema`/`fetch_schema` were collapsed back into one `schema/2`. + +### How the scheduler treats a missing schema + +The same "we know" versus "we don't know yet" question exists one level down, for credential schemas, and was answered deliberately there. `NPM.Schema.schema/2` returns `{:ok, {nil, nil}}` only for a 404 on `configuration-schema.json` at the pinned version (`npm/schema.ex:35`); every other outcome is `{:error, _}`, and the moduledoc (`:8-10`) says the split exists so a transient failure is never mistaken for genuine absence. A whole-package 404 is `{:error, :not_found}` from `Registry.get_packument/1`. + +What gets persisted is only `schema_data` (nullable); there is no column for which of the three states a row is in. Retry is inferred at tick time in `fetch_if_changed/4` (`scheduler.ex:505-541`) from `schema_data`, `updated_at` and a one-hour grace window (`@schema_grace_ms`, `:33`): + +- has a schema, same version: skip, touch `checked_at` only. +- no schema, same version, `updated_at` within the hour: ask again. A repeated nil touches `checked_at` but not `updated_at`, so the clock runs from the last real change. +- no schema, same version, `updated_at` older than an hour: stop asking. The persisted meaning of "we have given up" is literally `schema_data IS NULL AND updated_at < now() - 1h`. There is no marker. + +Consumers only see `has_schema: not is_nil(schema_data)` on `package_meta` (`catalogue.ex:23-31`), so nothing downstream can tell "confirmed absent" from "still inside the retry window". `keep_stored_schema/2` (`:552-560`) carries a previous version's schema forward when a version bump comes back without one, because jsDelivr may not have mirrored it yet. The operator path `refresh_package/2` (`force_refresh_one/3`, `:745-769`) does not do this and takes upstream as-is, so it is the only route by which a genuine schema removal lands. + +The scheduler's tick summary log is `Adaptors[]: refresh tick listed=N changed=N touched=N fetched=N icons=N healed=N not_modified=N errors=N duration=Nms` (`scheduler.ex:480-486`). Since `dcbd4dd8c1`, `errors = fetch_errors + (changed - persisted)` (`:477`), so an adaptor that fetched fine but failed to write counts as an error and a systemic write failure can no longer report a clean cycle. `touched = listed - changed - fetch_errors`. + +--- + +## 3. What happens when npm cannot be reached + +This is the behaviour a Services person or a client admin will ask about, and one claim in an earlier handoff is now false, so it is spelled out here. + +- Boot is never blocked. The scheduler runs the first refresh in the background; the app serves pages while it does. +- With a populated catalogue, a failed refresh writes nothing and leaves rows alone. The picker, credential forms and validation keep working from the existing rows. The log shows `Scheduler: list_adaptors failed: ` and a tick summary with `errors=1 changed=0`. The next tick retries. Recovery needs no restart. +- If only GitHub is unreachable, metadata still lands and icons are skipped that tick (`fetch_icons failed ... persisting records without icons`) and retried next tick. +- A single broken adaptor (bad packument) is skipped with a `fetch_adaptor() failed` warning, everything else lands, `errors=1`. +- With an empty catalogue and no upstream, reads that hit the gate wait up to 90 seconds. This is the part that changed on 14 Sep: `GET /adaptors/catalogue`, the credential form's type list, icon lookups, `Job` validation and `AdaptorService.install/2` all now block for the first-load timeout on a cold instance before they fail. The old statement "HTTP is served immediately regardless of catalogue state" was true before `220f7a4240` and is not true now. Do not let it into anything customer-facing without re-checking §7.2. +- After the timeout: `GET /adaptors/catalogue` is 503, the editor shows "Couldn't load adaptors. Please try again." with a Retry button, workflow save is rejected with "adaptor catalogue is not ready yet, try again shortly" (`job.ex:157-172`; the channel-side string is "The adaptor catalogue is still loading. Try again shortly.", `workflow_channel.ex:1247`), the credential form crashes (§7.3), and `install/2` returns `{:error, {:catalogue_unavailable, reason}}`. +- An imported snapshot survives the hourly refresh. A failed refresh does not wipe it. +- There is no switch that turns the subsystem off. `ADAPTORS_REFRESH_INTERVAL_SECONDS=0` disables the timer, after which the catalogue changes only on import or manual refresh. With the timer off and an empty catalogue, boot logs a warning pointing at ADAPTORS.md's offline section (`scheduler.ex:219-223`); with the timer on it logs `catalogue is empty at boot — refreshing now` (`:215-217`). + +--- + +## 4. Deployment surface + +### New environment variables + +All read in `lib/lightning/config/bootstrap.ex` (`:997-1083` for the `ADAPTORS_*` block). + +| Variable | Purpose | Default | +| --- | --- | --- | +| `ADAPTORS_STRATEGY` | Where the catalogue comes from: `npm` or `local` | `npm` | +| `ADAPTORS_LOCAL_REPO` | Path(s) to an adaptors monorepo checkout (the repo root, not `packages/`), comma-separated; first match wins on a name collision, and the log names every shadowed package on every scan | unset | +| `ADAPTORS_ICONS_PATH` | On-disk icon cache directory | `/lightning/adaptor_icons` (`config.ex:15`, resolved at call time so a release does not bake in a build-time tmp path). The official image sets `/app/priv/adaptor_icons` in the Dockerfile | +| `ADAPTORS_REFRESH_INTERVAL_SECONDS` | Refresh cadence; `0` disables scheduled refreshes | `3600` | +| `ADAPTORS_NPM_REGISTRY_URL` | npm registry | `https://registry.npmjs.org` | +| `ADAPTORS_NPM_JSDELIVR_URL` | CDN credential schemas are read from | `https://cdn.jsdelivr.net` | +| `ADAPTORS_NPM_GITHUB_URL` | Raw host icons are read from | `https://raw.githubusercontent.com` | +| `ADAPTORS_NPM_GITHUB_REF` | Git ref of `OpenFn/adaptors` icons come from | `main` | +| `ADAPTORS_NPM_HTTP_TIMEOUT` | Per-request receive timeout, ms | `30000` | + +All three upstreams are public, unauthenticated GETs. No tokens needed. An internal npm mirror works by setting the three `ADAPTORS_NPM_*_URL` variables and leaving the strategy as `npm`. + +### Removed and deprecated + +- **`SCHEMAS_PATH`: removed.** Delete it from deployment config. No references remain in `lib/` or `config/`. +- **`ADAPTORS_REGISTRY_JSON_PATH`: removed.** Same. +- `LOCAL_ADAPTORS=true` still works and warns "LOCAL_ADAPTORS is deprecated, use ADAPTORS_STRATEGY=local instead." (`bootstrap.ex:1063`), but only when `ADAPTORS_STRATEGY` is unset and `LOCAL_ADAPTORS` is the thing selecting local mode. +- `OPENFN_ADAPTORS_REPO` still works and warns "OPENFN_ADAPTORS_REPO is deprecated, use ADAPTORS_LOCAL_REPO instead." (`:1083`), only when local mode is on and `ADAPTORS_LOCAL_REPO` is unset. +- `ADAPTORS_STRATEGY=local` with no repo path at all fails boot with "ADAPTORS_STRATEGY is set to local, but neither ADAPTORS_LOCAL_REPO nor the deprecated OPENFN_ADAPTORS_REPO is set." (`:1030`). +- `ADAPTORS_PATH` is unchanged and unrelated (`bootstrap.ex:235`, default `./priv/openfn`). It is where the worker installs packages for execution. + +### Migrations + +Three, all required, forward-only, no backfill. `git diff main --name-only -- priv/repo/migrations` shows exactly these: + +- `20260514150000_create_adaptors.exs` creates `adaptors` and `adaptor_versions` (including the `deprecated` boolean). +- `20260827084128_add_adaptor_catalogue_indexes.exs` indexes `adaptors.updated_at` and `adaptor_versions.inserted_at`. +- `20260907112954_widen_credentials_schema.exs` widens `credentials.schema` from varchar(40) to varchar(100), for full package names. + +Existing credentials' short schema names (`http`) are rewritten to full package names (`@openfn/language-http`) at runtime by `Lightning.Credentials.SchemaReconciler`, a GenServer that runs once on start and again on every `adaptors_updated` broadcast, calling `Credentials.reconcile_legacy_schema_names/1` (`credentials.ex:606`). It only touches rows whose `schema` does not start with `@`, so it is idempotent; it does not bump `updated_at` and writes no audit events. A short name the catalogue cannot resolve is left alone. The UI still shows the short name; anything reading the database or the provisioning payload directly sees the long form. + +### Build changes + +`mix lightning.install_schemas` and `mix lightning.install_adaptor_icons` are deleted (`lib/mix/tasks/install_schemas.ex`, `install_adaptor_icons.ex`) and their `RUN` lines are gone from `Dockerfile` and `Dockerfile-dev`, along with the `COPY priv/schemas` and `ENV SCHEMAS_PATH` lines. RUNNINGLOCAL.md lost 74 lines about them. `bin/bootstrap` never called either task, so it is unchanged. A fork or custom build script calling those tasks will fail. + +The Dockerfile sets `ENV ADAPTORS_ICONS_PATH=/app/priv/adaptor_icons` (`:118`) and creates and chowns the directory to the runtime user (`:127`). `docker-compose.yml` mounts a named volume `adaptor_icons` there (`:15, :43, :74`). Anything else on ephemeral storage re-downloads icons on every restart. + +### Offline / airgapped + +Full procedure in ADAPTORS.md § "Running without internet access". Short version, on an online instance: + +```sh +mix lightning.adaptors.dump --path snapshot.json +tar czf icons.tar.gz -C "$ADAPTORS_ICONS_PATH" . +``` + +Or, with no populated instance anywhere, build from npm with no database: `mix lightning.adaptors.snapshot --path snapshot.json`. Without `--path` it writes `adaptor_registry_cache.json` under `priv`. It produces no icons. + +Offline: + +```sh +mkdir -p "$ADAPTORS_ICONS_PATH" && tar xzf icons.tar.gz -C "$ADAPTORS_ICONS_PATH" +mix lightning.adaptors.import --path snapshot.json --replace +``` + +`dump` and `import` both take `--source npm|local` (default `npm`) for moving a local-mode catalogue. On a release image: `bin/lightning eval 'Lightning.Release.dump_adaptors("…")'` and `Lightning.Release.seed_adaptors("…", replace: true)` (`release.ex:50, 72`). There is no release-image equivalent of `snapshot`; building a cold-start snapshot needs a source checkout with Mix. Plan for that if a client is airgapped and there is no populated instance to dump from. + +**The snapshot carries icon metadata, not icon bytes.** Copy the icons directory separately or icons are silently missing. That is the most likely airgapped mistake. + +Icons on disk are laid out as `///..`. Bytes on disk whose hash does not match the catalogue row are refused and the mismatch is cached as an error until the row's sha changes (`icon_cache.ex` moduledoc), so a hand-edit of the icon directory does not take and is not re-fetched on every request either. + +--- + +## 5. What has landed + +41 commits off `main` as of 14 Sep, 31 of them adaptor work. Against `release-2.19.0`, the PR's target, it is 36 commits: five of the ten non-adaptor commits are already on that branch (see §12). + +**Foundation (25-31 Aug).** `32b38a7e68` added the data model, the Postgres store, the source strategies and the refresh scheduler. `4df07659e6` served icons and the catalogue over HTTP and wired the editor up. `69f6595ed2` cut every caller over to the `Lightning.Adaptors` facade and deleted the old registry. `ff55f4cd7c` moved the supervision to `one_for_one`. + +**Hardening (2-8 Sep).** `4348ac2d15` added ADAPTORS.md and made the snapshot carry icon metadata so an airgapped mirror can be rebuilt. `790087de3f` cached the catalogue projection through the store. `4217880b62` made the npm org listing authoritative and hid deprecated adaptors from the pickers. `67f93a8d09` fixed a leaked tick chain in `refresh_now` and tightened per-adaptor fetch timeouts. `b76007fb18` stopped the supervisor crash-looping on a bad boot-time DB read. `9bcb43826a` gave schema-less adaptors an empty schema instead of crashing. `385024ef34` added the empty-catalogue boot warning and made the refresh interval configurable. + +**Security and ops (10-11 Sep).** `d65c3e61dc` rejected adaptor names that could escape the icon cache directory; `001161f66e` annotated the traversal checks; `7d51f345bb` served icon content types from literals. `b010fbb804` made the scheduler take its interval and the empty-catalogue warning as opts. `8f95c205bf` gave the icon cache a fixed path in the image and a named volume in compose. `f6325ee457` renamed the refresh interval variable from `_MS` to `_SECONDS`: the default is an hour and every other recurring tick in the app is configured in seconds or coarser; milliseconds stay the internal unit. + +`0007277c06` started the adaptors subsystem from the out-of-band setup commands. What prompted it: credential validation resolves adaptor names through `Lightning.Adaptors`, and nothing under `mix run --no-start` or `bin/lightning eval` had ever started it, so a colleague reproducing an unrelated bug hit an opaque `:persistent_term` `ArgumentError`. `Lightning.Setup.setup_user/3`, `Lightning.Demo.reset_demo/0` and `mix lightning.kickstart` all shared the gap; the fix routed all three through `with_minimum_setup/1` (`setup.ex:42`) and made `Supervisor.ensure_started/1` tolerate an already-running instance. It also fixed a real bug on the way past: `ensure_minimum_setup` checked for PubSub with `Process.whereis(mod)`, the module name `Phoenix.PubSub`, instead of the `Lightning.PubSub` name it registers under, so it matched an unrelated `:pg` scope and never actually verified PubSub was up. The fix is `Process.whereis(Keyword.get(opts, :name, mod))`. + +**The first-load work (14 Sep)**, the source of most of §7: + +- `e27931777e`: start Finch when the adaptors supervisor starts on its own, so the out-of-band paths have an HTTP pool (fixes Tesla "unknown registry" errors from setup commands, demo reset and isolated tests). +- `220f7a4240`: move the readiness gate into `Store`, gate every catalogue read on the first load, convert `resolve_name/2` and `parse_spec/1` to tuples, delete `get_adaptor/2`, update the `Credential`, `Job` and `AdaptorIconController` callers, raise the first-load timeout from 60s to 90s. +- `dcbd4dd8c1`: count the adaptors a refresh tick failed to write (the `errors` formula in §2). +- `5f70ab91d2`: settle the first-load gate on a source that lists no adaptors, via `Scheduler.completed?`. +- `931f16a020`: stop reporting an unreachable catalogue as a refused adaptor. `AdaptorService.known?/1` is gone, replaced by a three-way split in `install/2` (`adaptor_service.ex:311-336`): `{:ok, _}` installs, `{:error, :not_found}` gives `:adaptor_not_permitted`, and any other error gives `{:error, {:catalogue_unavailable, reason}}`. `MetadataService` reports `adaptor_catalogue_unavailable` separately from `no_matching_adaptor` (`metadata_service.ex:128, 131`). + +A code review on 14 Sep confirmed the three items the previous handoff asked for are done and tested: the zero-row retry loop, the silently dropped upserts, and the `known?/1` policy-versus-technical confusion. The facade conversion has no stragglers; `get_adaptor/2` and `resolve_package_name/1` have no remaining callers. + +### What a user sees differently + +- New versions appear within one refresh interval, or immediately on a manual refresh, with no page reload in an open editor (about a second after the refresh lands). +- The editor has a loading state ("Loading adaptors…") and a failure state ("Couldn't load adaptors. Please try again." with Retry). Previously the list was just there. +- Only adaptors that actually have a credential schema appear as credential types. Before, the list came from the on-disk schema dump. +- Deprecated adaptors are hidden from the picker and the credential type list. Jobs pinned to one keep validating and running. +- New superuser page, Settings → Maintenance (`/settings/maintenance`, `maintenance_live/index.ex`), with Refresh Adaptor Registry (flashes "Adaptor refresh queued.", fire-and-forget) and Refresh Adaptor Icons (flashes "Icon refresh started." then "Icon refresh complete — N updated, M unchanged."; can take up to two minutes). A non-superuser is redirected to `/projects`. +- New failure message on workflow save during a cold start (§3). +- "Adaptors in this project" in the editor is derived client-side from the open workflow's jobs (`useAdaptors.ts:155-166`). An unsaved adaptor counts; adaptors used elsewhere in the project but not in this workflow no longer appear. +- `GET /images/adaptors/adaptor_icons.json` and the `request_project_adaptors` channel event are gone. + +--- + +## 6. Decisions already taken + +These are settled. They are here so the reasoning survives, and so nobody relitigates them by accident while working §7. + +- **A credential save during a catalogue outage stays quiet and self-healing.** `resolve_schema_name/1` (`credential.ex:112-126`) stores the short name on `{:error, _}` rather than adding a changeset error, because the alternative blocks every credential save whenever the catalogue cannot answer, and the short form is a shape `get_schema/1` and the reconciler already handle. Adding the error also broke an unrelated test that only checks the 40-character length error (`credential_test.exs:100`), a fair preview of how widely it lands. The inconsistency with `Job` is noted in §7.3 and is about the other two surfaces, not this one. +- **`AdaptorService` reports a technical failure as one.** An unreachable catalogue gives `{:error, {:catalogue_unavailable, reason}}`, not `:adaptor_not_permitted`. "Not permitted" reads as a policy decision and sends whoever is debugging a failed install to the allowlist when the real problem is a timeout. +- **The first-load gate is a first-load gate, not a health check.** Once a source has ever loaded, `loaded?` answers true permanently. A catalogue that loaded in March and has silently stopped updating looks exactly as healthy as one refreshed a minute ago. Staleness is a separate concern with its own alerting, and it is not built. +- **The "a cycle completed" flag lives in scheduler memory, not a row.** Losing it on restart costs one refresh, not a loop, so it did not need persisting. §7.1 is about what sets the flag, not where it lives. +- **A zero-row cycle is a legitimate outcome, not a failure, at least for `:local`.** That is what `5f70ab91d2` encodes. Whether it holds for `:npm` is §7.1. +- **The npm org listing is authoritative, not npm search.** Search omits deprecated packages, and we need those to keep already-pinned jobs validating and running. +- **Deprecated adaptors are hidden, not removed.** They stay in the database and stay resolvable. Production has been checked and has no jobs on deprecated adaptors. +- **The icon cache path is set in the Dockerfile, not in bootstrap code.** Anyone operating Kubernetes is expected to know they need a PVC at that path. +- **The refresh interval is configured in seconds.** See `f6325ee457` in §5. + +--- + +## 7. Outstanding items + +Ordered by how much they matter. The first four are design calls for Stu, not bugs to hand straight to an agent; each has enough here to make the call without re-reading the code. + +### 7.1 An empty npm listing latches "loaded" forever + +**Where:** `scheduler.ex:271-272`, `store.ex:269-272`. + +`Scheduler.completed?` flips true on `{:ok, %{listed: 0, errors: 0}}` and never flips back. For npm, `Registry.list_adaptors/0` returns `{:ok, []}` whenever `GET /-/user/openfn/package` answers 200 with a map holding no `@openfn/language-*` keys (`scoped_package_names/0`, `npm/registry.ex:145-161`). A non-200 is `{:error, {:http_status, status}}` and a transport failure is `{:error, reason}`, so those do not latch. But a mistyped `ADAPTORS_NPM_REGISTRY_URL` that lands on a server answering 200 with `{}`, an empty internal mirror, or an npm incident returning an empty body all look exactly like "no adaptors exist". For local, a configured root with no `packages/` directory logs a warning and contributes nothing (`local.ex:131-146`), so a mis-typed `ADAPTORS_LOCAL_REPO` latches the same way; only a completely unset path is an error (`:106-114`). + +After one such tick on an instance with no rows, three things go wrong at once: + +- `GET /adaptors/catalogue` answers 200 with `{"data": []}` instead of 503, so the editor shows an empty picker with no error and no Retry button. +- `Job.validate_known_adaptor` (`job.ex:157-172`) says "is not a recognised adaptor" instead of the retryable "catalogue is not ready yet". `readiness_test.exs:202-209` pins the underlying behaviour: `fetch_adaptor/2` returns `{:error, :not_found}` when the load lists nothing. +- `AdaptorService.install/2` answers `{:error, :adaptor_not_permitted}`, the exact wrong answer `931f16a020` set out to remove, arriving by the other door. + +The tension was flagged before the fix went in: an empty listing is a legitimate outcome for `:local` (the directory really is empty) and essentially never a legitimate one for `:npm`. The latch was written source-agnostic. + +**Options, cheapest first.** + +1. Restrict the latch to `:local`. One guard on the `match?` at `scheduler.ex:272` using `state.source`. npm keeps the old loop behaviour on an empty listing (every read waits 90s and returns `:not_ready`), which is at least visible. Smallest diff; leaves "what does an empty npm listing mean" unanswered. +2. Require the source to have listed something non-empty at least once before the latch can set. Does not help a cold instance against a broken mirror, which is the case that matters. +3. Have the strategies distinguish "listed successfully, nothing there" from "could not list" at the boundary, mirroring what `NPM.Schema` already does for schemas (a 404 is knowledge, a timeout is not). For npm that means deciding whether a 200 with no matching keys is knowledge; the honest answer is probably that it is not, for an org that has hundreds of packages, so npm would return `{:error, :empty_listing}` and never latch. Most consistent with a distinction the codebase already makes; most work, since it touches the strategy behaviour, both implementations and their tests. + +**Related:** `refresh_all/0` in `lib/mix/tasks/lightning.adaptors.refresh.ex:46-48` exits 2 on `{:ok, %{listed: 0}}` with "Refresh completed but the source listed no adaptors." while `Scheduler.completed?` treats the same cycle as a successful first load. Pick one reading, or the operator gets a failing exit code for a state the app has decided is fine. Option 3 resolves this for free. + +**Reproducing it:** point `ADAPTORS_NPM_REGISTRY_URL` at a host that answers 200 with `{}` on `/-/user/openfn/package`, start against an empty database, and watch `GET /adaptors/catalogue` return 200 with an empty list rather than 503. + +### 7.2 Gating the catalogue read puts 90 seconds on paths that used to answer immediately + +**Where:** `store.ex:251-259`, with `catalogue/1` at `:187` and `icon_meta/2` at `:213`. + +Both now go through `gated/2`. On a cold instance whose refresh is failing, `GET /adaptors/catalogue` holds the request for the full 90 seconds before returning 503. Every editor session opened during the cold window holds a request process for a minute and a half. The same wait applies to the credential form's type list (`credential_form_component.ex:1178`), icon requests (`adaptor_icon_controller.ex:85, 123`), the channel's `request_adaptors` handler, and workflow save: `Session.save_workflow/2` calls `ensure_loaded` (`session.ex:387`) and its own `GenServer.call` timeout is `first_load_timeout + 10s` (`:243-247`) precisely so it outlives the gate. + +**Related leak.** When the caller's `GenServer.call` times out first, the scheduler does not know; it still holds the `from` in `waiters` and, when the refresh eventually finishes, `GenServer.reply/2` delivers `{ref, result}` straight into the caller's mailbox. This pre-dates the branch but used to be reachable only from `fetch_adaptor`/`ensure_loaded`; it now reaches every caller of the five gated reads. Where that lands: + +- Controllers: harmless, the request process is gone. +- `Collaboration.Session`: harmless, it has a catch-all `handle_info/2` (`session.ex:443`). +- LiveViews hosting the credential form component: the stray message goes to the parent LiveView. A LiveView that defines other `handle_info/2` clauses and no catch-all raises `FunctionClauseError` and remounts. Which hosts are exposed has not been checked. + +**Options.** + +1. Keep the gate only on the reads where blocking is right (`schema/2`, `fetch_adaptor`, `ensure_loaded`) and let `catalogue/1`, `packages/1`, `icon/3` and `icon_meta/2` return an immediate `{:error, :not_ready}` the controller can turn into a 503 with `Retry-After`, which the editor already renders as a spinner and Retry button. This gives back the pre-`220f7a4240` HTTP behaviour and keeps the crash fix the gate was built for. +2. Keep the gate everywhere but give the HTTP path a much shorter timeout than the boot path, via a per-call timeout on `gated/2`. +3. Either way, fix the waiter leak on its own: have the scheduler drop a waiter whose call has timed out (it can monitor the caller, or the store can pass a deadline), or have callers use a reference they can discard. + +### 7.3 The credential form blocks 90 seconds and then crashes + +**Where:** `lib/lightning/credentials.ex:589-596`. + +`get_schema/1` raises on any `{:error, _}` from `resolve_name/2` or `schema/2`. It is called from `json_schema_body_component.ex:21` and `credential_form_component.ex:653` with no rescue anywhere upstream, so during a cold or unreachable catalogue the credential form waits the full first-load timeout and then the LiveView process crashes: a 500 on an initial mount, a disconnect-and-remount on a connected socket. The crash was a known limitation before the branch's last day; the gate has made it slow as well as ugly. It also fires for a credential whose adaptor is simply not in the catalogue, cold or not. + +Three neighbouring surfaces now disagree about what to do when the catalogue cannot answer: + +- `Lightning.Workflows.Job` (`job.ex:157-172`) refuses with "adaptor catalogue is not ready yet, try again shortly". +- `Credential.resolve_schema_name` (`credential.ex:112-126`) silently stores the short name, and the reconciler repairs it later. Settled, see §6. +- `Credentials.get_schema/1` raises. + +**Options.** Make `get_schema/1` return an error tuple and have both call sites render a "the adaptor catalogue is still loading" state with a retry. That is the smallest change that removes the crash, and it lines the credential form up with the editor's existing loading/error/Retry pattern. What the user sees and reads in that state is a product question for Brandon's team; the error-tuple change itself is not. + +### 7.4 Seeding commands stall for 90 seconds when npm is unreachable + +**Where:** `lib/lightning/setup.ex:45-52`. + +`_ = Lightning.Adaptors.ensure_loaded()` runs on a cold BEAM with an empty catalogue, so `mix lightning.kickstart` (`kickstart.ex:60`), `Lightning.Setup.setup_user/3` (`setup.ex:23`) and `Lightning.Demo.reset_demo/0` (`demo.ex:16`) each block for the first-load timeout and then proceed as if nothing happened. The return value is discarded and nothing logs the pause. + +Concrete case: an airgapped deployer runs `mix lightning.kickstart` before importing a snapshot and waits 90 seconds per invocation with no explanation. + +Two consequences to decide on deliberately, not just fix the stall: + +- These commands now need outbound npm access to be fast. That is a new coupling between seeding and the network. +- On a box where no instance holds the HighlanderPG lock, the mix task acquires it and runs a full refresh, icon downloads included, as a side effect of seeding. + +**Options.** At minimum, log the reason when `ensure_loaded` returns an error so the pause is explained. Better, give the setup path a short timeout of its own (`fetch_adaptor/2` already accepts `opts[:timeout]`, `adaptors.ex:301`; `ensure_loaded/1` would need the same). Or skip the pre-load entirely and accept a lazy lookup inside `fun`. The comment at `setup.ex:47-49` explains why the pre-load is there: to avoid a source fetch inside `fun`'s transaction. Removing it needs that answered, for example by moving the adaptor lookups out of the transaction. + +### 7.5 PR #5077 and this branch have to be sequenced + +Checked on 10 Sep, re-checked 14 Sep: #5077 is still open and this branch does not contain it. Nothing in the branch history records a decision. + +#5077 ("Fix metadata breaking during overlapping or unparseable adaptor installs") fixes two bugs in `AdaptorService`, the on-the-fly npm install that puts packages on the worker pod: a lookup during an in-flight install could see a half-built placeholder whose stored version literal `latest` is not valid semver and raises on comparison; and the editor's metadata channel handler passed `job.adaptor` through unresolved, unlike the worker and AI assistant paths. The fix converts `AdaptorService` from an `Agent` to a `GenServer`, queues overlapping installs, and resolves `latest` in `workflow_channel.ex`. Files: `adaptor_service.ex`, `workflow_channel.ex`, `CHANGELOG.md`, three tests. + +`AdaptorService` is the one part of the old world this branch leaves essentially intact, so the rewrite does not supersede #5077 and both bugs are present here. Six branch commits touch `adaptor_service.ex` (`931f16a020`, `220f7a4240`, `2a6499a523`, `4348ac2d15`, `790087de3f`, `69f6595ed2`), and `931f16a020` changed `install/2`'s error handling, the same function #5077 restructures. Whichever lands second gets a conflict in `install/2`. + +Also relevant: #4801 targets `release-2.19.0` but the branch is based on `main` and `release-2.19.0` is not an ancestor of `HEAD`. Merging as-is would carry `main`'s commits into the release branch. Rebase onto `release-2.19.0` (which drops the five cherry-picked commits in §12 as duplicates) or retarget the PR; decide together with the #5077 order. + +### 7.6 Concurrent schema fetches are not coalesced across nodes + +Raised on 11 Sep, never built. `Store.read_schema/2` uses `Cachex.fetch/4`, which coalesces concurrent callers for the same key on one node, so two callers on the same node wanting the same schema do share one fetch. There is no coalescing across nodes and no in-flight registry in the scheduler or `NPM.Schema`. Not a defect; it is the obvious next thing if the cold-start window turns out to be painful in practice. + +### 7.7 The CHANGELOG undersells the upgrade + +**Where:** `CHANGELOG.md:46-48`, under `## [Unreleased]` → `### Changed`. + +One entry: "Lightning now keeps its own adaptor registry instead of fetching the list from npm at startup, so new adaptors and versions show up without a rebuild or redeploy. See ADAPTORS.md. #4801". It says nothing about `SCHEMAS_PATH` and `ADAPTORS_REGISTRY_JSON_PATH` being removed, the nine new `ADAPTORS_*` variables, the `_MS` → `_SECONDS` rename from `f6325ee457`, the deleted `install_schemas` / `install_adaptor_icons` tasks, the three migrations, or the icon volume. + +Anyone upgrading from the CHANGELOG alone will not learn they have environment variables to delete. This needs writing before the branch merges. + +### 7.8 Smaller known limitations, carried forward + +Not blockers, but they belong in whatever goes to Services: + +- **Up to an hour of staleness by default.** Workaround: Settings → Maintenance, or `mix lightning.adaptors.refresh`. +- **The icon cache needs a persistent volume.** The official image and compose handle it; any other deployment must mount storage at `ADAPTORS_ICONS_PATH` or icons re-download on every restart. Outside the image the default sits under the system temp directory, which most container platforms wipe. +- **The credential type list degrades silently.** `get_type_options/0` (`credential_form_component.ex:1176-1193`) returns `[]` for the adaptor block on `{:error, _}`, so the "new credential" list collapses to Raw JSON plus whatever OAuth clients exist, with no error shown. +- **Schemas missing upstream stop being retried after an hour.** See §2, "How the scheduler treats a missing schema". Workaround: `mix lightning.adaptors.refresh --name `. +- **`--name` refresh is the blunt instrument.** It bypasses change detection and takes upstream literally, so it will clear a stored schema if upstream now reports none. Intended, but it can remove data a periodic tick would have preserved. +- **Local mode shows version `local`** in the picker for every adaptor (`store.ex:341-342`), though the database holds the real semver. +- **Object-typed credential schema fields render as a code area**, not a structured form (`json_schema_body_component.ex:113`). +- **Hardcoded exclusions.** `@openfn/language-devtools`, `-template`, `-fhir-jembi` and `-collections` are never listed, along with anything npm marks deprecated. Changing the list needs a code change. +- **Multi-node partition.** A node that misses a change broadcast serves stale data until it sees the other node return, then re-warms from Postgres. Scheduler failover on leader death takes a few seconds. + +--- + +## 8. Questions that belong to other teams + +- **Release version.** Which Lightning release does this ship in? The PR targets `release-2.19.0` and the CHANGELOG entry sits under Unreleased. Brandon's team owns the answer. +- **Customer impact.** Which hosted or client deployments currently set `SCHEMAS_PATH` or `ADAPTORS_REGISTRY_JSON_PATH`, or use `LOCAL_ADAPTORS` / `OPENFN_ADAPTORS_REPO`? Those env files need editing before upgrade. Not visible from the repo; Aleksa's team for the client picture. +- **Which deployments need persistent storage at `ADAPTORS_ICONS_PATH`?** A per-deployment decision. Nahrek's team for the infrastructure side. +- **Are there airgapped or offline customers today?** If so, someone owns building and distributing snapshot files and a refresh cadence for them. Aleksa's team. +- **Public docs.** Is docs.openfn.org being updated? ADAPTORS.md is in-repo only and is the only written record of the offline procedure. Jack's team. +- **First-run timing on a real production network.** The concurrency limits are clear but the wall-clock time for a cold first refresh against npm from a production region has not been measured. Time it during UAT and record it; it directly informs whether 90 seconds is the right `first_load_timeout`. Whoever runs UAT. + +--- + +## 9. How to check your work + +```sh +mix test test/lightning/adaptors # 340 tests, the subsystem's own; 0 failures on 14 Sep +mix test # full suite +mix verify # format, credo, dialyzer, sobelow +``` + +Known flakes unrelated to this work: `FifoRunQueueTest` and `RunsTest` claim-ordering, both clean in isolation. + +The subsystem compiles under `warnings_as_errors`, so a warning is a build failure. + +### Driving a controlled upstream + +Real npm means waiting for real upstream events. For anything needing a controlled upstream (a new version, a new adaptor, an icon change, a registry that is down) use the record-and-replay proxy in `tooling/adaptor_cache/` (README there): + +```sh +bin/adaptor_cache up # prints the three ADAPTORS_NPM_*_URL exports +bin/adaptor_cache logs # watch cache=HIT / MISS / ERROR per request +bin/adaptor_cache publish @openfn/language-http 9.9.9 +bin/adaptor_cache down # simulate an unreachable registry +``` + +It also has `status`, `purge`, `check` and `scenario save/restore`. The cache directory defaults to `/tmp/adaptor_cache` (`ADAPTOR_CACHE_DIR`). `publish` deliberately updates both the packument and the search response's `latest_version`. Hand-editing only one is a silent no-op, because change detection diffs the search response against the database. + +### Manual refresh paths + +```sh +mix lightning.adaptors.refresh # full refresh, waits, prints counts +mix lightning.adaptors.refresh --name @openfn/language-http # one adaptor, ignores change detection +``` + +Exit codes (`lightning.adaptors.refresh.ex`): full refresh `0` on success, `2` if the cycle succeeded but listed nothing, on timeout, or on any other error; `--name` gives `0` on success, `1` if the adaptor does not exist, `2` on any other error. The task accepts only `--name`; there is no `--strategy` or `--source`. + +On a release image: `bin/lightning rpc 'Lightning.Adaptors.refresh(await: true)'` and `bin/lightning rpc 'Lightning.Adaptors.refresh_package("@openfn/language-http")'`. + +In the UI: Settings → Maintenance, superuser only (§5). + +--- + +## 10. Scenarios worth running by hand + +Condensed from the UAT plan written for Services on 10 Sep, corrected for the 14 Sep changes. Expected values are what the code does today. + +1. **Fresh install.** Empty DB, network up, defaults. Start; confirm pages serve immediately; logs show `Adaptors[npm]: catalogue is empty at boot — refreshing now`, then `scheduler started interval=3600000ms next_tick_in=0ms`, then a tick summary with `listed` in the low hundreds and `errors=0`. Picker lists adaptors with icons. `SELECT count(*) FROM adaptors` and `adaptor_versions` non-zero. `GET /adaptors/catalogue` returns 200 with entries carrying `name`, `latest_version`, `versions`, `repository`, `icon_urls`. Files under `$ADAPTORS_ICONS_PATH/npm//square..png`. Saving a workflow before the first refresh finishes waits up to 90s, then says "The adaptor catalogue is still loading. Try again shortly." That is correct behaviour. +2. **New version upstream.** `bin/adaptor_cache publish @openfn/language-http 9.9.9`, leave an editor open, `mix lightning.adaptors.refresh`. Refresh reports `changed=1 fetched=1`; `9.9.9` appears in the open picker without a reload; `latest_version` in the DB shows `9.9.9`; the catalogue ETag changes. +3. **New adaptor upstream.** `publish @openfn/language-brand-new 1.0.0`, refresh. Appears in the picker with a grey placeholder icon (`icon_urls` null); row count up by one. +4. **Icon change.** Replace the recorded icon bytes under the cache dir, refresh (or Maintenance → Refresh Adaptor Icons). Catalogue gives a new `sha8`; the old URL 302s to the new one with `no-store`; the new URL is 200 with the immutable headers; `icon_square_sha256` changed; summary shows `icons=` or `healed=` non-zero. An icon-only change is picked up even when the version did not bump; that is the `healed` counter. +5. **Schema change.** Edit the recorded `configuration-schema.json`, publish a version bump, refresh, reopen an existing credential of that type. The form shows the new field; the stored body is untouched. `schema_sha256` on the row changed. `SELECT schema FROM credentials` shows full package names (the reconciler). An adaptor with no schema does not appear in the credential type list but does appear in the picker; its `package_meta.has_schema` is false. +6. **Legacy schema names.** On an instance upgraded from a pre-branch version, `SELECT DISTINCT schema FROM credentials` shows short names before and full names after the first refresh. Unresolvable names are left alone. No audit rows. +7. **Registry unreachable.** (A) Populated catalogue, `bin/adaptor_cache down`, refresh: `list_adaptors failed` warning, `errors=1 changed=0`, task exits 2, everything keeps serving, row counts and `updated_at` unchanged; bring it back, clean tick. (B) Empty catalogue, no network: pages serve; save is rejected after up to 90s; `GET /adaptors/catalogue` is 503 after up to 90s; editor shows the load error with Retry. (C) One broken packument: `fetch_adaptor() failed`, `errors=1`, rest lands. (D) GitHub only down: metadata lands, `fetch_icons failed … persisting records without icons`. +8. **Manual refresh via UI and CLI.** §9. Negative test: a non-superuser at `/settings/maintenance` is redirected to `/projects` with a no-access flash and sees no Maintenance sidebar entry. +9. **Offline install.** §4. Deliberate failure to check: skip the icon tarball and confirm the catalogue populates but icons are missing. +10. **Local mode.** `ADAPTORS_STRATEGY=local`, `ADAPTORS_LOCAL_REPO=/path/to/adaptors`. Only checked-out adaptors appear, all showing version `local` in the picker; `SELECT source, latest_version FROM adaptors` shows `local` as the source and real semvers. Schemas and icons are read live from the checkout, so editing a schema needs only a refresh. Two roots: first wins, the log names shadowed packages. Negative: no path at all fails boot naming both variables. +11. **Deprecated adaptor.** Absent from the picker, the credential type list and `GET /adaptors/catalogue`; present in the table with `deprecated = true`; a job pinned to it still validates, saves and runs. +12. **Deprecated env vars.** `LOCAL_ADAPTORS=true` and `OPENFN_ADAPTORS_REPO` with none of the new variables: works as local mode and logs both deprecation warnings. +13. **Two nodes.** Only one logs refresh ticks; kill it and the other takes over within seconds; the follower sees the leader's changes; both nodes return the same catalogue. +14. **Catalogue caching.** 200 with `etag`, `cache-control: private, no-cache`, `vary: Cookie`; 304 on `If-None-Match`; new etag after a change; 401 `{"error": "Unauthorized"}` without a cookie. + +--- + +## 11. Loose threads + +- A `mix release` failure in dev mode ("Could not read configuration file... functions, references, and pids... `LightningWeb.Endpoint`") turned up on 11 Sep while reproducing an unrelated bug. It looks like a pre-existing `config/dev.exs` release-config problem rather than anything to do with adaptors, and it has not been confirmed fixed. +- `test/lightning/adaptor_service_test.exs:1-6` still has a moduledoc describing `AdaptorService.known?/1`, which `931f16a020` deleted. Stale documentation, not a failing test. +- `workflow_channel.ex:109-117` keeps a `request_adaptors` handler with no caller in `assets/js` other than the channel test and a test-helper README. If it is dead, delete it; if it is kept, note that it hides `{:error, _}` as an empty list (`:993-997`), the same shape §7.1 is about. +- `Registry.scoped_package_names/0` has no distinct branch for a 200 whose body is not a map; it falls through to `{:error, {:http_status, 200}}`. Harmless, just a confusing error to read in a log. + +## 12. Unrelated work carried on the branch + +Ten of the 41 commits are not adaptor work. Five are already on `release-2.19.0` and will drop out as duplicates on a rebase onto it: `f946809b0f` (sandbox merge collections fix, #5054), `ecc0cecca2` and `b0fabc55b9` and `f01b2beb6a` (AI assistant changes, #5161, #5143, #5155), `605a3bb594` (special characters in workflow, step and credential names, #5106). Five are branch-only and unrelated: `34a73347a3` (docs-style rule and skills), `67fce6335b` (locale pin for the JS test run), `dffd89e67b` (GitHubSyncModal test), `4d6b01ed0b` (sobelow 0.15.0), `43ee0c8fe1` (session store teardown in tests). None of them is part of this story; the first five have their own CHANGELOG entries. diff --git a/ADAPTORS.md b/ADAPTORS.md new file mode 100644 index 00000000000..5ceb1872642 --- /dev/null +++ b/ADAPTORS.md @@ -0,0 +1,118 @@ +# Adaptors + +The adaptor registry is the catalogue of adaptors, versions, credential schemas +and icons that the workflow editor shows. Lightning fetches it from npm by +default and keeps a copy in Postgres. For the Elixir side, start at +`Lightning.Adaptors`. + +## Using local adaptors + +Point Lightning at a checkout of the adaptors monorepo instead of npm: + +```sh +ADAPTORS_STRATEGY=local +ADAPTORS_LOCAL_REPO=/path/to/adaptors +``` + +The path is the repository root, not its `packages/` directory. Every +subdirectory of `packages/` with a `package.json` becomes an adaptor, named and +versioned from that file. Lightning also reads `configuration-schema.json` for +the credential form, and `assets/square` and `assets/rectangle` as `.png` or +`.svg` for the icons. A package without them still appears, with no credential +form or icon. + +To layer a private checkout over the public one, give more than one root, comma +separated: + +```sh +ADAPTORS_LOCAL_REPO=/path/to/private-adaptors,/path/to/adaptors +``` + +A package found in more than one root comes from the first root only. Lightning +logs a warning naming each shadowed package on every catalogue scan, not just at +boot. + +Lightning still accepts the old names for these two settings, +`LOCAL_ADAPTORS=true` and `OPENFN_ADAPTORS_REPO`. Each warns at boot only when +Lightning falls back to it: `LOCAL_ADAPTORS=true` when `ADAPTORS_STRATEGY` is +unset, `OPENFN_ADAPTORS_REPO` when the strategy is local and +`ADAPTORS_LOCAL_REPO` is not set. + +## Running without internet access + +Copy the catalogue from an instance that has internet access and has finished a +refresh. On that instance, dump the catalogue and archive the icons directory. +The dump holds the icon metadata only, so the icon files have to travel with it: + +```sh +mix lightning.adaptors.dump --path snapshot.json +tar czf icons.tar.gz -C "$ADAPTORS_ICONS_PATH" . +``` + +If `ADAPTORS_ICONS_PATH` is not set, the directory is `lightning/adaptor_icons` +under the system temp directory. + +On the offline instance, unpack the icons where `ADAPTORS_ICONS_PATH` points, +then import the snapshot: + +```sh +mkdir -p "$ADAPTORS_ICONS_PATH" +tar xzf icons.tar.gz -C "$ADAPTORS_ICONS_PATH" +mix lightning.adaptors.import --path snapshot.json --replace +``` + +The `mix` commands need a source checkout. A release image has no Mix, so import +there with: + +```sh +bin/lightning eval 'Lightning.Release.seed_adaptors("/path/to/snapshot.json", replace: true)' +``` + +With no populated instance to dump from, build the snapshot straight from npm on +any machine with internet access. This needs no database and carries no icons: + +```sh +mix lightning.adaptors.snapshot --path snapshot.json +``` + +Import it as above. + +If you run internal mirrors instead, any npm-compatible registry works. Set +`ADAPTORS_NPM_REGISTRY_URL`, `ADAPTORS_NPM_JSDELIVR_URL` and +`ADAPTORS_NPM_GITHUB_URL`, and leave the strategy as npm. Set +`ADAPTORS_NPM_GITHUB_REF` too if the mirror serves a branch other than `main`. + +An imported catalogue survives the hourly refresh. When the refresh cannot reach +its source it logs a warning and leaves the existing rows alone. + +The worker installs adaptor packages into `ADAPTORS_PATH` on its own. That is a +separate download and none of the above provides it. + +## Keeping the catalogue fresh + +Lightning refreshes the catalogue from its source every hour. To force a refresh +now, on a source checkout: + +```sh +mix lightning.adaptors.refresh +mix lightning.adaptors.refresh --name @openfn/language-http +``` + +Without `--name` it runs a full refresh and waits for it to finish. With +`--name` it refetches that one adaptor, whether or not its version changed. Exit +codes are in the task's `mix help` output. + +## Troubleshooting + +- An adaptor is missing from the picker: look for a `fetch_adaptor` warning + naming it in the log, then force a refresh with `--name`. +- A new version is not showing: the hourly refresh has not run yet. Force one, + or wait for the next. +- Icons are missing after an import: the icons directory never reached + `ADAPTORS_ICONS_PATH` on this instance, or the dump came from a Lightning + version that did not write icon metadata. Redo the dump and copy the + directory. +- A local package is ignored: an earlier root in `ADAPTORS_LOCAL_REPO` has a + package with the same name. The log names each shadowed package. +- A boot warning says a variable is deprecated: rename `LOCAL_ADAPTORS=true` to + `ADAPTORS_STRATEGY=local` and `OPENFN_ADAPTORS_REPO` to `ADAPTORS_LOCAL_REPO`. diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index bb723481c2d..e71a22eeede 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -186,7 +186,14 @@ For SMTP, the following environment variables are required: | **Variable** | Description | | ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ADAPTORS_ICONS_PATH` | Directory the on-disk adaptor icon cache is written to. Defaults to a subdirectory of the system temp directory, which most container platforms wipe on restart; point it at a persistent volume so cached icons survive a restart. | +| `ADAPTORS_LOCAL_REPO` | Path to an OpenFn adaptors checkout, used when `ADAPTORS_STRATEGY` is `local`. Comma-separate several. The first checkout that has a package wins. See [Adaptors](ADAPTORS.md). | +| `ADAPTORS_NPM_GITHUB_REF` | Git ref of `OpenFn/adaptors` that adaptor icons are read from. Defaults to `main`. | +| `ADAPTORS_NPM_GITHUB_URL` | Raw GitHub host that adaptor icons are fetched from. Defaults to `https://raw.githubusercontent.com`. | +| `ADAPTORS_NPM_HTTP_TIMEOUT` | Receive timeout in milliseconds for npm registry, schema and icon requests. Defaults to `30000`. | +| `ADAPTORS_NPM_JSDELIVR_URL` | CDN that adaptor credential schemas are fetched from. Defaults to `https://cdn.jsdelivr.net`. | +| `ADAPTORS_NPM_REGISTRY_URL` | npm registry the adaptor catalogue is read from. Defaults to `https://registry.npmjs.org`. Set this and the two URLs above to use an internal mirror. See [Adaptors](ADAPTORS.md). | | `ADAPTORS_PATH` | Where you store your locally installed adaptors | +| `ADAPTORS_STRATEGY` | Where the adaptor catalogue comes from: `npm` (default) or `local`. See [Adaptors](ADAPTORS.md). | | `ALLOW_SIGNUP` | Set to `true` to enable user access to the registration page. Set to `false` to disable new user registrations and block access to the registration page.
Default is `false`. | | `CORS_ORIGIN` | A list of acceptable hosts for browser/cors requests (',' separated) | | `DISABLE_DB_SSL` | In production, the use of an SSL connection to Postgres is required by default.
Setting this to `"true"` allows unencrypted connections to the database. This is strongly discouraged in a real production environment. | @@ -222,7 +229,6 @@ For SMTP, the following environment variables are required: | `PROMEX_UPLOAD_GRAFANA_DASHBOARDS_ON_START` | Instructs PromEx to upload iniital dashboards to a Grafana instance if set to 'true' or 'yes'. Defaults to false. | | `PRIMARY_ENCRYPTION_KEY` | A base64 encoded 32 character long string.
See [Encryption](#encryption). | | `QUEUE_RESULT_RETENTION_PERIOD_MINUTES` | The number of minutes to keep completed (successful) `ObanJobs` in the queue (not to be confused with runs and/or history) | -| `SCHEMAS_PATH` | Path to the credential schemas that provide forms for different adaptors | | `SECRET_KEY_BASE` | A secret key used as a base to generate secrets for encrypting and signing data. | | `SENTRY_DSN` | If using Sentry for error monitoring, your DSN | | `URL_HOST` | The host used for writing URLs (e.g., `demo.openfn.org`) | diff --git a/RUNNINGLOCAL.md b/RUNNINGLOCAL.md index f1fca559c66..01dcde32f67 100644 --- a/RUNNINGLOCAL.md +++ b/RUNNINGLOCAL.md @@ -140,100 +140,18 @@ you. [Learn more about configuring workers](WORKERS.md) -### Using Local Adaptors +### Using local adaptors -You can force lightning to use adaptor builds from your local -[adaptors](https://github.com/openfn/adaptors) repo. - -Note that this is a global toggle: ALL runs will use local adaptor versions, and -the adaptor picklist in the Workflow Editor will only suggest adaptors present -in the monorepo. - -Remember to re-build your adaptors after making changes (use -`pnpm build --watch` in the monorepo). - -To start, set up the following environment variables: - -- `LOCAL_ADAPTORS`: Used to enable or disable the local adaptors mode. Set it to - `true` to enable. -- `OPENFN_ADAPTORS_REPO`: This should point to the adaptors monorepo. This is - the same variable used when you pass `-m` to the CLI. It also accepts a - comma-separated list of paths to merge multiple repos into the registry; the - first path wins on dirname collisions, with a warning logged for shadowed - entries. Both the registry view and the bundled `ws-worker` resolve `@local` - adaptors against the same list, so a workflow run picks up the same package - the picker shows. - -Example configuration: - -```sh -export LOCAL_ADAPTORS=true -export OPENFN_ADAPTORS_REPO=/path/to/repo/ - -# Or, merge a private adaptor repo with the canonical one (first wins): -export OPENFN_ADAPTORS_REPO=/path/to/private,/path/to/canonical -``` - -You can also run the server directly in local mode with: - -```sh -LOCAL_ADAPTORS=true mix phx.server -``` - -Each path in `OPENFN_ADAPTORS_REPO` must contain a `packages` subdirectory. -Paths that are missing or unreadable are logged and skipped, so the rest of the -list still loads. - -#### Credential schemas in local mode - -Credential schemas (used by the credential form's type picker and validation) -are installed separately from the adaptor registry, into `priv/schemas/`. - -By default `mix lightning.install_schemas` will download schemas from the npm -registry. - -Set LOCAL_ADAPTORS to true and `install_schemas` will read each package's -`configuration-schema.json` from the monorepo. - -```sh -LOCAL_ADAPTORS=true mix lightning.install_schemas -``` - -This clears and repopulates `priv/schemas/` from the local repo(s) ONLY. Re-run -it after adding or changing a `configuration-schema.json` to get the latest -changes. Packages without a schema are skipped. - -Remember to re-generate the production schemas when you've finished, or else -your local app will use the local schema versions until `install_schemas` is -next run! +To run Lightning against your own checkout of the +[adaptors](https://github.com/openfn/adaptors) repo, see +[ADAPTORS.md](ADAPTORS.md). ### Caching the adaptor upstreams -Every adaptor registry refresh (background scheduler tick, or a manual -`mix lightning.adaptors.refresh`) makes a handful of npm, jsDelivr and GitHub -requests per changed package, which gets chatty fast if you're iterating on the -subsystem or just running `lightning.adaptors.refresh` repeatedly by hand. A -local record-and-replay reverse proxy under `tooling/adaptor_cache/` makes the -second and every later run local, with no network needed at all. - -```sh -bin/adaptor_cache up # start the proxy -bin/adaptor_cache check # prove all three upstreams cache correctly -``` - -Then point Lightning at it: - -```sh -export ADAPTORS_NPM_REGISTRY_URL=http://localhost:4874/npm -export ADAPTORS_NPM_JSDELIVR_URL=http://localhost:4874/jsdelivr -export ADAPTORS_NPM_GITHUB_URL=http://localhost:4874/github -``` - -See `tooling/adaptor_cache/README.md` for the full command list, how to read the -cache logs, and the caveats. **Never** put this URL in your global `~/.npmrc`: -it proxies npm closely enough that `npm install` would appear to work, while -resolving against a week-stale packument and writing `localhost:4874` into -`package-lock.json`. +For a record-and-replay proxy in front of npm, jsDelivr and GitHub while +developing, see `tooling/adaptor_cache/README.md`. The `ADAPTORS_NPM_*` +variables that point Lightning at it are described in +[ADAPTORS.md](ADAPTORS.md). ### Problems with Apple Silicon diff --git a/assets/js/collaborative-editor/components/AdaptorIcon.tsx b/assets/js/collaborative-editor/components/AdaptorIcon.tsx index 918c157dbe3..79d6ab96629 100644 --- a/assets/js/collaborative-editor/components/AdaptorIcon.tsx +++ b/assets/js/collaborative-editor/components/AdaptorIcon.tsx @@ -14,9 +14,10 @@ const sizeClasses = { lg: 'h-12 w-12', }; -// Reads the square icon URL for `name` directly from StoreContext, so callers -// that mock the `hooks/useAdaptors` module (e.g. FullScreenIDE tests) still get -// the existing placeholder fallback instead of crashing on a missing mock. +// Reads the icon URL directly from StoreContext instead of via the +// useAdaptorIconUrl hook in hooks/useAdaptors, so tests that mock that whole +// module (e.g. FullScreenIDE) fall back to the placeholder instead of +// crashing on a missing export. function useStoreIconUrl(name: string): string | null { const context = useContext(StoreContext); const adaptorStore = context?.adaptorStore ?? null; diff --git a/assets/js/collaborative-editor/hooks/useChannel.ts b/assets/js/collaborative-editor/hooks/useChannel.ts index 822db55e4b0..7fe4f6b7658 100644 --- a/assets/js/collaborative-editor/hooks/useChannel.ts +++ b/assets/js/collaborative-editor/hooks/useChannel.ts @@ -47,6 +47,7 @@ export interface ChannelError { * - optimistic_lock_error: Concurrent modification conflict (stale lock_version) * - limit_error: Usage limit exceeded (AI assistant, runs, etc.) * - nesting_too_deep: Sandbox nesting depth limit exceeded + * - adaptor_catalogue_unavailable: Adaptor catalogue hasn't loaded yet, retry shortly * * Optional for the same reason as `errors`. */ diff --git a/assets/test/collaborative-editor/components/AdaptorIcon.test.tsx b/assets/test/collaborative-editor/components/AdaptorIcon.test.tsx index f50f0484d01..5310d5190f7 100644 --- a/assets/test/collaborative-editor/components/AdaptorIcon.test.tsx +++ b/assets/test/collaborative-editor/components/AdaptorIcon.test.tsx @@ -1,9 +1,9 @@ /** * Tests for AdaptorIcon component * - * Verifies that icon URLs are read from the AdaptorStore (icon_urls.square) - * with the existing first-letter placeholder fallback when no URL is present - * or the adaptor is not in the store. + * Verifies that icon URLs are read from the AdaptorStore (icon_urls.square), + * falling back to a first-letter placeholder when no URL is present or the + * adaptor is not in the store. */ import { render, screen } from '@testing-library/react'; diff --git a/assets/test/workflow-diagram/components/MiniMapNode.test.tsx b/assets/test/workflow-diagram/components/MiniMapNode.test.tsx index 8aa20964a22..64cc1425aca 100644 --- a/assets/test/workflow-diagram/components/MiniMapNode.test.tsx +++ b/assets/test/workflow-diagram/components/MiniMapNode.test.tsx @@ -3,7 +3,7 @@ * * Verifies that the minimap renders job icons sourced from the AdaptorStore * via useAdaptorIconUrl, with the rect-only placeholder fallback when the - * URL is null. Trigger rendering is unaffected. + * URL is null. */ import { render } from '@testing-library/react'; diff --git a/assets/test/workflow-diagram/nodes/Job.test.tsx b/assets/test/workflow-diagram/nodes/Job.test.tsx index 4c853a382d5..6eefc9ff5e0 100644 --- a/assets/test/workflow-diagram/nodes/Job.test.tsx +++ b/assets/test/workflow-diagram/nodes/Job.test.tsx @@ -2,8 +2,8 @@ * Job Node Component Tests * * Verifies that job nodes read their adaptor icons from the AdaptorStore - * via useAdaptorIconUrl, with graceful string-label fallback when the URL - * is null OR no StoreProvider is mounted (LiveView workflow-editor path). + * via useAdaptorIconUrl, falling back to a string label when the URL is + * null or no StoreProvider is mounted (the LiveView workflow-editor path). */ import { render } from '@testing-library/react'; diff --git a/bin/e2e.d/manager b/bin/e2e.d/manager index 7e0de5eb987..51c59102541 100755 --- a/bin/e2e.d/manager +++ b/bin/e2e.d/manager @@ -225,8 +225,8 @@ warm_npm_cache() { if run_in_project_root "elixir --name $client_node --rpc-eval $server_node ' try do - adaptors = Lightning.AdaptorRegistry.all() - IO.puts(\"✅ NPM cache warmed successfully - loaded #{length(adaptors)} adaptors\") + :ok = Lightning.Adaptors.ensure_loaded() + IO.puts(\"✅ Adaptor catalogue loaded\") rescue error -> IO.puts(\"⚠️ Warning: Failed to warm NPM cache: #{inspect(error)}\") diff --git a/config/config.exs b/config/config.exs index 59c738b63dc..6c5f0388ec4 100644 --- a/config/config.exs +++ b/config/config.exs @@ -91,7 +91,7 @@ config :oauth2, adapter: Tesla.Adapter.Hackney # hackney 4 negotiates HTTP/2 via ALPN by default, where 1.x was HTTP/1.1 only. # Concurrent requests to one host then multiplex onto a single connection, so # retiring that connection fails every in-flight request at once -- around a -# quarter of the fetches in `mix lightning.install_schemas`. Pinned to HTTP/1.1 +# quarter of a bulk schema fetch when this was first seen. Pinned to HTTP/1.1 # to keep the transport hackney 1.25 used; revisit as a deliberate change if we # want h2 multiplexing. config :hackney, default_protocols: [:http1] diff --git a/config/dev.exs b/config/dev.exs index 4d55304f67c..bec7ce6b379 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -37,8 +37,6 @@ config :lightning, LightningWeb.Endpoint, ] config :lightning, - schemas_path: "priv/schemas", - adaptor_icons_path: "priv/static/images/adaptors", repo_connection_signing_secret: "39h9Qr6+v2wgzjlh4xQoJ90aDe+LY7qIvA5v7QLsTwIwGDfs8el9Z0oFk2Ege33E" diff --git a/config/prod.exs b/config/prod.exs index 409da93767f..be04af24e7e 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -28,8 +28,4 @@ config :phoenix, :filter_parameters, [ "token" ] -config :lightning, - schemas_path: "priv/schemas", - adaptor_icons_path: "priv/static/images/adaptors" - config :lightning, :claim_work_mem, "32MB" diff --git a/config/test.exs b/config/test.exs index 5f63d5036c1..574a8ab96d4 100644 --- a/config/test.exs +++ b/config/test.exs @@ -10,8 +10,6 @@ config :tesla, adapter: Lightning.Tesla.Mock config :tesla, Lightning.AuthProviders.OauthHTTPClient, adapter: Lightning.AuthProviders.OauthHTTPClient.Mock -config :tesla, Mix.Tasks.Lightning.InstallAdaptorIcons, adapter: Tesla.Mock - config :tesla, Lightning.UsageTracking.Client, adapter: Tesla.Mock config :tesla, Lightning.UsageTracking.GithubClient, adapter: Tesla.Mock @@ -126,8 +124,6 @@ config :lightning, Lightning.FailureAlerter, rate_limit: 3 config :lightning, - schemas_path: "test/fixtures/schemas", - adaptor_icons_path: "test/fixtures/adaptors/icons", repo_connection_signing_secret: "39h9Qr6+v2wgzjlh4xQoJ90aDe+LY7qIvA5v7QLsTwIwGDfs8el9Z0oFk2Ege33E" diff --git a/lib/lightning/adaptor_registry.ex b/lib/lightning/adaptor_registry.ex deleted file mode 100644 index ce3b06ab513..00000000000 --- a/lib/lightning/adaptor_registry.ex +++ /dev/null @@ -1,19 +0,0 @@ -defmodule Lightning.AdaptorRegistry do - @moduledoc """ - Holds `local_adaptors_enabled?/0`, still read by - `mix lightning.install_schemas` to decide whether to read credential - schemas from a local adaptors repo instead of npm. - """ - - @doc """ - Whether `Lightning.Config.adaptor_registry/0` has at least one local - adaptors repo configured (`LOCAL_ADAPTORS`/`OPENFN_ADAPTORS_REPO`). - """ - @spec local_adaptors_enabled?() :: boolean() - def local_adaptors_enabled? do - case Lightning.Config.adaptor_registry()[:local_adaptors_repos] do - [_ | _] -> true - _ -> false - end - end -end diff --git a/lib/lightning/adaptor_service.ex b/lib/lightning/adaptor_service.ex index 2c3cf3978eb..8372311bb54 100644 --- a/lib/lightning/adaptor_service.ex +++ b/lib/lightning/adaptor_service.ex @@ -10,6 +10,9 @@ defmodule Lightning.AdaptorService do The service requires at least `:adaptors_path`, which is used to both query which adaptors are installed and when to install new adaptors. + Another optional setting is `:repo`, which must point at a module that + does the actual querying and installing. + ## Installing Adaptors Using the `install/2` function an adaptor can be installed, which will also diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index e0fc35ef2fc..44b7ecbe3a2 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -289,6 +289,8 @@ defmodule Lightning.Adaptors do """ @spec refresh(atom(), keyword()) :: :ok | {:ok, Scheduler.refresh_counts()} | {:error, term()} + def refresh(opts) when is_list(opts), do: refresh(@sup, opts) + def refresh(sup \\ @sup, opts \\ []) do scheduler = AdaptorsSupervisor.global_scheduler_name(sup) diff --git a/lib/lightning/adaptors/icon_cache.ex b/lib/lightning/adaptors/icon_cache.ex index 22147f68139..852e284f2a3 100644 --- a/lib/lightning/adaptors/icon_cache.ex +++ b/lib/lightning/adaptors/icon_cache.ex @@ -3,14 +3,14 @@ defmodule Lightning.Adaptors.IconCache do Pure filesystem helper owning the on-disk adaptor icon cache. Not a GenServer. Three stateless functions over - `Lightning.Adaptors.Config.icon_path/0`, which resolves the - `{:tmp, suffix}` default at call time. + `Lightning.Adaptors.Config.icon_path/0`, which returns `ADAPTORS_ICONS_PATH` + when set and otherwise resolves the `{:tmp, suffix}` default at call time. Disk layout is **source-partitioned** and **latest-only**: ///. - Source partitioning means flipping `LOCAL_ADAPTORS` between restarts + Source partitioning means flipping `ADAPTORS_STRATEGY` between restarts cannot accidentally serve `:npm` bytes from a row that's now resolved via `:local` (or vice versa). Latest-only means a subsequent `write!/5` for the same key overwrites — content-addressable URLs diff --git a/lib/lightning/adaptors/local.ex b/lib/lightning/adaptors/local.ex index 26b7bb0668d..558401e2a1c 100644 --- a/lib/lightning/adaptors/local.ex +++ b/lib/lightning/adaptors/local.ex @@ -3,8 +3,8 @@ defmodule Lightning.Adaptors.Local do Filesystem implementation of `Lightning.Adaptors.Strategy`. Serves adaptor metadata, schemas, and icons from an on-disk OpenFn - adaptors monorepo checkout. Gated by `LOCAL_ADAPTORS=true` and - `OPENFN_ADAPTORS_REPO=/path/to/adaptors` at the runtime-config layer; + adaptors monorepo checkout. Selected by `ADAPTORS_STRATEGY=local` with + `ADAPTORS_LOCAL_REPO=/path/to/adaptors` at the runtime-config layer; this module only reads the resolved paths via `Lightning.Adaptors.Config.strategy_opts(__MODULE__)[:paths]`, an ordered list of root directories. @@ -108,7 +108,7 @@ defmodule Lightning.Adaptors.Local do paths when paths in [nil, []] or not is_list(paths) -> Logger.warning( "Lightning.Adaptors.Local: :paths is not configured " <> - "(set OPENFN_ADAPTORS_REPO or :lightning, Lightning.Adaptors.Local, paths:)" + "(set ADAPTORS_LOCAL_REPO or :lightning, Lightning.Adaptors.Local, paths:)" ) {:error, :no_repo_path} @@ -120,8 +120,7 @@ defmodule Lightning.Adaptors.Local do # First occurrence wins: each root is grouped (highest semver) on its own # first, then a name found in more than one root keeps only its earliest - # root's record. Matches install_schemas.ex:171-172's precedence for the - # same rule. + # root's record. defp discover_paths(paths) do paths |> Enum.flat_map(&scan_root/1) diff --git a/lib/lightning/adaptors/npm/github.ex b/lib/lightning/adaptors/npm/github.ex index 1e2a8096cd4..30f7fb5678f 100644 --- a/lib/lightning/adaptors/npm/github.ex +++ b/lib/lightning/adaptors/npm/github.ex @@ -73,9 +73,9 @@ defmodule Lightning.Adaptors.NPM.GitHub do such shape at all. Fans out via `Task.async_stream` with a bounded concurrency. Transport - failures for a single `(name, shape)` are dropped silently — the whole - pipeline only fails if every fetch crashes the supervisor, which is - not surfaced here. + failures for a single `(name, shape)` are dropped silently — they just + don't appear in the result map. This function always returns + `{:ok, partial_map}`; there is no error return. """ @spec fetch_all([String.t()], %{ optional(String.t()) => %{ @@ -182,13 +182,9 @@ defmodule Lightning.Adaptors.NPM.GitHub do Map.update(acc, name, %{shape => entry}, &Map.put(&1, shape, entry)) end - # GitHub raw does not gzip-compress PNG/SVG bodies, so no - # `Tesla.Middleware.DecompressResponse` is in play. The sha256 computed - # in `fetch_all/2` is therefore over the raw response bytes (which equals - # the decompressed bytes in our case — there is no compression layer to - # speak of, and no `accept-encoding` middleware is configured). If GitHub - # ever starts compressing 200 bodies, Tesla/Finch would need an - # accept-encoding/decompress middleware to preserve this invariant. + # GitHub raw doesn't compress PNG/SVG responses, so the sha256 computed + # in fetch_all/2 is over the exact bytes served. If that ever changes, + # this needs a decompress middleware to keep the checksum correct. defp do_fetch_one(client, name, shape, prior_etag) do suffix = strip_scope(name) cond_get? = is_binary(prior_etag) diff --git a/lib/lightning/adaptors/npm/registry.ex b/lib/lightning/adaptors/npm/registry.ex index a05e93dfb02..8d6199e1206 100644 --- a/lib/lightning/adaptors/npm/registry.ex +++ b/lib/lightning/adaptors/npm/registry.ex @@ -9,9 +9,9 @@ defmodule Lightning.Adaptors.NPM.Registry do Base URL via `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)[:registry_url]`, default `https://registry.npmjs.org`. - Search results are filtered down to `@openfn/language-*` packages, - matching the legacy `AdaptorRegistry` semantics; non-language packages - in the `@openfn/` scope (e.g. `@openfn/cli`) are rejected. + Search results are filtered down to `@openfn/language-*` packages; + non-language packages in the `@openfn/` scope (e.g. `@openfn/cli`) are + rejected. """ alias Lightning.Adaptors.Config diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 44e0477ea44..f06df454e3e 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -393,9 +393,6 @@ defmodule Lightning.Adaptors.Scheduler do :touched else case strategy.fetch_adaptor(name) do - # latest_version is bound in the head rather than read inside the - # Logger call. A log message is only built when its level is enabled, - # so a field read inside one isn't exercised. {:ok, %{latest_version: version} = record} -> Logger.debug("Adaptors[#{state.source}]: fetched #{name}@#{version}") @@ -601,9 +598,8 @@ defmodule Lightning.Adaptors.Scheduler do acc end - # nil → preserve existing etag on the row (do not clobber). - # value matching the row's current etag → no-op (avoid no-op write). - # value differing → emit the change. + # A nil etag never overwrites what's on the row. A value equal to the + # row's current etag is skipped too, to avoid a no-op write. defp maybe_accumulate_etag(acc, _etag_key, _row, nil), do: acc defp maybe_accumulate_etag(acc, etag_key, row, etag) when is_binary(etag) do @@ -662,12 +658,10 @@ defmodule Lightning.Adaptors.Scheduler do {:error, {:upsert_failed, Exception.message(e)}} end - # Project a list of adaptor rows to the prior-etag map shape expected - # by `Strategy.fetch_icons/1`: `%{name => %{shape => etag}}`. Rows - # whose etags are both nil are skipped entirely (no empty inner map); - # within a row, only shapes with a non-nil etag are kept. The consumer - # treats absence as "no prior etag, send no If-None-Match", so an empty - # entry would be wasteful but harmless — we drop it for clarity. + # Builds the prior-etag map for `Strategy.fetch_icons/1`. A row or shape + # with no etag is left out rather than kept as an empty entry — the + # strategy already treats an absent entry as "no prior etag, don't send + # If-None-Match". @spec prior_etags_from_rows([map()]) :: %{ String.t() => %{optional(:square | :rectangle) => String.t()} } @@ -691,7 +685,6 @@ defmodule Lightning.Adaptors.Scheduler do defp maybe_put_shape_etag(map, shape, etag) when is_binary(etag), do: Map.put(map, shape, etag) - # Used in the tick summary log. defp count_not_modified(icons) do Enum.reduce(icons, 0, fn {_name, shapes}, acc -> Enum.reduce(shapes, acc, fn diff --git a/lib/lightning/adaptors/seed.ex b/lib/lightning/adaptors/seed.ex index a102968d0f2..37c912065ea 100644 --- a/lib/lightning/adaptors/seed.ex +++ b/lib/lightning/adaptors/seed.ex @@ -76,7 +76,22 @@ defmodule Lightning.Adaptors.Seed do end end + @icon_sha256_fields ~w(icon_square_sha256 icon_rectangle_sha256) + defp normalize_snapshot_record(record, source) when is_map(record) do - Map.put(record, "source", source) + record + |> Map.put("source", source) + |> decode_icon_sha256s() + end + + # Reverses `Mix.Tasks.Lightning.Adaptors.Dump`'s base64 encoding of the + # raw hash bytes those columns hold; a plain-npm snapshot has no such keys. + defp decode_icon_sha256s(record) do + Enum.reduce(@icon_sha256_fields, record, fn field, acc -> + case Map.get(acc, field) do + nil -> acc + encoded -> Map.put(acc, field, Base.decode64!(encoded)) + end + end) end end diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index 37fa65fb8f7..0210489e86e 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -309,9 +309,9 @@ defmodule Lightning.Adaptors.Store do defp project_field(value, _field), do: value - # Strategies should emit `schema_data` as a JSON binary, but legacy - # call paths (and tests) may still hand us a map. Normalize here so - # the cached value matches what subsequent DB-backed reads return. + # The real strategies already encode schema_data to a JSON binary, but + # a strategy is still free to hand back a map, so normalize here to + # keep the cached value consistent with what a DB-backed read returns. defp normalize_schema_data(%{schema_data: data} = record) when is_map(data) and not is_struct(data) do %{record | schema_data: Jason.encode!(data)} diff --git a/lib/lightning/adaptors/strategy.ex b/lib/lightning/adaptors/strategy.ex index 1a82a73c1f0..735de0a1102 100644 --- a/lib/lightning/adaptors/strategy.ex +++ b/lib/lightning/adaptors/strategy.ex @@ -8,9 +8,9 @@ defmodule Lightning.Adaptors.Strategy do * `c:fetch_adaptor/1` — given a package name, return a structured `t:adaptor_record/0` covering version history, integrity hashes, - and dependency metadata. Icon fields are **not** part of this - record any more; the Scheduler stamps them on after joining the - bulk icon pipeline. + and dependency metadata. Icon fields are not part of this + record; the Scheduler stamps them on separately after joining + the bulk icon pipeline. * `c:fetch_icon/2` — given a package name and an icon variant, return the raw bytes plus extension. Used by the Store's rare lazy-miss fallback. diff --git a/lib/lightning/config.ex b/lib/lightning/config.ex index 9cfc32761f6..fed4d835484 100644 --- a/lib/lightning/config.ex +++ b/lib/lightning/config.ex @@ -7,11 +7,6 @@ defmodule Lightning.Config do @behaviour Lightning.Config alias Lightning.Services.AdapterHelper - @impl true - def adaptor_registry do - Application.get_env(:lightning, Lightning.AdaptorRegistry, []) - end - @impl true def token_signer do :persistent_term.get({__MODULE__, "token_signer"}, nil) @@ -453,7 +448,6 @@ defmodule Lightning.Config do @callback usage_tracking_run_chunk_size() :: integer() @callback worker_secret() :: binary() | nil @callback worker_token_signer() :: Joken.Signer.t() - @callback adaptor_registry() :: Keyword.t() @callback credential_transfer_token_validity_in_days() :: integer() @callback book_demo_banner_enabled?() :: boolean() @callback book_demo_calendly_url() :: String.t() @@ -472,13 +466,6 @@ defmodule Lightning.Config do @callback runtime_manager_port() :: integer() @callback max_credential_sensitive_values() :: pos_integer() - @doc """ - Returns the configuration for the `Lightning.AdaptorRegistry` service - """ - def adaptor_registry do - impl().adaptor_registry() - end - @doc """ Returns the Apollo server configuration. """ diff --git a/lib/lightning/config/bootstrap.ex b/lib/lightning/config/bootstrap.ex index bc2d64753d8..ca3a30f8372 100644 --- a/lib/lightning/config/bootstrap.ex +++ b/lib/lightning/config/bootstrap.ex @@ -241,20 +241,6 @@ defmodule Lightning.Config.Bootstrap do use_local_adaptors_repos? = env!("LOCAL_ADAPTORS", &Utils.ensure_boolean/1, false) - |> tap(fn v -> - if v && local_adaptors_repos == [] do - raise """ - LOCAL_ADAPTORS is set to true, but OPENFN_ADAPTORS_REPO is not set. - """ - end - end) - - # local_adaptors_repos also feeds the new Lightning.Adaptors.Local - # strategy below (dual-write). install_schemas.ex:177 still reads this - # exact key, so it is not moved, only copied. - config :lightning, Lightning.AdaptorRegistry, - local_adaptors_repos: - if(use_local_adaptors_repos?, do: local_adaptors_repos, else: []) configure_adaptors_strategy(local_adaptors_repos, use_local_adaptors_repos?) @@ -262,13 +248,12 @@ defmodule Lightning.Config.Bootstrap do # through Lightning.Adaptors.Config.strategy_opts/1: registry_url is the # npm search and packument endpoint (NPM.Registry), jsdelivr_url serves # configuration schemas (NPM.Schema), and github_url plus github_ref locate - # the raw icon files under OpenFn/adaptors (NPM.GitHub). The defaults match - # the @default_* attributes in those modules. + # the raw icon files under OpenFn/adaptors (NPM.GitHub). Defaults live in + # the @default_* attributes on those modules; bootstrap only overrides one + # when its env var is set. # # Point them at `bin/adaptor_cache` to serve all three from a local disk # cache while working on adaptors. - # Defaults live in the strategy sub-modules' own @default_* attributes; - # bootstrap only maps an env var onto an override when one is set. config :lightning, Lightning.Adaptors.NPM, [ @@ -280,14 +265,6 @@ defmodule Lightning.Config.Bootstrap do ] |> Enum.reject(fn {_key, value} -> is_nil(value) end) - config :lightning, - schemas_path: - env!( - "SCHEMAS_PATH", - :string, - Utils.get_env([:lightning, :schemas_path], "./priv") - ) - config :lightning, :purge_deleted_after_days, env!( @@ -1090,11 +1067,10 @@ defmodule Lightning.Config.Bootstrap do end # ADAPTORS_LOCAL_REPO wins outright when set. When unset, fall back to - # the (ungated) OPENFN_ADAPTORS_REPO parse above, warning only when the - # new subsystem is actually running the Local strategy — an operator - # who still needs OPENFN_ADAPTORS_REPO for the old registry while - # running the new subsystem on npm shouldn't be warned about a var they - # legitimately need. + # the (ungated) OPENFN_ADAPTORS_REPO parse above, warning only when + # Lightning.Adaptors is actually running the Local strategy — an operator + # running the npm strategy with OPENFN_ADAPTORS_REPO still set for the + # ws-worker shouldn't be warned about a var they legitimately need. defp resolve_local_strategy_paths(local_adaptors_repos, adaptors_strategy) do case env!("ADAPTORS_LOCAL_REPO", :string, nil) |> parse_repo_list() do [] -> diff --git a/lib/lightning/release.ex b/lib/lightning/release.ex index 6e59f2fe290..06ba2cf7aa3 100644 --- a/lib/lightning/release.ex +++ b/lib/lightning/release.ex @@ -38,9 +38,9 @@ defmodule Lightning.Release do @doc """ Populate the adaptor catalogue from a JSON snapshot file, without - reaching npm. The release-safe path for `Lightning.Adaptors.seed_from_file/2` - — there is no Mix in a release, so `mix lightning.adaptors.import` - cannot run there; this is what `bin/lightning eval` calls instead. + reaching npm. This is the release equivalent of + `mix lightning.adaptors.import` — a release has no Mix, so run this + through `bin/lightning eval` instead. ## Usage diff --git a/lib/lightning/runs/handlers.ex b/lib/lightning/runs/handlers.ex index 96cf9ce14fb..90d17230366 100644 --- a/lib/lightning/runs/handlers.ex +++ b/lib/lightning/runs/handlers.ex @@ -555,15 +555,14 @@ defmodule Lightning.Runs.Handlers do |> Repo.insert() end - # For back compat: older workers JSON-encode output_dataclip into a string - # before sending it (Lightning then decodes it); newer workers send the - # value already decoded. A bare string is ambiguous either way — e.g. a - # job can legitimately return "24", "true", or "{}" as its literal state - # — so we try to parse it as JSON and, if that fails, fall back to the - # string as-is. This can misclassify a literal string that happens to - # look like JSON (a job returning the string "24" ends up stored as the - # number 24), but returning a bare string as step state is already an - # edge case we're comfortable accepting the ambiguity on for now. + # Older workers JSON-encode output_dataclip into a string before sending + # it; newer workers send it already decoded. A bare string is ambiguous + # either way — a job can legitimately return "24", "true", or "{}" as + # its literal state — so we try to JSON-decode it and fall back to the + # raw string if that fails. A literal string that happens to look like + # JSON (a job returning the string "24") ends up stored as the decoded + # value instead; we accept that ambiguity since returning a bare string + # as step state is already rare. defp maybe_decode_dataclip(value) when is_binary(value) do case Jason.decode(value) do {:ok, decoded} -> decoded diff --git a/lib/lightning/workflows/job.ex b/lib/lightning/workflows/job.ex index f1139558614..dfdce002e44 100644 --- a/lib/lightning/workflows/job.ex +++ b/lib/lightning/workflows/job.ex @@ -153,6 +153,11 @@ defmodule Lightning.Workflows.Job do end end + # Rejects an adaptor the catalogue doesn't know about, so an unknown + # package cannot be persisted on a job. `fetch_adaptor` returns + # `:not_found` once the catalogue has loaded and genuinely lacks the + # package, but any other error means the catalogue itself isn't ready + # yet, so that case gets its own, retry-able message. defp validate_known_adaptor(changeset) do validate_change(changeset, :adaptor, fn :adaptor, adaptor -> with {name, _version} when is_binary(name) <- Adaptors.parse_spec(adaptor), diff --git a/lib/lightning_web/controllers/adaptor_icon_controller.ex b/lib/lightning_web/controllers/adaptor_icon_controller.ex index 4da1d83df4a..c78a8f7512d 100644 --- a/lib/lightning_web/controllers/adaptor_icon_controller.ex +++ b/lib/lightning_web/controllers/adaptor_icon_controller.ex @@ -2,9 +2,6 @@ defmodule LightningWeb.AdaptorIconURL do @moduledoc """ Single source of truth for content-addressable adaptor-icon URLs. - Called from `LightningWeb.AdaptorIconController` for redirect targets and - from `WorkflowChannel`'s `request_adaptors` payload. - `sha8` is the first 4 raw bytes of the icon's sha256, hex-encoded to 8 lowercase characters, yielding a deterministic content-addressable path segment. @@ -13,8 +10,8 @@ defmodule LightningWeb.AdaptorIconURL do @doc """ Build a content-addressable icon URL for `name`/`shape`. - Returns `nil` when the adaptor row has no ext or sha256 for the - requested shape — i.e. when no icon is available. + Returns `nil` when `meta` has no ext or sha256 for the requested shape + — i.e. when no icon is available. """ @spec build(String.t(), map(), :square | :rectangle) :: String.t() | nil def build(name, meta, shape) do diff --git a/lib/mix/tasks/install_adaptor_icons.ex b/lib/mix/tasks/install_adaptor_icons.ex deleted file mode 100644 index 6ac0f795bdb..00000000000 --- a/lib/mix/tasks/install_adaptor_icons.ex +++ /dev/null @@ -1,101 +0,0 @@ -defmodule Mix.Tasks.Lightning.InstallAdaptorIcons do - @moduledoc """ - Installs the adaptor icons - """ - use Mix.Task - - @adaptors_tar_url "https://github.com/OpenFn/adaptors/archive/refs/heads/main.tar.gz" - - defp adapter do - Application.get_env(:tesla, __MODULE__, [])[:adapter] - end - - @target_dir Application.compile_env(:lightning, :adaptor_icons_path) - - @impl true - def run(_) do - Application.ensure_all_started(:telemetry) - Finch.start_link(name: Lightning.Finch) - - File.mkdir_p(@target_dir) - |> case do - {:error, reason} -> - raise "Couldn't create the adaptors images directory: #{@target_dir}, got :#{reason}." - - :ok -> - :ok - end - - working_dir = tmp_dir!() - tar = fetch_body!(@adaptors_tar_url) - - case :erl_tar.extract({:binary, tar}, [ - :compressed, - cwd: to_charlist(working_dir) - ]) do - :ok -> :ok - other -> raise "couldn't unpack archive: #{inspect(other)}" - end - - adaptor_icons = save_icons(working_dir) - manifest_path = Path.join(@target_dir, "adaptor_icons.json") - :ok = File.write(manifest_path, Jason.encode!(adaptor_icons)) - - Mix.shell().info( - "Adaptor icons installed successfully. Manifest saved at: #{manifest_path}" - ) - end - - defp fetch_body!(url) do - response = Tesla.get!(build_client(), url) - response.body - end - - defp build_client do - Tesla.client([Tesla.Middleware.FollowRedirects], adapter()) - end - - defp tmp_dir! do - tmp_dir = - Path.join([ - System.tmp_dir!(), - "lightning-adaptor", - "#{System.unique_integer([:positive])}" - ]) - - {:ok, _} = File.rm_rf(tmp_dir) - :ok = File.mkdir_p(tmp_dir) - - tmp_dir - end - - defp list_icons(working_dir) do - [working_dir, "**", "packages", "*", "assets", "{rectangle,square}.png"] - |> Path.join() - |> Path.wildcard() - end - - defp save_icons(working_dir) do - working_dir - |> list_icons() - |> Enum.map(fn icon_path -> - [icon_name, "assets", adapter_name | _rest] = - Path.split(icon_path) |> Enum.reverse() - - destination_name = adapter_name <> "-" <> icon_name - destination_path = Path.join(@target_dir, destination_name) - File.cp!(icon_path, destination_path) - - %{ - adaptor: adapter_name, - shape: Path.rootname(icon_name), - src: "/images/adaptors" <> "/#{destination_name}" - } - end) - |> Enum.group_by(fn entry -> entry.adaptor end) - |> Enum.into(%{}, fn {adaptor, sources} -> - sources = Map.new(sources, fn entry -> {entry.shape, entry.src} end) - {adaptor, sources} - end) - end -end diff --git a/lib/mix/tasks/install_schemas.ex b/lib/mix/tasks/install_schemas.ex deleted file mode 100644 index 05abf17fbe2..00000000000 --- a/lib/mix/tasks/install_schemas.ex +++ /dev/null @@ -1,272 +0,0 @@ -defmodule Mix.Tasks.Lightning.InstallSchemas do - @shortdoc "Install the credential json schemas" - - @moduledoc """ - Install the credential json schemas - Use --exclude language-package1, language-package2 to exclude specific packages - """ - - use Mix.Task - use HTTPoison.Base - require Logger - - @default_excluded_adaptors [ - "language-common", - "language-devtools", - "language-divoc" - ] - - # Descending on purpose: the first attempt is generous to let jsdelivr warm - # a cold cache for packages it hasn't served recently. Follow-up attempts - # are shorter because we expect a now-warm hit and want to fail fast if not. - @recv_timeouts [30_000, 15_000, 5_000] - - # hackney error reasons that we treat as transient and worth retrying. - # Anything else (e.g. :nxdomain, :econnrefused) is logged with its reason - # and skipped immediately rather than retried. - # - # `:invalid_state` is a stale-pooled-connection race, not a real failure. - # hackney keeps a closing pooled connection alive briefly so requests that - # raced the checkout get answered rather than crashing the caller; it means to - # answer `{:closed, _}` (see the comment in `hackney_conn.erl`) but the - # catch-all it falls through to replies `:invalid_state`, so retry on both. - @retriable_reasons [ - :timeout, - :closed, - :connect_timeout, - :checkout_timeout, - :invalid_state - ] - - # Outer Task.async_stream timeout. Must comfortably exceed the sum of - # @recv_timeouts (50s) plus connect/DNS/body overhead. - @async_stream_timeout 75_000 - - @spec run(any) :: any - def run(args) do - # Evaluate runtime.exs so LOCAL_ADAPTORS/OPENFN_ADAPTORS_REPO are picked up; - # without this the local check always sees empty config and falls back to npm. - Mix.Task.run("app.config") - - dir = schemas_path() - - init_schema_dir(dir) - - results = - if Lightning.AdaptorRegistry.local_adaptors_enabled?() do - install_from_local(dir) - else - HTTPoison.start() - args |> parse_excluded() |> fetch_schemas(&persist_schema(dir, &1)) - end - - {installed, skipped} = - Enum.reduce(results, {0, 0}, fn - {:installed, _name}, {ok, skip} -> {ok + 1, skip} - {:skipped, _name, _reason}, {ok, skip} -> {ok, skip + 1} - end) - - Mix.shell().info( - "Schemas installation has finished. #{installed} installed, #{skipped} skipped." - ) - end - - def parse_excluded(args) do - args - |> case do - ["--exclude" | adaptor_names] when adaptor_names != [] -> - (adaptor_names ++ @default_excluded_adaptors) |> Enum.uniq() - - _ -> - @default_excluded_adaptors - end - end - - defp schemas_path do - Application.get_env(:lightning, :schemas_path) - end - - defp init_schema_dir(dir) do - if is_nil(dir), do: raise("Schema directory not provided.") - File.rm_rf(dir) - - File.mkdir_p(dir) - |> case do - {:error, reason} -> - raise "Couldn't create the schemas directory: #{dir}, got :#{reason}." - - _ -> - nil - end - end - - def write_schema(dir, package_name, data) when is_binary(package_name) do - path = - Path.join([ - dir, - String.replace(package_name, "@openfn/language-", "") <> ".json" - ]) - - file = File.open!(path, [:write]) - - IO.binwrite(file, data) - File.close(file) - end - - def persist_schema(dir, package_name) do - attempt_persist_schema(dir, package_name, @recv_timeouts) - end - - defp attempt_persist_schema(dir, package_name, [timeout | rest]) do - url = - "https://cdn.jsdelivr.net/npm/#{package_name}/configuration-schema.json" - - case get(url, [], hackney: [pool: :default], recv_timeout: timeout) do - {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> - write_schema(dir, package_name, body) - {:installed, package_name} - - {:ok, %HTTPoison.Response{status_code: status_code}} -> - Logger.warning( - "Unable to fetch #{package_name} configuration schema. status=#{status_code}" - ) - - {:skipped, package_name, {:http_status, status_code}} - - {:error, %HTTPoison.Error{reason: reason}} - when reason in @retriable_reasons and rest != [] -> - [next_timeout | _] = rest - - Logger.warning( - "Transient error fetching #{package_name} (#{inspect(reason)}); " <> - "retrying with recv_timeout=#{next_timeout}ms" - ) - - attempt_persist_schema(dir, package_name, rest) - - {:error, %HTTPoison.Error{reason: reason}} -> - attempts_used = length(@recv_timeouts) - length(rest) - - Logger.warning( - "Skipping #{package_name}: #{inspect(reason)} after " <> - "#{attempts_used} attempt(s)" - ) - - {:skipped, package_name, reason} - end - end - - # Read schemas straight from the local adaptor monorepos configured via - # LOCAL_ADAPTORS/OPENFN_ADAPTORS_REPO instead of fetching from npm/jsdelivr. - # Packages without a configuration-schema.json (e.g. common) are skipped. - def install_from_local(dir) do - repos = local_repos() - - Mix.shell().info( - "Installing credential schemas from: #{Enum.join(repos, ", ")}" - ) - - repos - |> Enum.flat_map(&local_packages/1) - # First occurrence wins, matching AdaptorRegistry's repo precedence. - |> Enum.uniq_by(fn {name, _path} -> name end) - |> Enum.map(fn {name, path} -> persist_local_schema(dir, name, path) end) - end - - defp local_repos do - Lightning.Config.adaptor_registry()[:local_adaptors_repos] || [] - end - - defp local_packages(repo_path) do - packages_path = Path.join(repo_path, "packages") - - case File.ls(packages_path) do - {:ok, entries} -> - Enum.map(entries, fn pkg -> - {pkg, Path.join([packages_path, pkg, "configuration-schema.json"])} - end) - - {:error, reason} -> - Logger.warning( - "Skipping local repo #{inspect(repo_path)}: " <> - "cannot list #{inspect(packages_path)} (#{:file.format_error(reason)})" - ) - - [] - end - end - - defp persist_local_schema(dir, name, schema_path) do - case File.read(schema_path) do - {:ok, body} -> - write_schema(dir, name, body) - {:installed, name} - - {:error, reason} -> - {:skipped, name, reason} - end - end - - def fetch_schemas(excluded \\ [], fun) do - get("https://registry.npmjs.org/-/user/openfn/package", [], - hackney: [pool: :default], - recv_timeout: 15_000 - ) - |> case do - {:error, %HTTPoison.Error{reason: reason}} -> - raise "Unable to connect to NPM; no adaptors fetched: #{inspect(reason)}" - - {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> - excluded = excluded |> Enum.map(&"@openfn/#{&1}") - - names = - body - |> Jason.decode!() - |> Enum.map(fn {name, _} -> name end) - |> Enum.filter(fn name -> - Regex.match?(~r/@openfn\/language-\w+/, name) - end) - |> Enum.reject(fn name -> name in excluded end) - - # Wrap fun so a worker crash (raise/exit) becomes a normal {:skipped, - # _, _} result instead of taking the caller down via the task link. - # ordered: true (the default) lets us zip results against `names` so - # the on_timeout: :kill_task path can also recover the package name. - safe_fun = fn name -> - try do - fun.(name) - catch - kind, reason -> - Logger.warning( - "Schema fetch worker for #{name} crashed: " <> - "#{inspect({kind, reason})}" - ) - - {:skipped, name, {kind, reason}} - end - end - - names - |> Task.async_stream(safe_fun, - max_concurrency: 5, - timeout: @async_stream_timeout, - on_timeout: :kill_task - ) - |> Stream.zip(names) - |> Stream.map(fn - {{:ok, result}, _name} -> - result - - {{:exit, reason}, name} -> - Logger.warning( - "Schema fetch task for #{name} killed: #{inspect(reason)}" - ) - - {:skipped, name, reason} - end) - - {:ok, %HTTPoison.Response{status_code: status_code}} -> - raise "Unable to access openfn user packages. status=#{status_code}" - end - end -end diff --git a/lib/mix/tasks/lightning.adaptors.dump.ex b/lib/mix/tasks/lightning.adaptors.dump.ex index 5191592e6ad..188d22792f3 100644 --- a/lib/mix/tasks/lightning.adaptors.dump.ex +++ b/lib/mix/tasks/lightning.adaptors.dump.ex @@ -20,9 +20,11 @@ defmodule Mix.Tasks.Lightning.Adaptors.Dump do `--source` defaults to `npm`. - Icons are left out. Their bytes live in the on-disk icon cache rather - than the catalogue, so a hash without them would buy the importing - instance nothing; it refetches instead. + Icon bytes themselves live in the on-disk icon cache, not the catalogue, + so they don't travel in this file. It carries the icon metadata + (extension, sha256, etag) so an imported row can serve an icon already + present at `ADAPTORS_ICONS_PATH` on the target instance instead of + refetching from GitHub; copy that directory across alongside this file. """ use Mix.Task @@ -30,11 +32,16 @@ defmodule Mix.Tasks.Lightning.Adaptors.Dump do alias Lightning.Adaptors.Catalogue @adaptor_fields ~w(name source description homepage repository license - latest_version deprecated schema_data schema_sha256)a + latest_version deprecated schema_data schema_sha256 + icon_square_ext icon_rectangle_ext + icon_square_sha256 icon_rectangle_sha256 + icon_square_etag icon_rectangle_etag)a @version_fields ~w(version integrity tarball_url size_bytes dependencies peer_dependencies published_at deprecated)a + @icon_sha256_fields ~w(icon_square_sha256 icon_rectangle_sha256)a + @impl Mix.Task def run(argv) do Mix.Task.run("app.start") @@ -68,9 +75,18 @@ defmodule Mix.Tasks.Lightning.Adaptors.Dump do adaptor |> Map.from_struct() |> Map.take(@adaptor_fields) + |> encode_icon_sha256s() |> Map.put(:versions, versions) end + # icon_*_sha256 columns hold raw hash bytes, not valid JSON text; encode + # them here, `Seed.normalize_snapshot_record/2` decodes on the way back in. + defp encode_icon_sha256s(record) do + Enum.reduce(@icon_sha256_fields, record, fn field, acc -> + Map.update!(acc, field, &(&1 && Base.encode64(&1))) + end) + end + defp parse_source(nil), do: :npm defp parse_source("npm"), do: :npm defp parse_source("local"), do: :local diff --git a/mix.exs b/mix.exs index a71344f34cc..1e654d1428e 100644 --- a/mix.exs +++ b/mix.exs @@ -229,8 +229,6 @@ defmodule Lightning.MixProject do "tailwind.install --if-missing", "esbuild.install --if-missing", "lightning.install_runtime", - "lightning.install_adaptor_icons", - "lightning.install_schemas", "ecto.setup" ], "ecto.setup": ["ecto.create", "ecto.migrate"], @@ -266,6 +264,7 @@ defmodule Lightning.MixProject do extras: [ "README.md": [title: "Lightning"], "RUNNINGLOCAL.md": [title: "Running Locally"], + "ADAPTORS.md": [title: "Adaptors"], "DEPLOYMENT.md": [title: "Deployment"], "tooling/benchmarking/README.md": [ title: "Benchmarking", diff --git a/test/fixtures/adaptor_registry_cache.json b/test/fixtures/adaptor_registry_cache.json deleted file mode 100644 index 9e58fe7ef73..00000000000 --- a/test/fixtures/adaptor_registry_cache.json +++ /dev/null @@ -1,232 +0,0 @@ -[ - { - "latest": "1.6.2", - "name": "@openfn/language-common", - "repo": "git+https://github.com/OpenFn/language-common.git", - "versions": [ - { "version": "1.1.0" }, - { "version": "1.1.12" }, - { "version": "1.10.3" }, - { "version": "1.2.14" }, - { "version": "1.2.22" }, - { "version": "1.2.3" }, - { "version": "1.6.2" }, - { "version": "2.14.0" } - ] - }, - { - "latest": "3.0.5", - "name": "@openfn/language-dhis2", - "repo": "git+https://github.com/openfn/language-dhis2.git", - "versions": [ - { "version": "2.0.10" }, - { "version": "2.0.11" }, - { "version": "2.0.2" }, - { "version": "2.0.3" }, - { "version": "2.0.6" }, - { "version": "2.0.7" }, - { "version": "2.0.8" }, - { "version": "2.0.9" }, - { "version": "3.0.0" }, - { "version": "3.0.0-0" }, - { "version": "3.0.0-2" }, - { "version": "3.0.0-3" }, - { "version": "3.0.0-4" }, - { "version": "3.0.1" }, - { "version": "3.0.2" }, - { "version": "3.0.4" }, - { "version": "3.0.5" } - ] - }, - { - "latest": "1.4.0", - "name": "@openfn/language-commcare", - "repo": "git+https://github.com/openfn/language-commcare.git", - "versions": [ - { "version": "1.2.4" }, - { "version": "1.2.5" }, - { "version": "1.3.0" }, - { "version": "1.4.0" } - ] - }, - { - "latest": "0.2.0", - "name": "@openfn/language-divoc", - "repo": "git+https://github.com/OpenFn/language-divoc.git", - "versions": [{ "version": "0.2.0" }] - }, - { - "latest": "2.0.0", - "name": "@openfn/language-asana", - "repo": "git+https://github.com/OpenFn/language-asana.git", - "versions": [ - { "version": "1.0.1" }, - { "version": "1.1.0" }, - { "version": "1.1.1" }, - { "version": "2.0.0" } - ] - }, - { - "latest": "3.0.0", - "name": "@openfn/language-mssql", - "repo": "git+https://github.com/openfn/language-mssql.git", - "versions": [ - { "version": "2.3.3" }, - { "version": "2.4.0" }, - { "version": "2.5.0" }, - { "version": "2.5.2" }, - { "version": "2.5.3" }, - { "version": "2.5.5" }, - { "version": "2.6.0" }, - { "version": "2.6.1" }, - { "version": "2.6.10" }, - { "version": "2.6.11" }, - { "version": "2.6.3" }, - { "version": "2.6.4" }, - { "version": "2.6.5" }, - { "version": "2.6.6" }, - { "version": "2.6.7" }, - { "version": "2.6.8" }, - { "version": "2.6.9" }, - { "version": "3.0.0" } - ] - }, - { - "latest": "3.1.12", - "name": "@openfn/language-http", - "repo": "git+https://github.com/openfn/language-http.git", - "versions": [ - { "version": "3.0.0" }, - { "version": "3.0.2" }, - { "version": "3.1.0" }, - { "version": "3.1.1" }, - { "version": "3.1.10" }, - { "version": "3.1.11" }, - { "version": "3.1.12" }, - { "version": "3.1.2" }, - { "version": "3.1.3" }, - { "version": "3.1.4" }, - { "version": "3.1.5" } - ] - }, - { - "latest": "1.0.4", - "name": "@openfn/language-kobotoolbox", - "repo": "git+https://github.com/openfn/language-kobotoolbox.git", - "versions": [{ "version": "1.0.4" }] - }, - { - "latest": "0.2.2", - "name": "@openfn/language-mailgun", - "repo": "git+https://github.com/openfn/language-mailgun.git", - "versions": [{ "version": "0.2.2" }] - }, - { - "latest": "1.2.0", - "name": "@openfn/language-mysql", - "repo": "git+https://github.com/OpenFn/language-mysql.git", - "versions": [{ "version": "1.1.2" }, { "version": "1.2.0" }] - }, - { - "latest": "1.1.4", - "name": "@openfn/language-openfn", - "repo": "git+https://github.com/openfn/language-openfn.git", - "versions": [{ "version": "1.1.3" }, { "version": "1.1.4" }] - }, - { - "latest": "2.5.1", - "name": "@openfn/language-primero", - "repo": "git+https://github.com/openfn/language-primero.git", - "versions": [ - { "version": "2.2.7" }, - { "version": "2.2.8" }, - { "version": "2.3.1" }, - { "version": "2.3.2" }, - { "version": "2.3.3" }, - { "version": "2.3.4" }, - { "version": "2.3.5" }, - { "version": "2.3.6" }, - { "version": "2.4.0" }, - { "version": "2.4.1" }, - { "version": "2.4.2" }, - { "version": "2.4.3" }, - { "version": "2.4.4" }, - { "version": "2.5.1" } - ] - }, - { - "latest": "0.4.7", - "name": "@openfn/language-rapidpro", - "repo": "git+https://github.com/OpenFn/language-rapidpro.git", - "versions": [ - { "version": "0.2.0" }, - { "version": "0.3.0" }, - { "version": "0.4.2" }, - { "version": "0.4.5" }, - { "version": "0.4.7" } - ] - }, - { - "latest": "3.3.3", - "name": "@openfn/language-postgresql", - "repo": "git+https://github.com/openfn/language-postgresql.git", - "versions": [ - { "version": "3.1.3" }, - { "version": "3.1.6" }, - { "version": "3.2.0" }, - { "version": "3.2.1" }, - { "version": "3.2.2" }, - { "version": "3.2.3" }, - { "version": "3.2.4" }, - { "version": "3.2.5" }, - { "version": "3.3.0" }, - { "version": "3.3.1" }, - { "version": "3.3.2" }, - { "version": "3.3.3" } - ] - }, - { - "latest": "0.9.2", - "name": "@openfn/language-openmrs", - "repo": "git+https://github.com/openfn/language-openmrs.git", - "versions": [{ "version": "0.9.1" }, { "version": "0.9.2" }] - }, - { - "latest": "0.2.1", - "name": "@openfn/language-twilio", - "repo": "git+https://github.com/openfn/language-twilio.git", - "versions": [{ "version": "0.2.1" }] - }, - { - "latest": "0.4.0", - "name": "@openfn/language-sftp", - "repo": "git+https://github.com/openfn/language-sftp.git", - "versions": [{ "version": "0.3.3" }, { "version": "0.4.0" }] - }, - { - "latest": "2.7.4", - "name": "@openfn/language-salesforce", - "repo": "git://github.com/OpenFn/language-salesforce.git", - "versions": [ - { "version": "2.1.0" }, - { "version": "2.2.0" }, - { "version": "2.2.7" }, - { "version": "2.3.0" }, - { "version": "2.3.1" }, - { "version": "2.3.2" }, - { "version": "2.3.3" }, - { "version": "2.3.4" }, - { "version": "2.3.5" }, - { "version": "2.4.0" }, - { "version": "2.5.1" }, - { "version": "2.5.2" }, - { "version": "2.6.0" }, - { "version": "2.6.3" }, - { "version": "2.7.0" }, - { "version": "2.7.1" }, - { "version": "2.7.2" }, - { "version": "2.7.3" }, - { "version": "2.7.4" } - ] - } -] diff --git a/test/fixtures/adaptors/dhis2.tar.gz b/test/fixtures/adaptors/dhis2.tar.gz deleted file mode 100644 index ffeab58a3e107a60d97a7ae4bccc4d6c69c0302e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24379 zcmV(#K;*w4iwFP!000006V!KQP@KWj=L)hcwz#_!+?~ZG$U+E^;4Xon0fK~G+$|6w zID|lepdmOcZov}VA-H>hAaDNnR^7L&y1Mt%b=6eQJkLz`^vth&x~*)j+`ZjAz4+a& zY+hM8*m?2WIy!j?@mqO$*?D{MdwIOE^0eb~f9(MHe?Bv^PkH9shs~dM&N(* z|4%3yp`(TmrH20N;%lfY>jMBlz<(M9>mM`qEVci~aNN~jcmV(e`~PV`j}cTF0ATK~ zp{!uwmwnjgn{6<)bp7~{9pGS(YZem|Q#gz0TyBYuZlmzmtLb5tbJ9)O&f?IW=hvNo zpD|m!&`TU3^1;!CGm5d0vryka$|WZ=U~KzOeq=xa_KBr%(m02}b%2>UZD;NL%VDh527m@Xp)z^PXVent#n z)AY8~1m!D76KH7wQaLA3T>r;Klok)Z(-DvFVR3{IKRHW0*+?-mIV&SL}E(_Kvc? z9o4;||KwayOKs?bT5hvMP-HYfx2i%$xvG))KydDMUkMb2x#Am~_cc<$J+HhpOjfG9 z-E6HB6Ojw*TFe4~z>*s%4v*+y(H$W2~Q|;WDbR%FXi}?br0@J*Ekcg7}FMY7h?k zQwI>|S6RGESv;Rv44OSrCqlhT8!sU?zv(h#8Gsiq#xFgEi)CeH)++UyFbJ6Y>ySyG)Gn`Npn;w7L9LTT3`=; z3c$L&Tn#u9jz)iFy7VN3FJ+UVngImI_-QzPzGM|&J&w7O0rH#Eo*8fcki1z@6C(of z!pPa?<(5!OC?zBb;+EwAIZjrJ!G)c&&w)a0*<&rie1OLQty|KR;SZrX_*ZtTK}lms zZ(55|WvhQ!ItqvD%FwqpD;P~f`g|Yn2eift!5S_Wp(^PGXQLxOn9lT<;D|D)h-_u4Zt^7 zym7K?tO8S@=aBz`g$W$ELaP!akwmu3Qlq7qfiu}B`;B;{h!rr;XNQJQG9(Ob!jHx3 zPH5I~OMUD+n`=sGHH$L=GI4eI7=nqFJ%;~0`g>&L!sVF42xaQ;g^jMU_k?hsKR~Lu zsg-1H`UngXlsM`ZE024k?>|#jpb{9bRIN7k!zx|T@vZ8$)Lpja*^ zuxrXcgK7NW{9(8Vs&M~E3i~x&mAN=JjKq7yJA8i$0iYjI`XJ&b);x?22;&bix$7^C zQ@K?9oYGq_kgFm2#pW!(?&y83Lrsyf*YxhgXxNM!D zPYW0BD_1B%pWr$Ql9X!#rk)dV8U0gC|FCGySSx?6gj~C>B~dSQ&3zZU!RGD<*A07x zVsXz6QA17jQp2PFX%$I}%8t#W3?Hb;C)kRf5T++aRq24 zXekM0FaoJ&kWx?&-JwIm0-Bwh*#pNgB{=tQdg3xKL>onN3*4}liLoF25U$;Z8Q zLYy%F0)L`{1PlV@Ps~S5Oj0C8?`wqc(=!{2ccgK7)DGLcHAg?G%^81~%2AuXiXYQu zvV$EF-cfFFhMkr^gzC4U)c=W@X3jE5rZ5gif&RQbVN+0OPbU@2Mr}jj%FKkxJ z1yA=V0-yh-7NZSy&|wlAE(R80yAx1J0@hw<54$VK{GNOGhhNtcerAeLaoS{5qgMMh z0v7Bo@3BUWL8S;43DhFUB;oTCUCi(0b5Gt|mI2pz)bdkz0*llB)*r}BSW z`Cy3=N4ZM6s+szpXqW6bA0w%1{GcdoE;=cYB0Y5Fr4afr`WU{VY=`Ggttb@os)3dx zpc(8?@Bm+RA8$dvt%-AxE?@6wnHo=Z;aut1LbgzJ(+$QP>_&r$M4t?>hnZl%4`cs@0q6vut-b||dR))P_Zky8 zr_V6te=vRA3bcgi_gz9&!WAl(7pzHZ;%iKWq!Gk&2wsUhl(b+$8af39hmG@)8HU zUF?1zQkUwZSMZ>aJ8`T}34LR7 z>kb56p9$=&Q>y;_%!LEk30=7*duC7K7*hCj_cB>EEz>i&HvSUN=%g#Jk zUxa<(e2j7s_^Q%*A|$Kt^+l_-_fE81J%h!4tmT>&k5yYdb&b*s@NONrYHmry#$pvo z3c#AhX#_Mb+MOs=w6v$FW@0XeuUT0-5gj$*5CuOXjEHBY^>X6(V9I4s{&~fk&VR%nIIHzj+v)HozO00E{Ti275<8UoawFdjbjqrL~-H6nD{P*mc(q zKT1~}s*gJJXD)8D&Wl(k??$y^OMxk&o;>4R%{1R&ubx3Dz}WddAElj7alExytkj4A zIF{n3`opkQv|l8cQxxLH;xBGk!laLU>DPqG+eL1Jr6tAzz{7w}QB@~H3tw7tm({@| znYWl1|2n?l5;FSD$m}iea`gzr4qe2TQNzY#+=GDfvE4xf0^?q0R7Z{9gMe)C3tZw` z&LK0-Uqx+dquP%6QR(Fxwe_kpl6g6hNxRj$ZzOt zLUM-~aH0+wijXBucfJE6C@yp6xfkjH>Zl_ePn<4+R9SKTa2tggkXyb#1x`>@oU+73 z53iq%g3@?cozFM#!3-;1TCIakD2NPbUa?H zv81c~;AkroZil7dFEKc|iGJ6!DTHB!8aS_1k|4Dc;>-4vTyRqQb*KWMw8U1{QTt(R zpcB(DDLY!fJqYqSz5`52W%9q0ix(M%^ZGS)<-80-NY`6V-Y?9TmWKtYG#?SN-ZdbC z@l?lpoWRMNAIS@7F=;Vqv9we(27OBLT?j#pnEs5p;krKT(%e7uz0sa(1eU;S4^(yH0M#1wMoey=RoB8)JwWuw0+S$_JhHTOY1X zE}Z^L_=fZI>nUP11IPhQW-Hmz$*(%;%~NyShi#FB@MF~j($%M3QGK=;t*jX3X#b50 z?+McY6Buvm!52^yj<-~q5mPp}HdI)>2MO$=$734VzrtTZYos2zOC;)&>0tS?(o;Nn|cI9##{{wvbDD zPV9UvU?+Znjd7-_R#)(4#vcvGuiPME&|QF(idY-z z%;E)5*`d&rZO9H1RMf06Oa%1da0*Ts@IYIjD55{J2242agV5Lsedq1?fncNX@9GX{ z_Ablb3M_in3pcdsVozLB;G~)KKPdb}kRW$43`iz?p{N^lF!0f*xO3BQ{aG>wE9&fx zGB)VF$5b>$pGYqGFELDH))Q$^H&GL`m_a3Y5TCr=OE28`FZudPTylR+=9^@)j%WFc zp#qFs=;ECLF3nj|jGO|O=sFT*uT6hbexxL%2t~x%qUv$#+M{G=L>$PVn}B_TyCd(2 zEs5#+6u_Bukur(N0h>Q&^kmlLSYIZd;_Kd%jatSDR8IX;G}?7AeeXl{iv9K46TW^% z?uOD|t>l15h7+M7Us(HA3Zpf*!YBO%cbOe?^ga4eBEER!=XVVF(7{lgoehF6VQT3_ zTc#{$Rq>TlJ?niiiTwlgR=2h1Wua6CZW6wkr}Y{yb%?ZF+FIO~8|kg)I7H&YS7@Wn z7ZT5QWwWJfEI(ixp&lK`pH1~D>Vkl`8|}$acb}({ckbN_LbCcWkh8SI|~~Z64sll4E(XJxeunquhDYjm=4`mvqE-y1fBGm^BHz>*5K3NX7GuwJjZm42Z+Fh4)#r&gU zDGsBWqD!wdH)5)j(+oDGkm53XPeb3Qvq#smr1c$F;f-S z6+t)cl5_=cq0{DI`{RMiWly|n+#U4_deqJcTIpMXkgXsT@C6`CQ$IuGA4>F{_ai@a zdM57IShWxe`S;6({AV#5RiRIaocX`c(-nulccR0iBynxkl8!jP*vmW%C86DHOmPC- z!>_RUFys^uLA$nv$A4nTB#7`s( zT&rptRjDCK2S~uOnyAx+VRLQx*Db&Dbqf|jt(X%q)oO7S zdloH*i%CJM<1AJ3@QJMe({H_FxungSI94?eOyoFt3RjORc2B`2b(c(yh6*5uo{@&U zrQIyr;d^Ci263whd$)-oB0l}|6i`CqNv4{hvZIHbUfmdd5+tzui|7hd66mm$a3zy< zVwR9oQpsoK9Y`RB!|xZ#C>Vj@E0){QeBqL|rf-R5b&#kLP_yKpTWx-uJ%HS@@YyZgakpaU`#T|dzP84%Cw70<1A5@9mvX$)wq8AtakQt+CG7CG#_^& zCuu$wF-Iu)r${PIarWhRML`U>McA-Hgv}*DSu0RX4merZ8cVU9;6Db)Q}ub<9r4nC zP$UU?3)wPWj?5KGyY|E?w1Fik6+goptEFY>2L>SNV}D^4^13+pP-&3)#?`uT22D$S zu-xF2;#eu+Yt3%9q|Ci=x_sV1HO5E_xnO(YBf{JO91A7qZ7NOr_w>=m?ubG-KyA>X z!ta2oO-lx4+n89cM9w&Ic}Wt<89PoLlBhCJ5`T5s1HitD^6B#{+M&CL(=6>BTp_3( zqX3y7Y+EW0hH{a;II+x8bTqrEn-22r-NTc45f5vEMY z9_u(^tjF)2e-ZKgULk_XpQ%eBd8LT0aWF;U0$*rXT7&%o=S+olhCF-kWBuz}N@ky9 zu2P?Fcd`08q6%#W-L6l}#u$2Qo0<-nY8&d7pc5gzgi(9+)6P}%>QZvU8I`Fg0TU$SEO9K@pK8jg60){OLm9x^E-XC6+3qJ!9D)qJx_{tj(#xJg{VhqUYHXL?I?o9dSO zuj4S^?oU7fEQqwZ^tmT$2)bc$W6xb$r|?1;r4*5 z;-Wa_Bhd_}Bht2SB+49+#;z5uRVti58NOwgTgt;I#>L}F6 zS0L<}LM}(`&lmW4@z^&$juWueOb46;?9v>d5U~WZF=T6TW)ZWIdXq7%#iI?%yvx}3 z)uN2Uc@#ZK*k3M2Pxb>sZDUC5 z&FdB14j?y|)?u&TRA|kzEE)_sR`U&dGZD(Ee)Neb8jZc`6((;(Tvxq@sCHmFpsc!A zNdV4KT)-@$qxbV=P-R{+08XlcF;#>omGkX4)L=8zkK9}6BrXf@Yut??qm`$_9gJAA zyFhh0$U%1%R6(FXN!yb3jgb-|47lj1q19r1h6>h)?0@H%M8Sy4Ebr10(iBsfoGb`Q z%IHD6ozH57mOKKKQ(yefAniASCfjyTG0-{?2jUDpV@a_>blh|jW)Q=2qzj!iYY< ze1i52GpY7rH;nk5=~dH=i%=h_cFM-yKm^t?lk%=cLa16<)q!(MZ|Ja|NmNRoFnkdUM5S7EhP>&9Tq5+Nqbc7~mgkL! zgpNdy}{iay;v z?(cq?E{D}JFn9ERWD$5!PBL<|7fk1c0G0?ZoL8aw%vZftJSvw?GDm>7!aq+j$ZMU{ z*-V|wR5O|!M0@uxfVPLltoM3RyJREb4QU|`1oeQ!5Kp=;t5U5VvRhM4gVWTTDyOM% zyT9zspDGXz&$jLXW34Juv?`hjLsL~eav)25oMYw_!v6UM1qP6u{S&o{J`5xvw0h10 zw?P0^+{H--5p<8+KlKdA)BCJR-)@w5wCOL1*i z#7awpSgcNFX=cZM!S%53`yxp!8i}IC@uwJVrqde%zR3*FWcB6;lQ>NplBc!ky#;SN zuY&R!JTAYwtL49gbYMwhO_?CmXB9v1Z-L;KfVexLL_NIQh22>p-Fibq$e&~=TOBvl zB;1=iB`0d}wo+ebhp_7-lb;W1u17)fS3ZH-Ll%%(JZ~`lW?sfTA^a!Y*Y{hU1GBxr zh}_r~Hu0C2JE#P#1mz}tikmCRWGWmoZesVJqbD0BM~|XHzmGtv49NC~WDm}Mn305F zk~AnM8Xp_nfqm396|@+qjfoz$3|2UZ&67H@XTB1v{u6?g{EAuBipk}eUXj1Tmwn%- z5oy%a2-~5y+#p@(wigT|MlL;wtBOZ8zWpa#_~o=Ih*D?yN(`C1j469yyN~Fscww+AeLh5+sl4Ii!Nf;pDI$uRW?YZ$N*w zB%hz<)^(B=?9JBhkg0errYU(M@3QW3dP?R@yc${Z+k=lLZP%G?q(Q`xCMMA}+15mBjTn?Vf*1k-D8_eBA>MtQW%RA z3_~AcuyUxhUbo6cSV3CU0#>viI%;Of=aUc=MAzaIuD!6k0~W_H0@J6JeWn;Ha^k8; z&oCS03d>^DfE;hT<|nHIV@oP`!T~bI;G*J6(d5Lz$)L#b2f*-f*mrUo8`Tsy#&O}d zuTNRYjh5hD2O3P1dzGJIc3ei_j;{F$jJXno8Ql%t$5;ev{uN@h1l2hMpx^AAIe7yM z|8D-82S!@{{DLnyS5ugLjiXdKWwrXjo{29~#viYmB-d0)^YiNPW~s|UJ45^*1rPlo zsS_4UTE56uQKRe#w}^?BJJq39-H8mEbT%zFnREH5MPMKa^J}USm#Dp1p(u60D+`uq z;&@XQA%T<&b)>sMf4UEC)ksVw=4S~wy4_8oG2x)RXAF0_RmVbcLp@%zQE6V!AuO*K z_tgO$g^M9)0ZB_X32R1+O|L;+>ZU`EA`Y9k?p*!H7dHOs7E0`XxuseD^|tq0<@}u* zpN2Sob3EXB>lXT0K&!>TZE`5vh=HMIuBoU8OdJ0GUrrDYM=;0}po0WiR{zA_w1gsR z+Ht0ViA;Flcd@Fb8}I=Y)D%h6lFBUEW+dk`R9$kT?q6!_OZ*B`!ZTOsMAT312?vD; z)C1Nv>t3uIcnQU@UZ^nEe<>43zTcUU*R)qxxUA&h0&H# z?IN`I)ukTh*v>nc@54Z`8%v3I&v|i_SN{sp7Aeq(H^b^drnm6>Yjcu0qjhxilUa)p z9e^YztHNR|LlD&(RQX-VlUL>y!gk0rXl_E^35jXIk%S;zJhdEggc$*=7qgVzIhXvj zmkp0f02rZmq-SgQnOw|^^FDv}dFrBJJ!Ga0Tv(Dtccm#Ct_JykuKfI?gSzCwR_xa4 z=K?HxX8eR1zwjIBbb8-Qi`OP%Jhup@I2pqMWNz$$SyU776Ms#(-&T8F8o$8p6B$*I z?(Y}{(!aD+>)%iv1$b(gz;VBNU@~hLqvBtq+ADiY%TF^6prwv5Gx-Pusi9S--X>87 z=E>N>h<}Bzk;z@Wl?z-2h$?_-c-2X_Cy!RPJZ1cAg$%fCi$U2F z9d(^KYS1D@C>JOI`1sNjx632z(KKHw>q)sbpMa8>ao~ANJ7oXWMs#y}!CqO{2M*I_ zSJG0ABxAYMjr3hZRw!cPoN+xUka4#2o}AUlE>X*PLiKczI%t^7v-ZWVu$Qi{t($l6;ca+ zCA@->iFXIsX`0}8X`;s#LrLZvk1B1Vs_E#*P$M=ez+O~7XZ`eaohi_ba<+r%R`T@t zR$C8SD3tLp%iew^D3i?J0Z(eDvp?glM?!0xT5gDb6h&InJL~ zt<5+KN`6gWE>?pu#jZ#|ig=d4oi(1ej8O=8&3=_COhpIYD^)hl#M3W;@5hi_P#ULV z8#&vax|;m`ubxQ&I5(K|L|p5d)I;8mgnBhA%Cz68UTrP&NCg3}LkCV)o)Lk1LjE9DRih+zR4$wYy_hR8jTS4EE?-aT^!U-VXe>x;0;FiP`Pi2N zq&D-+;%`fO#pr>7zsL^6tD|LKEd5npwJ&rbT1lxfp39kPt722nO)Lo@8yT+n*gM|< zGyFRmQ&6s#8L_{y;8ji$ML=jn0C->+B0^hud`*OQRvHUqr47U4n|#;&AMibC${sQ9 zg^oH&@kGn3bLbThv5f-+nQdUvGDIDAHYvbKBh$zoD>NO1kmPNp(ld2vBS6q7xl)3* zCQZdandgLPdQ=4x$eKflehbjfMrI7hh^Am2LDSRTb1g|4;=i_F9 zC#deyy3yu^oUyfwVLT`^=D_khw9~9Kbd5a!ncsaK1oKJ0z=bQ?}@a6SEH;Gp#r6}ac9DKk${H$ zAM>|WU+QYv1JkA31?2=NlW7M2hO)f4!{`4=JouG-lc6+DZ39!;P)+cokE-YisQ`nG zImb66MrDTn6`@b4u7BE_n;r3f``v^s5Z7wLtpGJ_H; z{;FHX4^Y=a=^gdd3Uf`p+A`H9G*A*y@=>;D+PN#b@dIMn-YlBz8k81*&}R0xjI+AK zg#|;Ie?dZbZhzNX$f<7dqovXAtIYXA-^(^bfRfb7VD%vsq(i~Db;5y^0*v^DzxTwe zWc#ZHDo8yi|lzq`2g948^C%XrI?Y`~@2L zhBTY!(`WD=-36hB;Sp=QtDyK+w#h;%%!!rIz?xmchDmC?z|80&-N7B&@d?IIT#0dB zmV<~IdFCt`NZLZp){BorR*nX^j!3tMXgj$V(lp(OsGU+0W6nRzpM(Fj7-t0~jg_ZY z>RNM8$b>z&^_I!aiO*O=hc|+F!W-w(^Ai~t!!=r`cP4%i7W_M)1MuMD=!(D_9gjYT zl}`vA02Th0>G}`0IW%zvUE;vkc;0ujFXw2}$7}E|R4ZCz|CQGr(dA4CTZ}j|p-l&b zMGe1|J&68Kia?%7Cc>6i#IXDq&h;2-#qv-Z@*Czw{j~-kG7Ry{4inEr$(mbf@CNFW zIdp%$iVetdL;YZT;rIr-dp?M}zwlrvoCy2z6kQ-_oGkKyX~eGJcxd>Jj!C0`CiUgfD?!*o@F=~h$# z5&5SRDlOV-w%p8r3M<&?ntxRP>|2piel_U#Ok?+l>FibSc+AgBC`NNq*dKv+_QYmJ zt!mmV8X=%b?+8Ymr~XpM`qsa|9l$Bgp&!wK-?pFv+bIHA7VDyyrSB33x@Rpx3O)o6 zAD+JWHc4^AHsnSWL?bv8k{k1i=S84?uGNcsOrEcjkd?(42YQVAz~6+Tr6Q>UCZ#lt zc>sB(H(1=ikt9yJ&<4^q!Bpvq5ETI*m&GSpu|tWRRhc@giR@WYMD=Mj&*SHbUn>~? zk~}4rHqzM)ttV`RK8^2XL19^RwId`g%BL1yxCY9C&JGPrwy&KL~P;Hp1HT1RdV6q=9Kj3u~wZ+DQPlFDwI6hhItsvI?)wc^~} zh&JdaBN~a{U%?OYdtW&dTS{j?ko|EslPD~YykaGLaPj|OF;?6BR3oybzIkX&EK%(W zc+3xpEbynPggD*X=ar9yl&rd)U1AP|twx6|M;f+1T@9vr<}26tp=gUPZaL1Oged0C z&yH`G6gjsrg1sA7(cOj@tvk;853slkhwi{p&^h8rsfo_1@SEXserZe=yY}b;9ZWX) z+YR5q>1kDf;&^xy-u!~&r+qgsryv@DW(^E`Yj{_vXHEP##Bx|kgd|dd=>Ih z;nFVY7|_2hcd8rK!F<

YTo}93a)i5GIIDi}L0}H#l*UNZjlu_c2nhx8$DE{$jpBg^%13rn$6a1Dw#$d3FG*$YL}p@O zXK^Ada#pR-<(Cy!>9g$S${C?T7y=Ly*tF~CY@E?Tw_(PwC%rnA6DGeely9iH&VyEy zKnZP4wt8|Z8GuCvWlSZOQFY=I;k=)(E`47t|9w-txzbyQtrs<|Jwl^HOd09PI>3N* zn7!z98kMK?uY}tKwUIzi%iprS?1QfM@xEiG>2k*1U1GWv^ka(WLDic2tk-k0M7=tC z{eyb-t;_?UmBdH&eT z3|WeH)9r^2@75I^7zOF~Z`7ZIW(_u70Z$&orKj78F7dx> z14O+E@ z^d8~SDKe-XLSLcSC)SQk16XnAn#V-^sV;6u_99x+l+-_JuwGMDi^u1Hz~M!=U2`}) znS6G~-;Jizcdi_*eNT~edX?cPy0+J*t;TldzdB1TNny$ibIk7&TBQ8)$Db}e6kFB7UBtvf0ZohJ@}p! z(Ejj62xCf+P>eKcumQs>um{stXMP?a2}kZ z9iubpiz_#FcYraOs1tl<@rZ^DJo6K=0pnFH%Lg$tI~ro$BH6VVMKK(Uqm96d zcwKtTt>lZUo^*x+sXU1hS4E?^2f)M^v^e~w+x6Uh?Cj5m=i#=Vq3oO3O@5f!$dSm; zuWdi!p%~LYa*s>a6kHm}fqqh+w0q9e-TXF_Gg7o%f^v+5!2XPWc6?-4r@B^pq6Q)o zS9X&)%tZ*d{82B|^Bm%F(am>;D`bX8IMi?kLRC*0uep8D}(Vy&+F z-^=&Hr4WBv-jbt^mx zgMLWk4E{Xx^C|3Jx;b1dh`d1ol0^20o>ZK}nU}WEJqDf=o{v4yUdfU?I47&PpU{B+ zK-$-1H>KV6)W@ytRoVJvmNar073s)hUF8_HirrsZxlcNMbQ$)~3cT-jTGZmu_?YwF@-hj9v>Z0n=<2NTAC(R+@5cb_{{~_*+ zU|5WEF!XVW&(A5u1GGo(6i3a3U*|LfcZ4%ed{@U9ov6jnxM&W|K=NyYfUgrU`a&Yr~m2rg?8y&b%~ zJpCAmVB+)Gwznib`8GV3V42F>dr4iY{9_l5vHG{?`Q0bwr<7q63#XU5lx{GU*sWJT zIg?c`>AOz!Q%@U3E=NM&#)r?hF&>ys3S3On1Fw3Uct?S^fROEY_G&On{{7j*@`hW) z^ou|m3U5&eKhd6=F&(}?Z{LLvCe2I9^$+jil*l1nJui$CwQ*}-VZIvCR2^es=qC4g z8_Hl&Nij2WAP>%f?b`Y)X?5|#6JM+>Yuo@&qg7~j+bYmn154kOs3$eu`*NY%Bg&Ui zv%AZN%x?>0&&49w!ke`HMmYY`daR+t++xzE&VeL_Kz_U0{83G1ozA}Mn(z7AARkmP z;=FMje?!L7P5D?kQ~kXxRFgEbNTEG+$ceNg54~0T`Sb^=j28_mCwA&z0R&3DL9ie` zc;$)9R_TJgc)PeH4_8H2b70i#;6rG&R|HlXQw;;N!s?oN>}Mdn0y=cz$5*G6 zVFU4vTCII@xS18W@8%3HYoIyA`#Y(D#2esRshF@hrEhAneROm3q-Eq+Y({j<4nxp^8<$cbC1t zUn_c@87qT_V<$&8)lpI&l>Sj?hKM!M%_FpLASsu!3C|m_gR#nI9dEOPEX|Eb_K3>X ztjU@dvClCwi#+9EvM08>k{wsgUfKI*Cco*g9;gTYX*;!CN7Mu|%KyI4E`Kg~fO`fu zINRTAa=qa;p#YR6sng@y(=x9B{MSUEVfv;SVo*XB0aBD3?56UKBl1t6_3J-H#TTAq z-W@pW&pAyRzC)%>P^vOz$!1$8VXnq-g4<=0?+|Ft@?pp;7Trm;M zn0GhrLVvW!?YTzD;C?<`@@tDncv?ZG=}Ks}`pXE_gAJg1h| zSgej%oyhBZf@Z$~5QM2mQ3*YNRFS^0vSsoi4%hhCknPYb>?6#+Um0SsnD+^);_}2_ zkeX3vd=VDOlP@shSW=7D&rgODO5-|f33o73vB$rDl1+D%MGA7ITB|uE&)6(rB~I+{7%6-tH-1?<7J0$KMJc7e1-NuRim5fb$ITw}zW> zSVvYGV1W%p@wfodBx4Uo`W38EHM>X%zFR2rT`7H>CvI1V*sasUi_J%gFC90B;bhlR zX^3St%1|rF2S4ZUO+!i?EJ8@uNXrYId;>uVs1~F!U~Q07&fFJ=8GK?R(hT9dTpAGe z69Ur6VAp=0;AXMTcC!!?`BK`L`ZrXMF1%w;W^%zbOKdjG+y3pnWIpz)mk25|>{4!a zzDkrCK-we!QF$tuDR7BQJ@F3ZJG^O52x!B#gT^dN@iOG4(qzWD+v|491lx`azcr)RNRG zdu;vYmY)21@VKLCEW{C_@YU;$jgtWYvZRicZtV@>?r*H0hA#5n7HvsizX=sAKT;m) zb}8^IJ;rla(AgBXl*H407w)cEj`d51K*7$J*dA8c0IYJm_!WjK7OngcAgjILH^s+7 zvzXxLb=oReY96M3VHTXF2K^z9xa!3U)yn}DYj*J}0SFYmC8o^y6b@eVZOZ8Q#$s{m z5(z}VGs{&eb2c`fYWxbt96{oHJ-SifL}ep~ASqX-Nx3xd#-Y;;W5syy<>;Pu5bhnu zyK`h2_KmK+*HrXzP_MxJvII`f%g@&DasNcmYR9_Mj%H2mADEFGV*r-bRdNJ`i_kH1 zS?ifAT(HEwrJ&R(4Yl!}I84A=QIdpZAoob}6$b4Z$-?+%51K61^8JN8Ho`=D=$5vD ze?k^R2SByc#b;vQuX{r#w_9Jvi+-M>!#S`+BLpJJd`@bmXCIa4h%Wx13$^9f-fO{q zIMgvTh~NCK4y>yVb%sh3Q)(%ay^L4UO)w@Hb7VE?DVYl?NHI}2G z^VW{bJp5VJw_b0 z*(rW?%&qG(WycA{p>spWo2s)qZ)1J2+Mqqy_im=B<;zAEXw(NJ;QclbXRr`$c#Q5F1+ZLYQ2g^-gN_!T6 z%%PcNiF#lYY>c)iW$zU73yCQhU;Nb)`5P9IQlHN39UpYs4b{5F+lyE8%iQ`@82v^% z_*tJE3s>9>p%GGqZ%A-UbG#|amto!8F)!@$gq#lBQ$mF|H6K~QJkTDsy+p9}cIIaz zqqsk{H-WV(ixjC{9%!-TE$awi>+3Bh7yru0HJRAH@g&nM-Gmpi27=uyOKAYVaIJnO zc*Un!SN4`B9%;MO_end@+&Jt+nCG|vPeCG%nR<@88r;`Hy{TOhh`0KF(o-l~V?alR zMxPzS#{RdL_}mX$^}wl@gzCb^amf~-<9sOsiReZbFt^6uVed6wm#A+bhB4)^ag45{ zJM93cOaPJkZ&#_9@bWJWlsJ#53=_TO*EcF3pU1x98p^A=g_g~%X1Xzwwh-3_xxuWn@H7)hr}*dEhR-Vom$l5U>hVQEF?}THmQr(t{B)*T-wE*3vm+im zx7A0mBq%Kl*)U7*1;yC@`TPnf$hv-S3Ha;=f5`f@m-7 zdCp_S*c(=+KqN48%qH>;+VQ1WVStRKyfQZP?UzJnd1Yqj%_T?kj`h35NA#YBUZxVe z_vyoa@Zw2Pd|T9=1MAFtMo>Wkx zn--IzKNmd9EPvB36cYZCfXPO_e=kuTc{%$<=YWjmpx*ccpcX7%zwRE0{B7F#5F!wF z=IP&(U1zPV9wA^Czfx?=6ydJ2%1kJE5$ADw&(DMUXlj&gfEl$-{_d^O$@|x94!DwhA_nXqGLIX^qiT1ydN%i5{NuM zS93Q@J$tPb8teHy+#5Q1M^c||b4Z7tpCV_Q0VOlHl!C}s7V2QDJ#Th4C%L=0f z{Y#xdJe{xhw7=!gMvdVAOhiy}e)zN;TjKU?48PPAIZhMZ>A4xU+KG0J(&@tanK<_{ z>vH{Bga8s=sbnXi$~~=ndu<%L7aMg4G>VX>#^V!bquZ(_J$v)y?8zdSxYE^Vx|#f( zyOJzDO4=oT^JUYNZ^$T=jO+Rvmf;n!v)i7S^4E1AE!F7lkU7L zLPZDm=U)QL!C6(fe(g`Gp)Af4704kq%Jxuv3$@LFsJZk;1COU{xu)2(5oZCb49cem zFB27ZB0LEUU&AF!@r|#A(Hy01WK$YZ9b)<#1|N{Yyz@z=)0_byAK&2fjLxEwE_c5# zg9|qB7C?PIAW7)hU%5V7jkTe1lT>a-SER1qJ+k$&sajte7#NBu0j+?1m}6Y{N1o1v zy&q2w!X6~Xc&0HEkB<0*s!*_*pUGx_%gWpUl*hheU9SAjmD+iI=0NP}MPqzNZXEkx z?Y!kz6y6v1J;2wmU3;Ik&pvNHM?wR7;qA~`VgY22zf;IV12QhW9pW3Rry@OB zVBpVF63a!3*Zm@zP>mFy6`?Y561j42 z6LJWCf4Xsi`D+^Tqvq;~#)X>U*P1o+D5CkE@UkOfA2J-<3W*mLWgfHY>-f>%P;!m| zW4?xT&Cw&8{K>o=*N@zzj&s|4qsku4V8S6HZpLKpONa3b}o34Cf!Wo(iE047ASUIx&jxjAvrx`0;~ z*Bg+}f$=k&^#jG#zjn*Vhcde~oD!$LhYWEWF~uI?ek0-bZIKuCp*Q#&X6V2yRI#Dn zr8I8ttD`hZcr=Z7q&<{exiN=ZRxBE6ERPR1BLMCs3<%)&KU|#dWYB5DE-C$PWJCGR2`W?^AnLD|1b=|hEs<|89$kphceO^{(zzGKY{?~_S~ihBW<6j{!B3m+s6^WHx(EP$6;I*tl9GVGrX$Xl&mZS0 z)Y#{PY1nSSKNuo}wJEc(>=3OPS-~G+nm!9FpTRn3f~6>!p@gW%)*YI zr1GKLLEBjGl2<={6<%*klV-$iJV2hQi81ZRZ}EPf3$JBfm3ih-$85gPn8hpoAd+=W zV8<-oew?70#Vocm0`x5AJ0onM;Z*stPPF;Zod z$Fs+l@Z=SogaEQ4uS6eek1;O(pi%s#Ke81gZ^!#|msA@!XG5;0ff-aJk|=FwAydfK zRz~FWQ=Q6)a}~?A|y;ANWdLu#-oU>nLG@o=mrE z-Jg!0xQ3)!L6Z#m3DB-$HmjE{$q7JKvBV0uB*mI3V5dTLx_a?t?L+NLC|(r4p1e%u zdVJj9G(iNI_piV;-Dk;+&~f{tH1`58vAItBSN~mjSYVsVq!D$j8$q&|*JXdOjT(wv z*@4-Jmi4H64TugFUULeV6UQ>*8`O>J9;5b%+(O+UtI=vwjr%{$-UlZ(4I1!nDT!#9 za{&feHh}%q?pV6-Ln#~h$Ooy+EI0VNN127fs_HHTCyy8EJCft`q0>kC62XBU`-Tnkd21V)^L&nXC{Z8e_}I>7 z@02VuiFjqFRZBoRoZ&Crvje)qKOhiHZGh8}dJBduLF*nP%B6l?*egBbT73BE?TV5d zQNHvhNg`&Ne+cw7ds!GNxW)V~Ezg$ka7RU;l zzSC(@NG*>*VqG2SdjYW9{Itji7Qak9nJ%z1R}1mZ9si3RVxoCb1qY!EB;H zD>E#NFUOZ`07>e`{eS6Jcq90oh``fCYo+a%WZ7V_k(Dlnd`OFu2o z5v^gJ{Zd;Ds9{_T3Hp$*AtWLa8tNbAvRpi2n-Sy9T>9JqD`chbucLceCHT{d|2i~| zV+CmAUx2)4^kqk|QF?oOnXZTO2tLz-Do!uqtw5^@^t7Q&FL^R`_7^IdV#BYy3KU;; z0F2s5NbxxV<>%cb&nQHQqYL;{XyS`fdJYJfNcy*A>F7b;T3QkM%gWS3mAQ_*;kp28 zl-Ol80nGGD;bsQ(uAang2|4{vHI@VjT=ZrzKtd zE#$VelcAxaOnnkn)d?(V5Jlaw_ z+;wo}vsNrDEtECpi=!rXCd>FV;;CSEO>SlPZ+MOue-*g-bA|@#UBWy8Li7P`<0eCI z`6=`Nr~qnAuX!=Ibn`jUTh57Wb@OZ(G+7So;G9bSj*E>k``e-B8zawW)1l7!v-;A^ z8VOx}Fv!#DmuL9JBzTVYEzKGZMM;o{c@?1^^oC*+kH!;-LLb_cdTVlEdr#UqTu?Z{nv3|X31WQoN_;uA{F#e)D9Cp9h zh;_43w2jQtU3H8}VO1)e6)R0!E&H6x8cs-W**iDO^i3pM<+Rei#6uh6_n1UI$;9 z=>vrq6&b4Bu@{J6TP3uu?FK8#ad%I|vOtT@sfLMDIF>*}96osm0VPH)x@Neoy0KU+ z2m5%+V1bkWi(-$R6odIRH-cP*hX0sbk^)L_4xPHZfYk!1`l+XkCK$3m%4L=)Azl*q zkb;Jlv?_7n7P0paFrScIh$|Yi_dTl1p%3&rLMuRfDl3+8^kG{O&ZFkc9-#ppu<+@> zSbBUd)zf(AiVqZbC!LNAP25kEX;#6B2}q}P-dV;e*J*>e*?HaogYMz9JC;@Lwd2xt zv@d&eTfe;9SjqMF1NsuzFI;`n1p}(fJL+I!Ul)l-bZu|?j8II40Bt+s`_(53Z8yot zc@_HDEiY`>C$d}7>UX4g{5j1VH1VxX02+*~Ja)@-!w1E?kO|0W?L4AW5=I_cYknL>Up7&quh zmFtQRBjmz_ne)%&Q@>+c+2~xxKBEF?$|y0kFrY}C-@c)jWkDauQ)6RblH4!tX=bT} z5r+6atGzj4fslHFEn$%ldc7cB#Ap5$fmr%4a)e1axr2b7hNWcTZmrl8{C$}Po11sP z6?5?JX{S3+`*}%Vv}kC?m5fDrtxuE^=h4ILV`Y%t{)-|Ubv1VB=yx1q5Y@o2s&G_I zs~^W>Yh&xDlUQWYR1@v{b_35_&J&C)DekhEHpNp=T@{YYMA!Pn9(u>N0w(b&G4-%E za2DPUkZMw?{4z^@-!%FAnd*?uNXdl=iOZ@XE#``U&=Rycn~U==DVu)m=1H#<9G9A@ z_O^zvgycbyOmm>tDS_vn=1ZE<->|=}ZJLHuSS1NG{de2*VLf`>mOAa7&4S%n9BN~G zkAA$cKDrj0N&bo<<~W@yI^z+gFroEp^s0wZ1v=|J2RZvrM7ZR_cUJz&_6X$Nv}>&< zzQLsuC3Rv+n1ZE^2Ome1@kaU$7UX|HPf|K(_i{JQ_bfrhr@SFX`QE5m@dwg1#YF0) zJtN`(9@7tAKlIZ2YJj$CsgxKu#W&H~Aj<@R48=I%fk8@jW<<9_NY^D`g7pvPNCH>i zKl2MVU9ur-K3D%FQ!2^-@s1`=3vUlN_i=pIY`mLz%r^pbhE{<&2tJAMHPwYGV8!=g z>G8@Za_qkE_xo@U=EQ&CfJ+>m59@CUdA%25SWsK>w#=OQWd-B?Y zWSaj(rz56#K%39mn<+EnINtd-}HVXQK14!0H0)^d)d=^`>DMIKRV7O~W6M2Rz zAmknPXb_Ms88?SJihsD(Ez#EfShOb$FVQ_foGh99JyX)PjNRea&*tRyJft+y&uPur zsuF!VO8wGVxO5@&!DiO0=+d2d?&k>j%gJ}?KPB+dfK5R1oNxGc^&BLwzmT4GS|TwA zjnm=2i{RV5ak7ZrbdD3<%#!63Lp{D$TJcGJ(&vzkqSGqf5!3Rd8X??HTYd%M^xAK+ z)hg?zj_(7y?6*Mge;fo z6VN0b(}a_KCssDFa?9FFjg3MUtW=xwicjJXSTJUwHdtup?RYBzi|uRFRsIYp514i% zoyTO$QEH9h$N!Mfw{Dtl>nE@|vOA_PkQHQul`jQgvvUz666&7=eR*jN*Tz)2KPSpq(L*mT%MPcva@JK#-+3+OpawsgPpX#4}U$@3fjMmfJw!{{7?<2BEb8qh+Ef<3r3b^bFK^ zzk$$H$IZ#CEz8%LB<*|lr*x&%Kd$7Lj6}kXzFXLD0gI6E{m>A~afiVOVY{@Y3IhDGCK_cS;HfS=dw(UC)YGS3?P*W9SX|L?hqb+?-_w1L*r0D>V1KBamTe z4n4CBrPP1p63YV;RQ6@buU~WrxnXgJxoD>)@e@!vQS9rXW{FpG%$@D zR#P>9+bas^wzk7H3u&>OtoKT1q>sG82LR|3xsVEgM&MH_r3XvJ>bwANmeO{`uy4>t z*yNDZyK8zMf zJmUl$1;r&L*?nx5DW#>T-KZU4-(C z_GcZsOvGo77TjK9rWp6u*nwDCmvZ5w-gHa84}tqByy~jF3FnG~#x{bTej+iFe!Y!U zIjSc#;cvOWo)@6MI^h9yW_=+%MyVi6Ddf1khnTyVpS@%m)*6LWU|VWfwfi*(1~g38 z-D_aJfSS13H1N$b_BwxM6z|3CMYkX6js1xv;dmzBhA|#LaFre-VYio~fjrxyb-juu zMUYmmEc?+eXg=41@oIQ@9Xl~8%o{&G8Bemw(An>yUWz5Vl^rXMei1#X z2v0jrvUqWPZF(SsSy$Qp_2U;AU3A}57Q#I9K_J)rWp}*&g;Epwa6a1l*_|u0X)7=Q ziLOaam5dI171URArO#i|Mm66xKAT6vYppy@Vv_6%KX56lES0j)db@3E-?9OZi<9!V z;w7>|t`LTBY3g(js2dFfnwGBuRRVL;^&=wB5o^yZQk|M>4YuUvW?_dc=JJh zI6qPKDL&+xO030)Kh!PiOClugv1y$RuRsZE-oZn+hr#uR&4U9Tx?&<#XXBkckAKTbj%win2R7*OH3gfgSOAue+9pVDJv)LSQj@ zj|=Dr>Ln;NoD$F@gBck^)HS%H-al}BBshVT=wd`2W6u;FAblL%fp8PnDY<-`OG3df z$1WBxoTBoVA?QLnU)huSVxP4H9Q&Os z4c&)%-_7@j?WG;Koc6qCzE2cRq8N$r2I?gi=dFFr4bpo2X|cZ*!VEW7b&~xEA_nx8 zWBrPQfegdn$^t0dqHLx_IYFrjLXfF$7mx@nK;fu)h4!q7+9erinJkS5tClDr1xB)X z3y&m7!RsaNB@od4$I+^mOR)8Ok1dUhCjU3pAgmifsSgoI4`0O@Ed`_!6B;t#V&e^RPgYY2A+QO%hvw zFJ}*{I@oFKB&_~WltRzs5D*EG`QmP!ir-Ph?Iz^Sthxr9IC)+PbYWdi_G(wk^6_YWk?Dn?yy)6&tRl(bf_o8QgK*ySG}|*3Zyg;%cZO1 zb2pW@;|HH>T_9ct-cb6;u^ z(f~!Fs(5M!?!P-;O@3%6vAR7CSj3DG`Xl~+p>qko!=YvM?bhcNrk>|)B+e?CxW}*6-pP^9bX;RZK^{Ljr0`h#p z29o)psx<4k&|#Yd)+EM=Ui#vA>0Cm+TgFwdI4$7Zi`TT<*cs}7sSLs=^%N8{6pPu2 zc39+=7c~z#vx$o-cxFHNZSEf*&L|eY={*4I`|tTdHnO>C*+{!ZL9E&8Sp2mJ1N%PncJ)~bnPR+|Zl|GaB<0v}iHa80gs{n3>Ytno6% zYc`G+{y}OB&znE_er(?)<_PF}J{?|8YB{8YVkY=NI>@tttP6Yuw0S~A-Nb&^tuRYd z)vzWqXD$6sXKv2s#=>wEmr@DVTMVn=8qkFifG(T-$~V>C0?q)DY_E8PI&S^62;E?M z%`|C~`mcd-IdGlj_`>tpRD7fxgrB{&+gJM*6F1n&*~@yyUNsxrq$lB?!IMgYJnN=r zY@~Q=I`$LwH@i4Hd_7+A!hH01m4^rOkr34ZG zAbIq-ZG)Q#-B*P+!wOuJ!3UKnm(k17e5d2JpmwUK4{d-t45m8vGiW~ipOlNFsiP3C zT8v7%>aLSb5qd5PWu#TxrTWiN0e3EIX+nzj54ETG{bQa+{)jfGVCX=l=S+u3!{3P0 z6?U3L#7JuiNkzxoW5_g5k(Z1TSa%}#igEJ+pn-)-53zYu4<73mvGH>T_bK6J{3Rjfp zIvkFnEtwN6^Q@Qt7tY;h(RD)_O5czbYH#mm85z(K@_zT^P~fjvsSgBqXmmwEIra4L zJ+lls2cDjsD<)9bQBdffTh!v#0&Szh%lEAKrO-XaOyU-O^Na3(Gb-kXyWAx`-WO;q zd{HRT=ua3gg4h8$-z9{X)(&#BV={~?WCtwe5oeILLN_eV2-4P z@;E?7?muR&sG>rrKzNxRxk}w{HbsI_T9_Exz`@asfa97>S2ww~~s`LeG z6e9L8DuZ_6Jh}cLW`ok6-;V!rbQK|5y;Xqkq^V_|0^eq&??I2K(uoRI=u`QV{OJTn z@BK5s3k81H+e2k&OE9XLKgj!|xjWieDKC z7Cbq+Ph*l$C^1D**-BM6M#_|6<12@0As!oL>i)Oqu>)o}SvTy?c0>5wME0u$Mak!_ z(679DwdEIf5s$0s@7pKlVXY!vZJQ689?kyymz)^+Qnl$v*2X(>!m|hj>)7Ez7S_mVgv}b37#R_ zAWa4FNYE0wSaj%!S4HYQ2Ks1hd=C(b;wQm0{gi%v{oh0LK&0d5PY!*6BQ5HTgMl%$ zidNk^)k@0N21=&5g-+{$cZFzi?I=R_Wuu#`{f;8+`@Od($3p^|MBZJRCG`kN;aYYxA^)0)Bp58{ZId|>i+=DtS3AG GW&i+=xPGPp diff --git a/test/fixtures/adaptors/http.tar.gz b/test/fixtures/adaptors/http.tar.gz deleted file mode 100644 index 1963a4d5fa7b5e700c5f17e9e74a655c4982193f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5641 zcmV+k7WU~MiwFP!000006YW@cP?Jr!-y}dnlYjz>O0$3)T?@~nUpMMb%Q86hgNg#SfLPShLR9s9_6lxch5|x5*|BlfAE%|tPTX{kv|6M-+ zOh)8i<^L0k)zQArNXJ13RT*!ntKJ3x1}Gnj1}g4&7T7@rt-HFh7XZhWC?CwDBkTnL ztfeLLbq$#i*jUV4?0aS@>UGKU)gD9yjT!9~P} z4$k`Eu8wL|v3#3D(%!NNN{T6mgEYI^uk@F^@1c;bJYej%XPMUwxsv5xK{(I_mKj{cN@q*8*R zaM9VTziE{c5%4K5{(TW{R5>@$A!RG0(FRn?YbxD4*a9J3*3tZGqNOP!n;Ot@(a{Q{ zlo#PyBqcqO9SxKa&hjuhOAlQfR#gdf_R;!9#B6CKw&=*+QqP5$EA0!j-fM-wvmUt+ zV@eNNm}*)UIpcM~=S3!jc6>)YTeL9N7yi`sFaClT4Lg~ZW#^i z83L=@i0I+<^^8qXP`97OgPIsmRbYe%?EF+ttm8PdnM;lPg`D*F=pv(mc~Bydkq#x+ zaDvWI98i-EW~w~{v_stiL+8Y|5*pllu9LX)d_+ffL{l3R9TqmXa&HLURLz9Kisr9) zB*B{snP{-*>{on}-~cdqT>xn}RDnub)tng$@F}1GebM9|0ntQ80?Ste9HplKe+cf> zpwmYKkVS%V+T>nz@q>XT-qSFgTPSYq;ZW29z^)-q*i~G+Ndw}zun)^jKAr^Frkm8D z;Q;3Uu>WB!1@N`UT&@NItq%IpAjc;Pz?+5w= zN@uCyZN?m^vy}huPAT;vUP|XPMi8`zs8V}-u?G%N$v45&C|8r#pC799knl0kRH=v? zHIj&9gH%LA@ZVG*Vu#JR_EXu#CO zs5QT2Z;vX=y3k+qx=*) zP0Q@JHYdBicv3tr$}%%+O`Os5U0G~2YL!)Z)LPW}jUAY89mV<|So7vg?+R-f<`~{@ zRvuhy(@c7iUh6%Px!##xmGfr|ZQPmiSdAVxis97Ubg1K~7kG)WJ&PBcuPVNLv%=Hj zG8{LFX((RM*c8OSJFXx(pqRBeHkQwn>@d^dPX#J1u9A=t^P#jn)}oi4)7_=zUZX)| zK%S-pMDo#}kh1*XzV?AzWsePa3*M8RL)&D{XZnyjv)Ml&soc-iuvO2}^%NGTGfl_) zkllQ2FmyXF_gI^@0GH1X3Lk&PQ|dFk+#iZZ3xaxH>wO4dl`qdKnKE@NJcMcR>3S2x z6Bbq?wa+E6ep_lmg1$nJ+##Edg>a`=k$2x2;CXXCS#U!Z$R(36-Cvf4R_TpgEAxKk?E@n3`QwJ1$;&eNj@0sGCb{9zx!$P80|=i^xz6(>F{27*|5pG zH>cX+Y;t46eD6BPoyM0m=6i)Nz2;{RxVin96K(GWRy{rtrWx8hX?C&ZrzqM^8@Qyw6(gGb3ffJX15i^wV_^oT-HIsfq?YzgB zBf*1L);4*sJZZ7SOcYvQ$C<8U{_JpbB=BQKpXMOk6AI9Ex3GPDyysH$SZksr>E!?P zX$a2tzQ#1_te#w{G);@qJ~gWlRgr6bYueu?QO^&MyIoV>}Baq4oOsEZ?!>y=hxYKrX8ny{}==u5X{! z`r=kjtGmVaFZC^^ZU=j*ap{l;*W_wn)Hj!I=CYfu(2e1J@LUK_cGxAG%RMTUx9XEN zb`PLkUoo}Y_-He&p?o2>t}9+sP~c4{zU%e@LK&mwVX@63LtO4_%nh6qo(_WlWniJX z(YVEEQo8HhE0h44NAGB7GA|W5852-#a{n|KTPWOpLq^WOZ@y07@LlxDd~VoKh=$+9 zQWj*5H?DDS2j1~dN}g}Fm!&&&X(kjGg)Tk%&G#8ogi4=d8-Eom$9yTX?I-i%Uggnc z@~_VRZWQAOuGf_P)#!|rC;Yx&7+7pIYCwe(7P!`lyp>^B-}L0_q?)|pPVX}}&fDbS zl3}e+_uY6Xhdj@Dn4k|`bw8Pfr0U{I%x62WB@n0Qb>7ATBu&BD+Xp3Quk?pJ3^v!T zJ4^LY)^lPgM0IhwxG?*b8?*4p3SH&VJ5Ofx%BWur8|=PK80l)nNN=G>jqj??8%vE0 zV$Ou2MZULF+QEB$=dqB_OZR~4DVeVC$V8u=e&cP+CoX=GqeuGHsF)6F_3}0X0UEow z+Y_=~_wdd|miqp^uA}zD>J%U6JDd!g(jYk=XTy+jwfM~q$$&%a=}DeXcr|Q4@dJ9s z7N5bjY$z+VyHfc6%8!wbLzxvPjf>D?i{S(E$jIkfKc<8QFYfur%8x9BYu=vw%ft0- zpwL&}E%J{92jgc`gFThQim#o}x=0MdQoE0?Fmow*%csSVS2+FVjf1p~dXhAKB6`7? zDx#Jh#@ypzUr7k7gl=5l^urR8!lgME=I8AibL`USWQ)reNW#+aAih6069$X;(G~Iub;7 zKjJuPRuAo+*Wq~)IgJ3JKwO>HO|3MlUGMPTPu_g?v!rh*6Tr6;(%;X61ZA*SZ$LAYqR;<=by+NM^o8(L1;H^l0vlxFX0_E(*b4rxc>DRLdYM zeP-AF!wg|f6R*v6C!dcVJ-bh5<%MZ@s~Gy!&y}&O#mt;o=Co{R)Vj3DTR5c;s=B3S zB#*wZ$lsnT+vO0p=fV!$;u5nRM- zqz?>!09^nMH7WCTbby_l_vfQbY-^%|5IkpnK_%+D_KiHrSmV_Wx6;Q_2O;nzLzv}8 zM-{{Neo!p>t*VrW=cOf8@L zsu_{kG30LRYigin$DG)XX)9*_*i*Gfz1@`@BySug1#3DR)VDPs2^rvGL@5e&M2qbp z+Z8(#2MDfr6%S5nZc2~u^R4rPD)DBU*-7^1W&8FBG#l)v-&Twrn_2pikonOay_qOG z6LN3-pUhgHeRhtLYeH^t-00;K3otz%j2qfedai>Hw9Drcxa6Z>sx<4=aoTI zdFCH|Um?iZRjk`1-}pXQ|2dlZ^);B<*7A9S*}D`G1uGx1A%EL_u5Wm0&#b%(CT-eL zBe6*e3x0xs?WywKdo97&H*jtF*7RS%9XVNX2jLYgdlq5yejw;YrAx9#3qd}K*{78+ z=hbpGmtyIy8oAPfds4+g!I^q%sYcKBE@0redTee~W#8}y`U+!Whk?1FCgizK%M7_d zo}+NB70-W=K2~C`;FBszxj=KsW^xgsc?6%+6qjf+R$~^xm*0@dwe8ea`lmxllizvL zJ;~iZYGF7ht{xSWr|;=Sd1kbqX6x;iOLkpY^ zOy2G;)_gtPFd-eH+$L#5^W5gJIet8>TD;l&vQFE*l=c$UD?U3J{aw%-TpgOh?EaCe zJzIN8y3Tu7tqG~=Dr%v=Kf-x+B`?`3T^k#}tsCV;a~kX9dq>axeC*ZfMx7DQZU$Dq zi<~vpE1R3s>{LTbrOkYI$IqD6~_D&lCJs4;0jb$P>Q zNpV8Y!5Wc=i!5lx!rz}1LYRZMWzmYhZnVvOAsa&_JdqZk2V+;sEuqSgi7ouj?k zk5com(=nR(MkDpNp*YA-bw`^v{{AK&!ZtUFIO+deYfx@cws*I)P`v16^1W_LZMAS!0b%p&X8j2?qpifX|#f2b+)mCo<(c?C9f2N?tP)q;{CA%uHmE| zwY)pf^z=4QS3@_SzNEa;3?n_g!WS*`TJ420#oJ;Ykx$LI0=5M35iGnp`IrVxRn7`9 zW!1J}HrVirGTpvs9R@2IvP&IBS4}-1!K9_os5N`*_pJU^gmf;d2$J}R5xT1kFfjOP z*oE3t1>k%xN}7g^ty6<9u%?s8Q;V@HDc9&hk$Bk%-E8yxALh;%|1kRT-a{t3aHel3ZMy1;N_!f@}1*C?(=|W(6Y7N7=0*9Gjc7me1rBP7u z>wIV2<-&OsDE$(-#i_!_0_JIOY{Er5C%x31*@d9yemlWLE;uOqDGdMhDIme)90vuW zztCSre7BZzQSZGG3q#S*%OrtXM}ZSxXn}nt6H@W#s|LwWLcR}SfC`!xTErDtilCeB z6>+Fh0&FoD&H&(2hd;zp0&E)uCuzfB@*G1OE#DU;NGWRJ)Uw4?W-klyrg0f9Db){_@X<*eLw_SSkDq za&z;%2>=ZTP7d!bc*7uOaujAU*PIz&Il#E7z_bJYtgab|03hK>%0I~r8j5i1hXZj7 zY(N=giM*Dueoc=6lNZB@kL;AxG1!H^6l1$CTpU6uVe z7Ym>=i%2H~u!6|@f00f#RDkmPTQC)!x>~7V1_-F=5F?Kv-mpO6zV!k4L;Z-u{rXyN}i{aP4zN+)$l#%RE# zg8qD*P7z6k4M9Ed$Npr;0sdo3Q+y{H02NM}DvV2q#DD?{RAH$)Bo;JSFL#rU2s_J9 zjzI&bEPYEhItykhoQVVLe0a_ffMqd#;+8#8Z-)ZnXqsO-n>0M&duniaG$)GRKOPOX zBiW!zG&dM~_6{4wNPVLAHyVWS>!}l(VQMR7PZu0VcbLF13pox8l=c)k+Wi%K@~T@X jS>GFK+NveWmJj|96VLyl|ImNv|D65{Y{eO^02}}SSF0Ih diff --git a/test/fixtures/adaptors/http_dhis2.tar.gz b/test/fixtures/adaptors/http_dhis2.tar.gz deleted file mode 100644 index 35151b03864d69b556671db5fd8c0124aab51828..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29838 zcmV(#K;*w4iwFP!000006V!KQP@KWj=L)hcwz#_!+?~ZG$U+E^;4Xon0fK~G+$|6w zID|lepdmOcZov}VA-H>hAaDNnR^7L&y1Mt%b=6eQJkLz`^vth&x~*)j+`ZjAz4+a& zY+hM8*m?2WIy!j?@mqO$*?D{MdwIOE^0eb~f9(MHe?Bv^PkH9shs~dM&N(* z|4%3yp`(TmrH20N;%lfY>jMBlz<(M9>mM`qEVci~aNN~jcmV(e`~PV`j}cTF0ATK~ zp{!uwmwnjgn{6<)bp7~{9pGS(YZem|Q#gz0TyBYuZlmzmtLb5tbJ9)O&f?IW=hvNo zpD|m!&`TU3^1;!CGm5d0vryka$|WZ=U~KzOeq=xa_KBr%(m02}b%2>UZD;NL%VDh527m@Xp)z^PXVent#n z)AY8~1m!D76KH7wQaLA3T>r;Klok)Z(-DvFVR3{IKRHW0*+?-mIV&SL}E(_Kvc? z9o4;||KwayOKs?bT5hvMP-HYfx2i%$xvG))KydDMUkMb2x#Am~_cc<$J+HhpOjfG9 z-E6HB6Ojw*TFe4~z>*s%4v*+y(H$W2~Q|;WDbR%FXi}?br0@J*Ekcg7}FMY7h?k zQwI>|S6RGESv;Rv44OSrCqlhT8!sU?zv(h#8Gsiq#xFgEi)CeH)++UyFbJ6Y>ySyG)Gn`Npn;w7L9LTT3`=; z3c$L&Tn#u9jz)iFy7VN3FJ+UVngImI_-QzPzGM|&J&w7O0rH#Eo*8fcki1z@6C(of z!pPa?<(5!OC?zBb;+EwAIZjrJ!G)c&&w)a0*<&rie1OLQty|KR;SZrX_*ZtTK}lms zZ(55|WvhQ!ItqvD%FwqpD;P~f`g|Yn2eift!5S_Wp(^PGXQLxOn9lT<;D|D)h-_u4Zt^7 zym7K?tO8S@=aBz`g$W$ELaP!akwmu3Qlq7qfiu}B`;B;{h!rr;XNQJQG9(Ob!jHx3 zPH5I~OMUD+n`=sGHH$L=GI4eI7=nqFJ%;~0`g>&L!sVF42xaQ;g^jMU_k?hsKR~Lu zsg-1H`UngXlsM`ZE024k?>|#jpb{9bRIN7k!zx|T@vZ8$)Lpja*^ zuxrXcgK7NW{9(8Vs&M~E3i~x&mAN=JjKq7yJA8i$0iYjI`XJ&b);x?22;&bix$7^C zQ@K?9oYGq_kgFm2#pW!(?&y83Lrsyf*YxhgXxNM!D zPYW0BD_1B%pWr$Ql9X!#rk)dV8U0gC|FCGySSx?6gj~C>B~dSQ&3zZU!RGD<*A07x zVsXz6QA17jQp2PFX%$I}%8t#W3?Hb;C)kRf5T++aRq24 zXekM0FaoJ&kWx?&-JwIm0-Bwh*#pNgB{=tQdg3xKL>onN3*4}liLoF25U$;Z8Q zLYy%F0)L`{1PlV@Ps~S5Oj0C8?`wqc(=!{2ccgK7)DGLcHAg?G%^81~%2AuXiXYQu zvV$EF-cfFFhMkr^gzC4U)c=W@X3jE5rZ5gif&RQbVN+0OPbU@2Mr}jj%FKkxJ z1yA=V0-yh-7NZSy&|wlAE(R80yAx1J0@hw<54$VK{GNOGhhNtcerAeLaoS{5qgMMh z0v7Bo@3BUWL8S;43DhFUB;oTCUCi(0b5Gt|mI2pz)bdkz0*llB)*r}BSW z`Cy3=N4ZM6s+szpXqW6bA0w%1{GcdoE;=cYB0Y5Fr4afr`WU{VY=`Ggttb@os)3dx zpc(8?@Bm+RA8$dvt%-AxE?@6wnHo=Z;aut1LbgzJ(+$QP>_&r$M4t?>hnZl%4`cs@0q6vut-b||dR))P_Zky8 zr_V6te=vRA3bcgi_gz9&!WAl(7pzHZ;%iKWq!Gk&2wsUhl(b+$8af39hmG@)8HU zUF?1zQkUwZSMZ>aJ8`T}34LR7 z>kb56p9$=&Q>y;_%!LEk30=7*duC7K7*hCj_cB>EEz>i&HvSUN=%g#Jk zUxa<(e2j7s_^Q%*A|$Kt^+l_-_fE81J%h!4tmT>&k5yYdb&b*s@NONrYHmry#$pvo z3c#AhX#_Mb+MOs=w6v$FW@0XeuUT0-5gj$*5CuOXjEHBY^>X6(V9I4s{&~fk&VR%nIIHzj+v)HozO00E{Ti275<8UoawFdjbjqrL~-H6nD{P*mc(q zKT1~}s*gJJXD)8D&Wl(k??$y^OMxk&o;>4R%{1R&ubx3Dz}WddAElj7alExytkj4A zIF{n3`opkQv|l8cQxxLH;xBGk!laLU>DPqG+eL1Jr6tAzz{7w}QB@~H3tw7tm({@| znYWl1|2n?l5;FSD$m}iea`gzr4qe2TQNzY#+=GDfvE4xf0^?q0R7Z{9gMe)C3tZw` z&LK0-Uqx+dquP%6QR(Fxwe_kpl6g6hNxRj$ZzOt zLUM-~aH0+wijXBucfJE6C@yp6xfkjH>Zl_ePn<4+R9SKTa2tggkXyb#1x`>@oU+73 z53iq%g3@?cozFM#!3-;1TCIakD2NPbUa?H zv81c~;AkroZil7dFEKc|iGJ6!DTHB!8aS_1k|4Dc;>-4vTyRqQb*KWMw8U1{QTt(R zpcB(DDLY!fJqYqSz5`52W%9q0ix(M%^ZGS)<-80-NY`6V-Y?9TmWKtYG#?SN-ZdbC z@l?lpoWRMNAIS@7F=;Vqv9we(27OBLT?j#pnEs5p;krKT(%e7uz0sa(1eU;S4^(yH0M#1wMoey=RoB8)JwWuw0+S$_JhHTOY1X zE}Z^L_=fZI>nUP11IPhQW-Hmz$*(%;%~NyShi#FB@MF~j($%M3QGK=;t*jX3X#b50 z?+McY6Buvm!52^yj<-~q5mPp}HdI)>2MO$=$734VzrtTZYos2zOC;)&>0tS?(o;Nn|cI9##{{wvbDD zPV9UvU?+Znjd7-_R#)(4#vcvGuiPME&|QF(idY-z z%;E)5*`d&rZO9H1RMf06Oa%1da0*Ts@IYIjD55{J2242agV5Lsedq1?fncNX@9GX{ z_Ablb3M_in3pcdsVozLB;G~)KKPdb}kRW$43`iz?p{N^lF!0f*xO3BQ{aG>wE9&fx zGB)VF$5b>$pGYqGFELDH))Q$^H&GL`m_a3Y5TCr=OE28`FZudPTylR+=9^@)j%WFc zp#qFs=;ECLF3nj|jGO|O=sFT*uT6hbexxL%2t~x%qUv$#+M{G=L>$PVn}B_TyCd(2 zEs5#+6u_Bukur(N0h>Q&^kmlLSYIZd;_Kd%jatSDR8IX;G}?7AeeXl{iv9K46TW^% z?uOD|t>l15h7+M7Us(HA3Zpf*!YBO%cbOe?^ga4eBEER!=XVVF(7{lgoehF6VQT3_ zTc#{$Rq>TlJ?niiiTwlgR=2h1Wua6CZW6wkr}Y{yb%?ZF+FIO~8|kg)I7H&YS7@Wn z7ZT5QWwWJfEI(ixp&lK`pH1~D>Vkl`8|}$acb}({ckbN_LbCcWkh8SI|~~Z64sll4E(XJxeunquhDYjm=4`mvqE-y1fBGm^BHz>*5K3NX7GuwJjZm42Z+Fh4)#r&gU zDGsBWqD!wdH)5)j(+oDGkm53XPeb3Qvq#smr1c$F;f-S z6+t)cl5_=cq0{DI`{RMiWly|n+#U4_deqJcTIpMXkgXsT@C6`CQ$IuGA4>F{_ai@a zdM57IShWxe`S;6({AV#5RiRIaocX`c(-nulccR0iBynxkl8!jP*vmW%C86DHOmPC- z!>_RUFys^uLA$nv$A4nTB#7`s( zT&rptRjDCK2S~uOnyAx+VRLQx*Db&Dbqf|jt(X%q)oO7S zdloH*i%CJM<1AJ3@QJMe({H_FxungSI94?eOyoFt3RjORc2B`2b(c(yh6*5uo{@&U zrQIyr;d^Ci263whd$)-oB0l}|6i`CqNv4{hvZIHbUfmdd5+tzui|7hd66mm$a3zy< zVwR9oQpsoK9Y`RB!|xZ#C>Vj@E0){QeBqL|rf-R5b&#kLP_yKpTWx-uJ%HS@@YyZgakpaU`#T|dzP84%Cw70<1A5@9mvX$)wq8AtakQt+CG7CG#_^& zCuu$wF-Iu)r${PIarWhRML`U>McA-Hgv}*DSu0RX4merZ8cVU9;6Db)Q}ub<9r4nC zP$UU?3)wPWj?5KGyY|E?w1Fik6+goptEFY>2L>SNV}D^4^13+pP-&3)#?`uT22D$S zu-xF2;#eu+Yt3%9q|Ci=x_sV1HO5E_xnO(YBf{JO91A7qZ7NOr_w>=m?ubG-KyA>X z!ta2oO-lx4+n89cM9w&Ic}Wt<89PoLlBhCJ5`T5s1HitD^6B#{+M&CL(=6>BTp_3( zqX3y7Y+EW0hH{a;II+x8bTqrEn-22r-NTc45f5vEMY z9_u(^tjF)2e-ZKgULk_XpQ%eBd8LT0aWF;U0$*rXT7&%o=S+olhCF-kWBuz}N@ky9 zu2P?Fcd`08q6%#W-L6l}#u$2Qo0<-nY8&d7pc5gzgi(9+)6P}%>QZvU8I`Fg0TU$SEO9K@pK8jg60){OLm9x^E-XC6+3qJ!9D)qJx_{tj(#xJg{VhqUYHXL?I?o9dSO zuj4S^?oU7fEQqwZ^tmT$2)bc$W6xb$r|?1;r4*5 z;-Wa_Bhd_}Bht2SB+49+#;z5uRVti58NOwgTgt;I#>L}F6 zS0L<}LM}(`&lmW4@z^&$juWueOb46;?9v>d5U~WZF=T6TW)ZWIdXq7%#iI?%yvx}3 z)uN2Uc@#ZK*k3M2Pxb>sZDUC5 z&FdB14j?y|)?u&TRA|kzEE)_sR`U&dGZD(Ee)Neb8jZc`6((;(Tvxq@sCHmFpsc!A zNdV4KT)-@$qxbV=P-R{+08XlcF;#>omGkX4)L=8zkK9}6BrXf@Yut??qm`$_9gJAA zyFhh0$U%1%R6(FXN!yb3jgb-|47lj1q19r1h6>h)?0@H%M8Sy4Ebr10(iBsfoGb`Q z%IHD6ozH57mOKKKQ(yefAniASCfjyTG0-{?2jUDpV@a_>blh|jW)Q=2qzj!iYY< ze1i52GpY7rH;nk5=~dH=i%=h_cFM-yKm^t?lk%=cLa16<)q!(MZ|Ja|NmNRoFnkdUM5S7EhP>&9Tq5+Nqbc7~mgkL! zgpNdy}{iay;v z?(cq?E{D}JFn9ERWD$5!PBL<|7fk1c0G0?ZoL8aw%vZftJSvw?GDm>7!aq+j$ZMU{ z*-V|wR5O|!M0@uxfVPLltoM3RyJREb4QU|`1oeQ!5Kp=;t5U5VvRhM4gVWTTDyOM% zyT9zspDGXz&$jLXW34Juv?`hjLsL~eav)25oMYw_!v6UM1qP6u{S&o{J`5xvw0h10 zw?P0^+{H--5p<8+KlKdA)BCJR-)@w5wCOL1*i z#7awpSgcNFX=cZM!S%53`yxp!8i}IC@uwJVrqde%zR3*FWcB6;lQ>NplBc!ky#;SN zuY&R!JTAYwtL49gbYMwhO_?CmXB9v1Z-L;KfVexLL_NIQh22>p-Fibq$e&~=TOBvl zB;1=iB`0d}wo+ebhp_7-lb;W1u17)fS3ZH-Ll%%(JZ~`lW?sfTA^a!Y*Y{hU1GBxr zh}_r~Hu0C2JE#P#1mz}tikmCRWGWmoZesVJqbD0BM~|XHzmGtv49NC~WDm}Mn305F zk~AnM8Xp_nfqm396|@+qjfoz$3|2UZ&67H@XTB1v{u6?g{EAuBipk}eUXj1Tmwn%- z5oy%a2-~5y+#p@(wigT|MlL;wtBOZ8zWpa#_~o=Ih*D?yN(`C1j469yyN~Fscww+AeLh5+sl4Ii!Nf;pDI$uRW?YZ$N*w zB%hz<)^(B=?9JBhkg0errYU(M@3QW3dP?R@yc${Z+k=lLZP%G?q(Q`xCMMA}+15mBjTn?Vf*1k-D8_eBA>MtQW%RA z3_~AcuyUxhUbo6cSV3CU0#>viI%;Of=aUc=MAzaIuD!6k0~W_H0@J6JeWn;Ha^k8; z&oCS03d>^DfE;hT<|nHIV@oP`!T~bI;G*J6(d5Lz$)L#b2f*-f*mrUo8`Tsy#&O}d zuTNRYjh5hD2O3P1dzGJIc3ei_j;{F$jJXno8Ql%t$5;ev{uN@h1l2hMpx^AAIe7yM z|8D-82S!@{{DLnyS5ugLjiXdKWwrXjo{29~#viYmB-d0)^YiNPW~s|UJ45^*1rPlo zsS_4UTE56uQKRe#w}^?BJJq39-H8mEbT%zFnREH5MPMKa^J}USm#Dp1p(u60D+`uq z;&@XQA%T<&b)>sMf4UEC)ksVw=4S~wy4_8oG2x)RXAF0_RmVbcLp@%zQE6V!AuO*K z_tgO$g^M9)0ZB_X32R1+O|L;+>ZU`EA`Y9k?p*!H7dHOs7E0`XxuseD^|tq0<@}u* zpN2Sob3EXB>lXT0K&!>TZE`5vh=HMIuBoU8OdJ0GUrrDYM=;0}po0WiR{zA_w1gsR z+Ht0ViA;Flcd@Fb8}I=Y)D%h6lFBUEW+dk`R9$kT?q6!_OZ*B`!ZTOsMAT312?vD; z)C1Nv>t3uIcnQU@UZ^nEe<>43zTcUU*R)qxxUA&h0&H# z?IN`I)ukTh*v>nc@54Z`8%v3I&v|i_SN{sp7Aeq(H^b^drnm6>Yjcu0qjhxilUa)p z9e^YztHNR|LlD&(RQX-VlUL>y!gk0rXl_E^35jXIk%S;zJhdEggc$*=7qgVzIhXvj zmkp0f02rZmq-SgQnOw|^^FDv}dFrBJJ!Ga0Tv(Dtccm#Ct_JykuKfI?gSzCwR_xa4 z=K?HxX8eR1zwjIBbb8-Qi`OP%Jhup@I2pqMWNz$$SyU776Ms#(-&T8F8o$8p6B$*I z?(Y}{(!aD+>)%iv1$b(gz;VBNU@~hLqvBtq+ADiY%TF^6prwv5Gx-Pusi9S--X>87 z=E>N>h<}Bzk;z@Wl?z-2h$?_-c-2X_Cy!RPJZ1cAg$%fCi$U2F z9d(^KYS1D@C>JOI`1sNjx632z(KKHw>q)sbpMa8>ao~ANJ7oXWMs#y}!CqO{2M*I_ zSJG0ABxAYMjr3hZRw!cPoN+xUka4#2o}AUlE>X*PLiKczI%t^7v-ZWVu$Qi{t($l6;ca+ zCA@->iFXIsX`0}8X`;s#LrLZvk1B1Vs_E#*P$M=ez+O~7XZ`eaohi_ba<+r%R`T@t zR$C8SD3tLp%iew^D3i?J0Z(eDvp?glM?!0xT5gDb6h&InJL~ zt<5+KN`6gWE>?pu#jZ#|ig=d4oi(1ej8O=8&3=_COhpIYD^)hl#M3W;@5hi_P#ULV z8#&vax|;m`ubxQ&I5(K|L|p5d)I;8mgnBhA%Cz68UTrP&NCg3}LkCV)o)Lk1LjE9DRih+zR4$wYy_hR8jTS4EE?-aT^!U-VXe>x;0;FiP`Pi2N zq&D-+;%`fO#pr>7zsL^6tD|LKEd5npwJ&rbT1lxfp39kPt722nO)Lo@8yT+n*gM|< zGyFRmQ&6s#8L_{y;8ji$ML=jn0C->+B0^hud`*OQRvHUqr47U4n|#;&AMibC${sQ9 zg^oH&@kGn3bLbThv5f-+nQdUvGDIDAHYvbKBh$zoD>NO1kmPNp(ld2vBS6q7xl)3* zCQZdandgLPdQ=4x$eKflehbjfMrI7hh^Am2LDSRTb1g|4;=i_F9 zC#deyy3yu^oUyfwVLT`^=D_khw9~9Kbd5a!ncsaK1oKJ0z=bQ?}@a6SEH;Gp#r6}ac9DKk${H$ zAM>|WU+QYv1JkA31?2=NlW7M2hO)f4!{`4=JouG-lc6+DZ39!;P)+cokE-YisQ`nG zImb66MrDTn6`@b4u7BE_n;r3f``v^s5Z7wLtpGJ_H; z{;FHX4^Y=a=^gdd3Uf`p+A`H9G*A*y@=>;D+PN#b@dIMn-YlBz8k81*&}R0xjI+AK zg#|;Ie?dZbZhzNX$f<7dqovXAtIYXA-^(^bfRfb7VD%vsq(i~Db;5y^0*v^DzxTwe zWc#ZHDo8yi|lzq`2g948^C%XrI?Y`~@2L zhBTY!(`WD=-36hB;Sp=QtDyK+w#h;%%!!rIz?xmchDmC?z|80&-N7B&@d?IIT#0dB zmV<~IdFCt`NZLZp){BorR*nX^j!3tMXgj$V(lp(OsGU+0W6nRzpM(Fj7-t0~jg_ZY z>RNM8$b>z&^_I!aiO*O=hc|+F!W-w(^Ai~t!!=r`cP4%i7W_M)1MuMD=!(D_9gjYT zl}`vA02Th0>G}`0IW%zvUE;vkc;0ujFXw2}$7}E|R4ZCz|CQGr(dA4CTZ}j|p-l&b zMGe1|J&68Kia?%7Cc>6i#IXDq&h;2-#qv-Z@*Czw{j~-kG7Ry{4inEr$(mbf@CNFW zIdp%$iVetdL;YZT;rIr-dp?M}zwlrvoCy2z6kQ-_oGkKyX~eGJcxd>Jj!C0`CiUgfD?!*o@F=~h$# z5&5SRDlOV-w%p8r3M<&?ntxRP>|2piel_U#Ok?+l>FibSc+AgBC`NNq*dKv+_QYmJ zt!mmV8X=%b?+8Ymr~XpM`qsa|9l$Bgp&!wK-?pFv+bIHA7VDyyrSB33x@Rpx3O)o6 zAD+JWHc4^AHsnSWL?bv8k{k1i=S84?uGNcsOrEcjkd?(42YQVAz~6+Tr6Q>UCZ#lt zc>sB(H(1=ikt9yJ&<4^q!Bpvq5ETI*m&GSpu|tWRRhc@giR@WYMD=Mj&*SHbUn>~? zk~}4rHqzM)ttV`RK8^2XL19^RwId`g%BL1yxCY9C&JGPrwy&KL~P;Hp1HT1RdV6q=9Kj3u~wZ+DQPlFDwI6hhItsvI?)wc^~} zh&JdaBN~a{U%?OYdtW&dTS{j?ko|EslPD~YykaGLaPj|OF;?6BR3oybzIkX&EK%(W zc+3xpEbynPggD*X=ar9yl&rd)U1AP|twx6|M;f+1T@9vr<}26tp=gUPZaL1Oged0C z&yH`G6gjsrg1sA7(cOj@tvk;853slkhwi{p&^h8rsfo_1@SEXserZe=yY}b;9ZWX) z+YR5q>1kDf;&^xy-u!~&r+qgsryv@DW(^E`Yj{_vXHEP##Bx|kgd|dd=>Ih z;nFVY7|_2hcd8rK!F<

YTo}93a)i5GIIDi}L0}H#l*UNZjlu_c2nhx8$DE{$jpBg^%13rn$6a1Dw#$d3FG*$YL}p@O zXK^Ada#pR-<(Cy!>9g$S${C?T7y=Ly*tF~CY@E?Tw_(PwC%rnA6DGeely9iH&VyEy zKnZP4wt8|Z8GuCvWlSZOQFY=I;k=)(E`47t|9w-txzbyQtrs<|Jwl^HOd09PI>3N* zn7!z98kMK?uY}tKwUIzi%iprS?1QfM@xEiG>2k*1U1GWv^ka(WLDic2tk-k0M7=tC z{eyb-t;_?UmBdH&eT z3|WeH)9r^2@75I^7zOF~Z`7ZIW(_u70Z$&orKj78F7dx> z14O+E@ z^d8~SDKe-XLSLcSC)SQk16XnAn#V-^sV;6u_99x+l+-_JuwGMDi^u1Hz~M!=U2`}) znS6G~-;Jizcdi_*eNT~edX?cPy0+J*t;TldzdB1TNny$ibIk7&TBQ8)$Db}e6kFB7UBtvf0ZohJ@}p! z(Ejj62xCf+P>eKcumQs>um{stXMP?a2}kZ z9iubpiz_#FcYraOs1tl<@rZ^DJo6K=0pnFH%Lg$tI~ro$BH6VVMKK(Uqm96d zcwKtTt>lZUo^*x+sXU1hS4E?^2f)M^v^e~w+x6Uh?Cj5m=i#=Vq3oO3O@5f!$dSm; zuWdi!p%~LYa*s>a6kHm}fqqh+w0q9e-TXF_Gg7o%f^v+5!2XPWc6?-4r@B^pq6Q)o zS9X&)%tZ*d{82B|^Bm%F(am>;D`bX8IMi?kLRC*0uep8D}(Vy&+F z-^=&Hr4WBv-jbt^mx zgMLWk4E{Xx^C|3Jx;b1dh`d1ol0^20o>ZK}nU}WEJqDf=o{v4yUdfU?I47&PpU{B+ zK-$-1H>KV6)W@ytRoVJvmNar073s)hUF8_HirrsZxlcNMbQ$)~3cT-jTGZmu_?YwF@-hj9v>Z0n=<2NTAC(R+@5cb_{{~_*+ zU|5WEF!XVW&(A5u1GGo(6i3a3U*|LfcZ4%ed{@U9ov6jnxM&W|K=NyYfUgrU`a&Yr~m2rg?8y&b%~ zJpCAmVB+)Gwznib`8GV3V42F>dr4iY{9_l5vHG{?`Q0bwr<7q63#XU5lx{GU*sWJT zIg?c`>AOz!Q%@U3E=NM&#)r?hF&>ys3S3On1Fw3Uct?S^fROEY_G&On{{7j*@`hW) z^ou|m3U5&eKhd6=F&(}?Z{LLvCe2I9^$+jil*l1nJui$CwQ*}-VZIvCR2^es=qC4g z8_Hl&Nij2WAP>%f?b`Y)X?5|#6JM+>Yuo@&qg7~j+bYmn154kOs3$eu`*NY%Bg&Ui zv%AZN%x?>0&&49w!ke`HMmYY`daR+t++xzE&VeL_Kz_U0{83G1ozA}Mn(z7AARkmP z;=FMje?!L7P5D?kQ~kXxRFgEbNTEG+$ceNg54~0T`Sb^=j28_mCwA&z0R&3DL9ie` zc;$)9R_TJgc)PeH4_8H2b70i#;6rG&R|HlXQw;;N!s?oN>}Mdn0y=cz$5*G6 zVFU4vTCII@xS18W@8%3HYoIyA`#Y(D#2esRshF@hrEhAneROm3q-Eq+Y({j<4nxp^8<$cbC1t zUn_c@87qT_V<$&8)lpI&l>Sj?hKM!M%_FpLASsu!3C|m_gR#nI9dEOPEX|Eb_K3>X ztjU@dvClCwi#+9EvM08>k{wsgUfKI*Cco*g9;gTYX*;!CN7Mu|%KyI4E`Kg~fO`fu zINRTAa=qa;p#YR6sng@y(=x9B{MSUEVfv;SVo*XB0aBD3?56UKBl1t6_3J-H#TTAq z-W@pW&pAyRzC)%>P^vOz$!1$8VXnq-g4<=0?+|Ft@?pp;7Trm;M zn0GhrLVvW!?YTzD;C?<`@@tDncv?ZG=}Ks}`pXE_gAJg1h| zSgej%oyhBZf@Z$~5QM2mQ3*YNRFS^0vSsoi4%hhCknPYb>?6#+Um0SsnD+^);_}2_ zkeX3vd=VDOlP@shSW=7D&rgODO5-|f33o73vB$rDl1+D%MGA7ITB|uE&)6(rB~I+{7%6-tH-1?<7J0$KMJc7e1-NuRim5fb$ITw}zW> zSVvYGV1W%p@wfodBx4Uo`W38EHM>X%zFR2rT`7H>CvI1V*sasUi_J%gFC90B;bhlR zX^3St%1|rF2S4ZUO+!i?EJ8@uNXrYId;>uVs1~F!U~Q07&fFJ=8GK?R(hT9dTpAGe z69Ur6VAp=0;AXMTcC!!?`BK`L`ZrXMF1%w;W^%zbOKdjG+y3pnWIpz)mk25|>{4!a zzDkrCK-we!QF$tuDR7BQJ@F3ZJG^O52x!B#gT^dN@iOG4(qzWD+v|491lx`azcr)RNRG zdu;vYmY)21@VKLCEW{C_@YU;$jgtWYvZRicZtV@>?r*H0hA#5n7HvsizX=sAKT;m) zb}8^IJ;rla(AgBXl*H407w)cEj`d51K*7$J*dA8c0IYJm_!WjK7OngcAgjILH^s+7 zvzXxLb=oReY96M3VHTXF2K^z9xa!3U)yn}DYj*J}0SFYmC8o^y6b@eVZOZ8Q#$s{m z5(z}VGs{&eb2c`fYWxbt96{oHJ-SifL}ep~ASqX-Nx3xd#-Y;;W5syy<>;Pu5bhnu zyK`h2_KmK+*HrXzP_MxJvII`f%g@&DasNcmYR9_Mj%H2mADEFGV*r-bRdNJ`i_kH1 zS?ifAT(HEwrJ&R(4Yl!}I84A=QIdpZAoob}6$b4Z$-?+%51K61^8JN8Ho`=D=$5vD ze?k^R2SByc#b;vQuX{r#w_9Jvi+-M>!#S`+BLpJJd`@bmXCIa4h%Wx13$^9f-fO{q zIMgvTh~NCK4y>yVb%sh3Q)(%ay^L4UO)w@Hb7VE?DVYl?NHI}2G z^VW{bJp5VJw_b0 z*(rW?%&qG(WycA{p>spWo2s)qZ)1J2+Mqqy_im=B<;zAEXw(NJ;QclbXRr`$c#Q5F1+ZLYQ2g^-gN_!T6 z%%PcNiF#lYY>c)iW$zU73yCQhU;Nb)`5P9IQlHN39UpYs4b{5F+lyE8%iQ`@82v^% z_*tJE3s>9>p%GGqZ%A-UbG#|amto!8F)!@$gq#lBQ$mF|H6K~QJkTDsy+p9}cIIaz zqqsk{H-WV(ixjC{9%!-TE$awi>+3Bh7yru0HJRAH@g&nM-Gmpi27=uyOKAYVaIJnO zc*Un!SN4`B9%;MO_end@+&Jt+nCG|vPeCG%nR<@88r;`Hy{TOhh`0KF(o-l~V?alR zMxPzS#{RdL_}mX$^}wl@gzCb^amf~-<9sOsiReZbFt^6uVed6wm#A+bhB4)^ag45{ zJM93cOaPJkZ&#_9@bWJWlsJ#53=_TO*EcF3pU1x98p^A=g_g~%X1Xzwwh-3_xxuWn@H7)hr}*dEhR-Vom$l5U>hVQEF?}THmQr(t{B)*T-wE*3vm+im zx7A0mBq%Kl*)U7*1;yC@`TPnf$hv-S3Ha;=f5`f@m-7 zdCp_S*c(=+KqN48%qH>;+VQ1WVStRKyfQZP?UzJnd1Yqj%_T?kj`h35NA#YBUZxVe z_vyoa@Zw2Pd|T9=1MAFtMo>Wkx zn--IzKNmd9EPvB36cYZCfXPO_e=kuTc{%$<=YWjmpx*ccpcX7%zwRE0{B7F#5F!wF z=IP&(U1zPV9wA^Czfx?=6ydJ2%1kJE5$ADw&(DMUXlj&gfEl$-{_d^O$@|x94!DwhA_nXqGLIX^qiT1ydN%i5{NuM zS93Q@J$tPb8teHy+#5Q1M^c||b4Z7tpCV_Q0VOlHl!C}s7V2QDJ#Th4C%L=0f z{Y#xdJe{xhw7=!gMvdVAOhiy}e)zN;TjKU?48PPAIZhMZ>A4xU+KG0J(&@tanK<_{ z>vH{Bga8s=sbnXi$~~=ndu<%L7aMg4G>VX>#^V!bquZ(_J$v)y?8zdSxYE^Vx|#f( zyOJzDO4=oT^JUYNZ^$T=jO+Rvmf;n!v)i7S^4E1AE!F7lkU7L zLPZDm=U)QL!C6(fe(g`Gp)Af4704kq%Jxuv3$@LFsJZk;1COU{xu)2(5oZCb49cem zFB27ZB0LEUU&AF!@r|#A(Hy01WK$YZ9b)<#1|N{Yyz@z=)0_cwG>pshjLxEwE_c5# zg9|qB7C?PIAW7)hU%5V7jkTe1lT>a-SER1qJ+k$&sajte7#NBu0j+?1m}6Y{N1o1v zy&q2w!X6~Xc&0HEkB<0*s!*_*pUGx_%gWpUl*hheU9SAjmD+iI=0NP}MPqzNZXEkx z?Y(7CoI%$p_{7+$lxiq%Lhty7t5ck0igG2usb41cap4;NyK9w{@8t4b4hfE^ zTZRp2Eo}+!D-Xn}kBtb?3bjZ{P73?r;tr7>=$sx;iB%nLFk)#O-QZ97OW(}u#N5z% zTi}o269i_V^{f8dd|$#z(Ozl+8LU;g1RX2}gcr@KU!;$n3fMYu9`j{T(K>Dah&8^wR_R zNc61vu56llEd4&8ZXBTg>H7~>UKu^TP}2HQxn>xKz0et4bcExI-^R3p>rC<>i%u~h zW_Sip@TgaZy*5c@_=qffBI|~w_glN|){8kxv_VYFB?`a25K%)Q_h7V z&55Pz1#v}w%?pBgOpH$r&wT`ye{@L31k%~op5i2YhV`>*(?%V&%^-*^nnN#Y0&g%j z43Iu)^7&etm!fU+KdeOMd506(hFSs%-4T^6-xzcUJ9W=vNL}usUOZ)+@193Bt9M5v9+-w@)PSGHI zgPL_hd`1zERkn9W=FLHduc13D_T>Zj8}UH)bF5MYOPF%NU#>w+WMgRZg*3BQ5`8|S z9e%OXA?<6J#q*|A2Rl>wTMCD~$SsMuQ{#F(U4{dOj~Izltz`%cHd{`xo6IROaY2FC zKm8#aV@KUAsY;7%Ofse$%t7i9-k2I`b}U7VG4qjM9g$^*a%@fEfY2iDMadVr$S4Y5q6>lp6)JnL7dA(xzdmAg z?u7MfI8|Z2cbE$4D`if)A*j5A&iB+dhI`%k-~?mIXMDXrdY-f8AJq$$*HP!ZEBS#q z)5_&Pkdyzm=qx2m_;DX}Z9<%g(yW5#a0`@9e)`k=?>(VSC`og+*SmOXZS!UXN>5RJ zGT1_eE%f+OU)1Cn-9?H25p(-+G5UqOPUz7;^o=-x`d&lX?7n?e8z*KqzG>1wH#s`Y zG1lWD4)1xj{q9cn)B8X=Cihyi(@+ z`9zrID0Z9@U!#59lY$bb61PH11t0MZXi+ej)BGa*7(kTMNAWg(h%%50pA1xQ@4^tb z2(&05^vDgExJ+QtJ?^O=#|9QWOWW!n(tiq3x1W`r_W;E;RI82j+yw{uG%HMKlSbKN z#S1uJ_V}48$rH#~(rJ@39CfULk$${u)?V|1;WQX8s)sd><@d1d107(i5lSL;`-29b z{o?BTUa)P+@;xZsC zr;f4-oOVn6vHRK!Y7R%=YVl@@zxY7$o5ddb;!AB}!a4nvZvDj(UmHZ%EogwMg5|NP$ z3)#0UOVZDhS7aryvqd*?<4~j4nV8ejiTvd+Av938C{c3oP+9qrQgBbEngG_6m?`Mo z%qzo9&PYesYtmyx76K{i0H`qpE1076GzR3 z$BCspM`xls>9iz+zmj3@dsRMANwehdn;yHt!^amG=ox0aoIn05CDMhi(D6k$kI8dS zb&bn%u0JLm*MZTGR={SSIS4B4?>k&|qTAcc6rB%_h^HIn^OFkLN|8#Ooz3$4mrp*a z?=P0oMg?ED=g4@s0@}^Eco>g?lJgGSw}gB+5jpG%WHEUkI}fmEaeKDJDJUT>s;VK+ zm*q%#$}+9kf;9kBdAB9}yCc~H4okHlA4OS8Cn82B)YlK-SDJLxd&kyHhdH^w z;YQo(?Np#m_S|z(x~H-2-9)h074Ob4@^L{{7L5&0E%QFWq_au`-%x4^j6WNT#SWCV zXspYpj2YwS>2Tn$CXz%lw1P4}qM#9Mzn_Ep?e(6k0k=35vDWiSmIC7@%A-A+6^!)0 zkz?28OMVzj4#Y$c^R*c^>-d}TjU5z6X{A<0f*Ha^$t%PiNlb~Ml*V22bF{zsA{0)` zEDGMJ!Cc+;aP(fDyc+H6xNwcz$<^Pb`)Ifgf->!-iW94X-#vu*1jjq}cB=Fp_3MOF>w_g4nI(dLCD^4nX$e~Hv<7?v;5xhR{;TODmo1Qm8zVb4f zGQx=`j#aF8w%A+(zku~aqab&B2ys72Iid^)akw6+`$tFQ7ZnBNuhcOz1 zpcU%@N-3gppt+(hUmzn~ck;#JV~+1Ko#rCchEuGJ1bk#1$E?DHaIACq8%6or=qW+?uU{rI0;^72{~i zpsf&_VMDq%fnKdd!BhXjDKS(PPh)IK(+TfRep*r2vl_{ftzyQ-BCOZh=4dAErd~wP zEwFjL=om=6qhD29J1$&Dx-mC2_ek207F=(0Qx@2LXXzHr=~Z0bQO3k^vlV(q(fqz! zTb?!-P&3DQSaBlNd=rmYP@s(3at?3*hM%bi3Oc+McUF^aiCpQ8bnQ}JJi^N^*!ck) zlSVa1b(6LVhnHDAKZfl%NVwAZxYdUpwOe!OQ}=(GWgShB8HZxKhBhldGQ=FGfkrE? z;K|pw@1^QU$$1fvMYM`5zcNfAm9iYhWL8!96v9IwLQO7Kx4Dkfpi*bt4Jdu!n8C$| z2B|CbpLFjL>BIgk>eQ->lxPSJFiHgR(q*4XCj3S+sA>M54fp2|D2;TfWTP0cD=B;m3`NXp-@ z&IdFp+qRU+@5~m>N2BF8w)ZH0!aossQ%4>rkwm0Cd%z1d4M48G(Jn=%eSSp1 zyb}^Ey!e}eqpT$aaW`dGrGoL|QuYC9TtSeOvDq8;NA)@zNjKpT&kIW2!g=#|yNPaR zu?nswwUKgE!v^{3cxy6oq#yRQalD@Br#la7CU#eXUnwO#K-tT@k5Gdc#{y!6W6*;a zA1gkEbV&KPUjpL{v#2AXwwj*l5u&Mr4ROQy8lz7lA3Sw;RN7QQd*J!6V{-;$9XO+I zA&@hq0+A(`QHYzq23#s!aGxkCM(#v{d87;`YsR}VlCmgN{E5g9U$Va`R((;2$lw;^ z19O{<{zKFToJU;VbxF5^hPpYjOD(@~2gj2nBjpi2u%GD-nbVT%o~{@TgJv>EmBv$| z>N5!iZ;%Nzk=>@K08o)pz#H5tjuZ9P$oyV4b{iMkPhX@v%KT#FcPRvTDK_M^JE*i7 zv=1yr3-?i~ts`AeJznAhjRVG$zlQE_ zWi^bWb~v7YdCcZ?_(1P@MpMdGIsfZn(swqzg^QoKH`BZ$3U^{y9T|z=O{6CM5yC(M zn?U@$Tkv+pJS@5=my&!+C@urprp|g7!oGQ9{W5CPCYpaUO`KgoUiVsd#WlgG+wzM% zg=*oBfT|P85Oh0n*&FuQdB5?MYEcJiOgGpzli-!$=}-HUjT^Ai*_WS2roL^d(;Ex& z##p|wrmlV{i^`^tL$k=7C7-nG>i7tqe4>@2S$2q_qu-4D|gP#q21BvoN2z=nZvBv zkA-uI-FUidU?10sQ(aW|E<59KQDD>TD?HtQkz=09>(_YI_|r*1HxomgFXjhAxHuNO z1<_Q{MP~!FumyPEY722BdU{F~qOIW8SsGKIs3us5t7)Q`-+G#(vLt}Kae*Q6-kz6n zwIjxv9@i`@HIw?#i+}yu5$qz!2$|*QR6cm=TA=Wb?xd`iCc)jAAn(a^9-3NZ8vU&g zc>iI10X5H>m-aGtn9d>U7=8w+dsqwASGTt|Xijsp!A<=Afu9aB)tmr8u3(X&@F^^@!c2*2Cr#xi`(K{% zy9uex7Aykj+xA3N4JDUY)s3T>vChVK3c-alhaaVKnC@hy7{!SuKTve82(;H0K&=9A zFi)OlI~W?$PoseDuTebnj;%h_{qxA_W%z^V?`@-=K)B@G=(B5<93b}LZCaT~>%Rnt zZyxH($el|{Ja|n{DDpfFLtUYkc1){+q~X_IUNEbvd0PXID*efNmuO1T(0dF3pp0Wd zNC9==*CevsfAf{u02lhg7MY-5@H(Q2ev#A?_2O--oFtD&_{3OAqpfZ3n&?w&7vze; zgQHvXe4&EN7ZaPv5A#HEQFQue=|bGuW8la)`h)4>J4d2WVKY&zh+jXMGTl&46e+i; z8Pv6t^=`Os$vN{Xsqa2!|dZWZ`OL&jEl8o8Y zefB@HDJ!zYp3C&93TE*Bfs z5jpr~NCrg+mgpPA9@C#%-Ou7w%Cjo14B$;MJ!qDeOzK}R3NAT!JoERmIcc)~$0eOO ztoan7iuq}G(>KnCMwr(;DdH$S=LUaq zk>od@c4dH-0;PAjuRw~{*)KoZ+ne~loci9(Tv*0f4*A*eKqF7V<;~d36SG{`iq}yR z&=`at)&YY7oJEzg8#eSRoFEzTG3_!%Q&&7K1D?LYkFvw zVIP>~^RfeIf3Z+cGMJsbW^U&Sf64?CKp-m<5`-gyynVYXuby)hG?Of}kIiM_vYEhNSkAhD|(({ne zbH=r|FB7aAst$z1b`1a#`J#<@Z_t8ocbhxE@+3RrOd;xJ`YdUq@?SpOmZ-#^wce0e zB^ST`+e5z^t%kndHyQ$bV0Qb(v0Sd`2@QA!FqxQuhsh=L+Vt<)HiWf|eirje(*Q!%YD*sk0E_7f4eA{Kq}2|8^dgYxenfx!=ePBGNbC}VZ4gZyW1t7lx} z2%&Zw`Qz|28B19ABUT?`J%&k%Y_m%!*Y{)Fmmbz(*~>8GsSVN_7yeICh@ZUi!^QkN z8wcp%9{qY29zq02*8jmA&mQ%*(d*d#T(R^e((Zu_DAZLR7I>6!&$sDkNfGxql8XjE*2c zdo>uthHrv2L&k7SD>fyTzQc zr`naI4Zi4ml%DZd7PGjZ^ScI8RiTfaq~ciJ=o>R2WQICN-F2jRqMqmb$Cwut$KQpW zw1xh9K1axViA*3>c&MUsm?KQPZcg$315;1e<4y);OzY8;pqfK|LQTCxAQUF{-N7^g zqcxA!p2vkwagAv5yyA=zuc6b8`Jk0HVu+A82riDfL+y-b83i-j#V#(U3*DGv?tfo_ zb!;gMbHgP4r^U?o@#$EjqOk(1jd@1+5om~{t)aCHX#lg(RL|V_F6J0H^ziVu2tkNw zJIt2=y%?$ILx$qPo%ftn%tgxEuluC zo{ay#Q`q|5wI$K#Ni(j4e-XJJbS~Q;c7!9$Es?^=2BLQ)`ieph)CyYDB4{anlbF24 zL(dVNuP(8M8hC1^{{+I}ryuG^lqZ|Ia|p7zYH=5Q%M(qb0|(4v89va2bW!HVh-N}H zZ)sMYqgBE49s%Ur;nSpOR4;-jG^J!xWb&D?cjzUSmsAcPf5FKod@`5rzPW#VI4zU^ zzUu(|+;h(zw(*6PoC&XkA2KJBebE+UY+-MR0>6MXMr>Iw^2P}g;pP2$P7KU+CbJ5^ zSZh*gqf?spk15_YI3XTWY_*Fovm0#x==b#9N9Va{a^iH6EszUG{KKg33DgnT?R+Y@ z1kbo%R-O(k9k1`nBBI*o8Q9DSo3x(!u0v{$th{!OZ{AciRegTm?B=D`aNEZ+OwFaB zN|s&?A}q*dy?e=~QZjG`gfe+Q;c2~fSB2UWX*Q50PCO3)6H8!L8;>nIMkQb%>|q?t zO&xA(x2Wj8pO3p3&X~*RqUtq;98#Vn;3CdCNNMT_ll4dc$p8D2|0Q@mM&`nBc&7Y~ zI}OLz&xFqSHmLXzUHwOJl=K1^O)wW?haEm00@ zCPQKDN3}tv*BCvcPTHP~&DMVK-ZH1@);G2PLQYqh$>JD?nhJ1BTa%ArQ%~}o#bhxx z#xt*IHc#Z^3%*(G-PIUt5d$Hu{S$=PzwSWC1a@WeI~DYYIqi{dvJ~&fZh0gkkCXKv z>2RzHF;}VIf0Ujtbuk4SBrXMqGn4j(Xv)l9SmviwGESMm@ox8S#VlXoM6q=q=VXS+Z zqP5Z>!66uuZ-W#0pO!uhdpv;AXk&Nu1`^1>Iu^PZ#{X1T^)W{P$ww2}p06Hlc*$G2_GpUHwdQ>(8kKSgjy&zyRk232(fRijX4JD3^E{m}GZQqlea>WOhByn9}ooi@2)IiWko0uxWD>9=4U{^l#0jw#yA9VQ+AG;8vf^;(EOUEDszj$Q8 z-$RB+6Z7}AnjPr$7v&N{D2~D1{x(iHj1Zzt%xU}$yadP-A#!Znm#yl8<)NCoVArP` zzrFavIB-$*e?$YW?*k1tU+c|m2Bnu3x%}B9YMQ`ua%Iy56OmVDaD169WMV6EyT2;y ze*EEjJWN{Yf}?Z)MK&zFb5(`wdA9t{k7VGO_|Y>Gpw{uiui!7XU}Sx`#GOc zPnA(CSlQ;fvmjrW^T(1xP8O*e)pn|ZjSLuDA?ukD@3X+jF~;_APWFKA4%h<+s8dV~ z=3Y}Zi|IS=Mis8Lf1)kNX)S}7{w4~$7Se#K>0V0 zW4isafblhVq{NI`E%EPlD7J>FZ*q7g7)Y~f=*E5G_67#+VgP2BT`F8xuWIN?Xaa$9 zZgos<^RpImlo=7YwQ&finF1qF0S54Ep8TzpW)DVL!2mzalVJut0XZN)Bzpq-DiZ`o z;FB~Pd>W*dffIodw2|{tKX4djfrG4rxnNxs1_n}uq?rzmfd@nW)x7YRg~JfQN?2Pg z6hj6CLxyD?EMv>WLBNZytT#OLuu6JB^CC+Y4%fwyJ(L?dN91!NGw(JI;>?U7S(pG0 z9S$}ZOm+*Lc_E|CbN&p*i8T9{lBo?ZLDkj(nm2Hr;{7ZUD5Chz$xPc}KUc&9WPaTR zE;65b6mEnIbP(2eY|%z(0qt9a5cR13CbBRtgu_3I#YBBKY+pxE=EYB;vVEBhNb5Es zt$D|c&+r6ra0UsRJUaSv!V5IsWHP|!=RQl}gT2p}WaApCXijo5k$<5JUe1G8aKOYj z4iFUy*srGr1_F?Pf`}hs0}Y@a-~`|e%>ORMLVl;CKr*xJ+mb_CdkAq5p!K~s6X4c5 zLKuSg+n#eWxV3-~3&CW)=bj7(06f=6K-yDzKrXFz-53V2NWlQy;)UyKNb408F#9w{ zRZa=8`XedF?SAS4nHL~h%iPBfo*`Vr(IH)xdNBHXW>QkBEIB}L zCI$u;{sG>sWt=|%0GuZV9PAQiCjX(*x@QI8PZ*=BM{gM|QsCkY%A@kMA8j%?59vKS zfH>m>ts#0ait2u_oc#rUHhQp}^*@8c$`g$DgCFq$-)l&%LgC|KFaS$Y1r}%MTGe)@ z<#Ex(XMxrlY2=K)U?druMK~IK83RbY1N`BoV|?j)ujEdX$O;|2WiKwozYPq@XX`DK zf@)O+fX2(SjD;h-u{YDKzP^0Um>|GeKu!Kp2SMZ<1>&*|vv~QVF`ZpKF>v0&1*{2{ zkVpxoQ++GQB8t(2EDC)94)QKd>#{z%`xcg&jwAviSvI?!g_u0nJ?7(w=TaB?=x-Mc zx**NwjpyTsS{4DLZjS(eRx)5-;4UJZt+_VM{bt2lSkM?7$&WMcR6GyK*RKwCznRmr z7Br49iIMgXq53kz;d^tvol)hn7wgd+-BnBwrAMAws(Ufn^D02;jy;+CyT{&9$m#e1 z{uU_?P?mZ(W@M<}^{w>!8Y9!Zz)MBReaP9+U~^QY`F1X@Cdpz1%o^Z8w0$u;W}jLr zUsic#$X1fV?%UX#$zD|Ux~lRZWz6po8>xU_yv*V_|Hti-w~kQ2Cp4pUp$X{?H&MU! zyI9JM^*r7VF$sZMcgZzQ7KU6IQk@@*5fQ1)(`b9_ZMEokiAlwD6%YI-2TV@yBD`+R z8M9X|xzzNs_1^uG9Y5?*Nlr{}aGTFK8c46r-kF6P45YqSz(vk*)2f`{j0kj|B7#ZBfVo974P*f^S@u&2lR-Utd2l6*RrP3ROu5G z5VarEwWYRJGK}U%phGNVAb2k${cMjq8=d>~JsvM4jm%S}v;DSe$69Wbkauxd%yZORQD|K>%e_R<_CJTDZ8TIY%wLT2*oD@*8fD- z36$Bc-p1+I+q1-Yc#Q0t-gVAQ7k4*Gg0`TavI$t_zY}H`Z?H={QkSJti9N%Z)*z_2MYsm4#k=+N5U>n*?J>Deo`BJzwUn>hOrB)?>0_(bgc z!nY%ehNIOD8_u(~)y|A!uT!oolebEZdC~f{T^%IwA_tp50Z$@0HC6Q+PH!zI)7S4G z3_tlb+Z88tLGgc#J!4<{^Od_r^MGQn#waaZopMbo^hd7wQzpTGiw)J>aw!ua`YYw` zYD~@morAmcr5Uy426C~G_xw}tQTH4azL~SS<_orjB0XaPer}h1A+)}Je*9+p@FwMv%kuxghMvg!gC`2fr{b{iA}%CPjbnP_i$#*+{@@7 zF65vmmK?)!H5yIH$CtP0RsUo0vL)sut+{eDqH!=vg`F)YpltBPEku@E&Dr#fSag4P zq$Ss9oomGx{9M;m<+$Y(-?04B{tss?+C18K=L>nCp$p;Ob%yV#f!WQ1%bY9d#`j-J zxXpdMA2xDP6aLRU=eIM_*2sAqSvXe2x*){-%UTTQpTn=}lH!2vm}QoB?qXQ_I@#Q( z2yvn(89hrxTh}#r99Mq_Mu%YdRdiqP*H?=@RI+;cU4c)@s~KGkBo~|F0950PRa5H+ zi3J5oz4JdG9cf=&O_olowY_s>xZmV?_LJ2*XjivH#3|euS!%M@k0?cBN@#R5V7pLZ zUwd&|`cPohKibbktMMU5w3y5MgundOZb?DbCr2W#sXd&UyCN4N_}+|XJsIeoCF(*V zc)6lww{O1zR{phn97ws)z#>^TbINBXN}P>9}_o?UJu^&{;NxMw|_~CcOn9$L?JElzDShhJQec( zXTGw)&{n2^7~QXeuf8t(Lbt0Y#(B9{@Lgbfs{fzpp53#@Xl3)2dFNe06V#>`x!50H zdqqf2Z3d~lSby&P_Ms1Fx5w$#bgYfR$HnpCnn~%ec5p2yH^fYR1g9Wl+wau9LywUw zUG?d0T7T1p2yPsF(*Ps5fgD6MY-3$hjjBO=T#uA(Z&L+KZU}Nl_)#h)=GmLC&-R!Z z#-fl;icRjE;I_FadUQnFmKM!iHRMxuzf(Qn18UU@(JBdj-&}G=m2wQ<^E=?Zqv>n9 z*z-HVzNWb>Pd02A|F$zbRjZ+a6>z#RoxJnc<4$!@b>)7p-^g&3ajzrJgJZbEUWeZ0 zXhy~RiHZsEId9JU%)yQcom|($|A1wAshyvqA+ekubryYSp^2*NmyZmod>aznK;nIrgT9(H}<^ zqiy9N<=4F^6)I3C3axQpJMu#5fm-LknrJN~%-nr^hx5vnySY$0Al~yW{$Pi($$o|1 zuAY9^_7-Eoq7G0yB)lqg*GVk-V!h&$ip!b~F{VMs_tHPWf4=bX_#s!U#sw+Bb)q&E zI_rkaS;AB0J*>Qg@-r0?B1iVoH^3nLA2RZ2ZLSv%A{&4$a-bCuu0;@Ty@xc^- zY1Ho`U=go*bL(^Edbdh&T>pgAYY!t`H7laHUhbX}qMyUH*O+I6DZY{hp~9fnhrT1H z8zE=|JPt00!S1jT{a02c_WNViZ!M&6DOFBH=5APySbUdxPO*pe4`K za4Rz7^gB+IyO-Llp;m@hf1-9W8r*NlsY(?&9l^*M4w5;yz+df*^slQ-%^sFrVIxai ztv~t={;PO6$nJ=lFgEe0vRZOSC%ubUQqfdBepL-{y3P|b!CxKtkc`dnRz+#`0rF2O zclrB?LkxsRlT!94-PX#C9UTu&bg=tzWDU*n2s~Pv$>{zHES{;k=Wt&?2@VBSPFY_4 zzD&^3dva}DSql;|>aQ0(xj^}SDEsOn_ucI<*2BZ+a93mHIp73c`0*0r>L>OPV)=c{ zH?hVc$+|Zhl$z!+x zeq6t=IG`g(!C_Y!>uvgMsfm>PXQ6usntf#~9vqNY?M{6k5)EhTjlEgE zHD}PB+4Yp~^vkHQ+AlaIJ6VTrPG(0o!NT)&TwL#u8NJkGe}=?ap1f<0+Vb4ZEEF_t zzWdYb)Fpc5w(rlm{uUi#6S<4O87VFFzS|ofcw1k;9@f?pJ2S?9(XeG_yIGn(df*u# zhmHyxdm#g1hOg1b(0sdF!@|q0`flpxc@clv9wAGt1j~QFqUKO_{J-2dG<)8p_Lj;E zxSxL+9YmkOmC+Kc-Q3Arll7LSYkq&LHZQzVi^(~%!u-KuL{|I2bj4@}}}+fn&0 zH>Sm_$0$+3>Rr#y|44t1-FVDY^H1gWi+A=MOENjA=k>vgZCO-CcM#z8>LBKpgz@h) zc~6b>Tr_hZ{JR{o_i8@|jAic2ds5ffCG^f~*3_Ub#YpQ3lj5DRyW?Tp{# zk13F5{4r}(<4X+rpEQ@~yrll9SNaJqXn@aDOO9#udTYB^7GVj0VZ|MIKZrJ&+ww&O zYch+ai&t>zLkan0PydBWN`D0j&E>rsAf;?!OCKvuWlSPCLY~{N_Eh{?J!F>L2p#I& zHfOlm_9It*?QW?((mKDpUi&G$@im=WrL09?xiA2Uj#EQnMlH*~`Tfb(7a$Z~%WC7| zTVj7M2MYL$lR~!6Tczh3GD3v}MsCU+vqRT~z6bcDk3UzdtXk{u>!*6|EjSu;PX(?WWBn;(%XA&wdH(5`t#*NnlCk6O0O=Hb*A2sL|z33P&5DIR<-LF}a};sV9|6;n7X zp#ZQg5`~mcx5Xxu1&}(k{kqu$KZ2p?d@UM+^thjSQx#1H=yw=n$alThV34~9Qb4FW zR1?RWgOG0lo7KNo?7Uj#q3Gn-{^gv*BW4&--WhUAE5||%Y+xbDxQfpyTovreInkd- zt=Qw}z(Da57x-_RcdRoL)jf#L=4kDH(P8Ri?eF@;c*RQ`7iGh)K#K+Ec?+2Fz+2xGQz_gh*nA+QE`aZYXLT32nYB-)BnW(V5RJ}Q**}! z7=Zih0=fTrmw?7EnJ__ikAH;p9)EsrZk}5;00#qhHbsxzKxky*_sGH(?eRa^fao!R z6`Qh$TE;*y0K|Ss{jbXmtb5|-(PNRDWPmJ?$@5jv{3|XJ4s`c>y)nZ>Q>wnFgoGLt zV1MRB&uqoBuQF>uLm-C?N&c1kUn2n$m%VupUUdH^6JT^X`r;vo?%%co%pyMA4K#o}L4Xqw znARL-Ob9S`Nh6Tpv=&c4Gz~&2Wu!r4t@UpQQ$TJFl3wji3WU6SenHv%1E_p=3X;=o zY!LRtLjy|NgiCVoKQ)CQyGICV%zI{hhSvf5ql=)h$%8A}6I(bB9Ueg-z6rU1=sqXX zq2eM4Np9|1)`4c${3HxyeSg^{M{XwMhK&f*C-qmHH3X*2FQE{2b06*wQ8ds^7{tQd zkFcWxmxsC@gPv(TB!lRj3{g%0FEdmAzwXx@Z152bKS}%*Z9s7-dZfcoG#+JR9bdo&Z3Z zgl+U>}{=5Bm`|kz-{{xw6jk^GT F005X+P$vKY diff --git a/test/integration/web_and_worker_test.exs b/test/integration/web_and_worker_test.exs index f3ed97af504..cb3eeb08d2a 100644 --- a/test/integration/web_and_worker_test.exs +++ b/test/integration/web_and_worker_test.exs @@ -406,7 +406,11 @@ defmodule Lightning.WebAndWorkerTest do end describe "webhook with delayed response (after_completion)" do - setup [:register_and_log_in_superuser, :stub_rate_limiter_ok] + setup [ + :register_and_log_in_superuser, + :stub_rate_limiter_ok, + :seed_default_adaptor + ] @tag :integration @tag timeout: 120_000 @@ -1014,6 +1018,15 @@ defmodule Lightning.WebAndWorkerTest do end end + defp seed_default_adaptor(_context) do + Lightning.AdaptorTestHelpers.seed_adaptor_package( + "@openfn/language-common", + "3.0.2" + ) + + :ok + end + # The server side (handle_delayed_response/2) can legitimately wait up to # `webhook_response_timeout_ms` (30s by default). Finch's HTTP1 default # receive_timeout is 15s, so the client was aborting requests the server diff --git a/test/integration/workflow_edge_cases_test.exs b/test/integration/workflow_edge_cases_test.exs index 0e43fc43020..66933c136cc 100644 --- a/test/integration/workflow_edge_cases_test.exs +++ b/test/integration/workflow_edge_cases_test.exs @@ -40,7 +40,11 @@ defmodule Lightning.WorkflowEdgeCasesTest do %{uri: uri} end - setup [:register_and_log_in_superuser, :stub_rate_limiter_ok] + setup [ + :register_and_log_in_superuser, + :stub_rate_limiter_ok, + :seed_default_adaptor + ] # --------------------------------------------------------------------------- # Test cases — each one only needs to supply the job body and assertions. @@ -67,7 +71,7 @@ defmodule Lightning.WorkflowEdgeCasesTest do end @tag :integration - @tag timeout: 10_000 + @tag timeout: 20_000 test "job that uses too much memory very quickly is properly killed", %{ uri: uri } do @@ -172,6 +176,15 @@ defmodule Lightning.WorkflowEdgeCasesTest do %{run: run, step: step, work_order: work_order} end + defp seed_default_adaptor(_context) do + Lightning.AdaptorTestHelpers.seed_adaptor_package( + "@openfn/language-common", + "3.0.2" + ) + + :ok + end + defp start_runtime_manager(_context \\ nil) do opts = Application.get_env(:lightning, RuntimeManager) diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index 2100067ce8b..e9f237b8fef 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -332,6 +332,20 @@ defmodule Lightning.AdaptorsTest do end end + describe "refresh/1 with a bare keyword list" do + test "defaults the supervisor when given only opts" do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> {:ok, []} end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + assert {:ok, _counts} = Adaptors.refresh(await: true, timeout: 2_000) + + Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() + end + end + describe "refresh_package/2" do test "delegates to Scheduler.refresh_package via global_scheduler_name/1", %{ sup: sup diff --git a/test/lightning/config/bootstrap_test.exs b/test/lightning/config/bootstrap_test.exs index f259dd9de98..54cad2638e8 100644 --- a/test/lightning/config/bootstrap_test.exs +++ b/test/lightning/config/bootstrap_test.exs @@ -538,9 +538,9 @@ defmodule Lightning.Config.BootstrapTest do end describe "adaptor registry" do - test "raises an exception when LOCAL_ADAPTORS is set to true but OPENFN_ADAPTORS_REPO is not set" do + test "raises when LOCAL_ADAPTORS is set to true but no repo path is set" do assert_raise RuntimeError, - ~r/LOCAL_ADAPTORS is set to true, but OPENFN_ADAPTORS_REPO is not set/, + ~r/ADAPTORS_STRATEGY is set to local, but neither ADAPTORS_LOCAL_REPO nor/, fn -> Dotenvy.source([%{"LOCAL_ADAPTORS" => "true"}]) @@ -548,28 +548,34 @@ defmodule Lightning.Config.BootstrapTest do end end - test "local_adaptors_repos defaults to [] when OPENFN_ADAPTORS_REPO is set but LOCAL_ADAPTORS is not set" do - Dotenvy.source([%{"OPENFN_ADAPTORS_REPO" => "/path"}]) - Bootstrap.configure() + test "LOCAL_ADAPTORS=true with ADAPTORS_LOCAL_REPO boots the local strategy and only warns" do + log = + capture_log(fn -> + Dotenvy.source([ + %{"LOCAL_ADAPTORS" => "true", "ADAPTORS_LOCAL_REPO" => "/path"} + ]) - adaptor_registry = get_env(:lightning, Lightning.AdaptorRegistry) + Bootstrap.configure() + end) - assert adaptor_registry[:local_adaptors_repos] == [] + assert get_env(:lightning, Lightning.Adaptors)[:strategy] == + Lightning.Adaptors.Local + + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == ["/path"] + assert log =~ "LOCAL_ADAPTORS is deprecated" end - test "local_adaptors_repos is a one-element list when both OPENFN_ADAPTORS_REPO and LOCAL_ADAPTORS are set with a single path" do + test "OPENFN_ADAPTORS_REPO with LOCAL_ADAPTORS=true becomes the local strategy's single repo path" do Dotenvy.source([ %{"OPENFN_ADAPTORS_REPO" => "/path", "LOCAL_ADAPTORS" => "true"} ]) Bootstrap.configure() - adaptor_registry = get_env(:lightning, Lightning.AdaptorRegistry) - - assert adaptor_registry[:local_adaptors_repos] == ["/path"] + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == ["/path"] end - test "local_adaptors_repos parses comma-separated OPENFN_ADAPTORS_REPO into an ordered list" do + test "comma-separated OPENFN_ADAPTORS_REPO parses into an ordered list" do Dotenvy.source([ %{ "OPENFN_ADAPTORS_REPO" => "/private/repo,/canonical/adaptors", @@ -579,15 +585,13 @@ defmodule Lightning.Config.BootstrapTest do Bootstrap.configure() - adaptor_registry = get_env(:lightning, Lightning.AdaptorRegistry) - - assert adaptor_registry[:local_adaptors_repos] == [ + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == [ "/private/repo", "/canonical/adaptors" ] end - test "local_adaptors_repos drops empty segments and trims whitespace" do + test "OPENFN_ADAPTORS_REPO drops empty segments and trims whitespace" do Dotenvy.source([ %{ "OPENFN_ADAPTORS_REPO" => " /a , ,/b ", @@ -597,9 +601,10 @@ defmodule Lightning.Config.BootstrapTest do Bootstrap.configure() - adaptor_registry = get_env(:lightning, Lightning.AdaptorRegistry) - - assert adaptor_registry[:local_adaptors_repos] == ["/a", "/b"] + assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == [ + "/a", + "/b" + ] end end @@ -851,20 +856,6 @@ defmodule Lightning.Config.BootstrapTest do assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == ["/new"] end - - test "LOCAL_ADAPTORS=true and OPENFN_ADAPTORS_REPO dual-write both the old registry and the new Local strategy" do - Dotenvy.source([ - %{"LOCAL_ADAPTORS" => "true", "OPENFN_ADAPTORS_REPO" => "/path"} - ]) - - Bootstrap.configure() - - assert get_env(:lightning, Lightning.AdaptorRegistry)[ - :local_adaptors_repos - ] == ["/path"] - - assert get_env(:lightning, Lightning.Adaptors.Local)[:paths] == ["/path"] - end end describe "per_workflow_claim_limit" do diff --git a/test/lightning/install_adaptor_icons_test.exs b/test/lightning/install_adaptor_icons_test.exs deleted file mode 100644 index 254a6dfceb7..00000000000 --- a/test/lightning/install_adaptor_icons_test.exs +++ /dev/null @@ -1,128 +0,0 @@ -defmodule Lightning.InstallAdaptorIconsTest do - use ExUnit.Case, async: false - - import Tesla.Mock - - alias LightningWeb.Router.Helpers, as: Routes - alias Mix.Tasks.Lightning.InstallAdaptorIcons - - @icons_path Application.compile_env(:lightning, :adaptor_icons_path) - |> Path.expand() - @adaptors_tar_url "https://github.com/OpenFn/adaptors/archive/refs/heads/main.tar.gz" - - @http_tar_path Path.expand("../fixtures/adaptors/http.tar.gz", __DIR__) - @dhis2_tar_path Path.expand("../fixtures/adaptors/dhis2.tar.gz", __DIR__) - @http_dhis2_tar_path Path.expand( - "../fixtures/adaptors/http_dhis2.tar.gz", - __DIR__ - ) - setup do - File.mkdir_p(@icons_path) - previous_shell = Mix.shell() - Mix.shell(Mix.Shell.Process) - - on_exit(fn -> - Mix.shell(previous_shell) - File.rm_rf!(@icons_path) - end) - end - - @tag :capture_log - test "generates http adaptor icons correctly" do - mock(fn - %{method: :get, url: @adaptors_tar_url} -> - %Tesla.Env{status: 200, body: File.read!(@http_tar_path)} - end) - - assert File.ls!(@icons_path) == [] - InstallAdaptorIcons.run([]) - - assert_receive {:mix_shell, :info, [msg]} - assert msg =~ "Adaptor icons installed successfully. Manifest saved at: " - - icons = File.ls!(@icons_path) - assert length(icons) == 2 - assert "http-square.png" in icons - assert "adaptor_icons.json" in icons - - assert File.read!(Path.join(@icons_path, "adaptor_icons.json")) == - Jason.encode!(%{ - http: %{ - square: - Routes.static_path( - LightningWeb.Endpoint, - "/images/adaptors/http-square.png" - ) - } - }) - end - - test "generates dhis2 adaptor icons correctly" do - mock(fn - %{method: :get, url: @adaptors_tar_url} -> - %Tesla.Env{status: 200, body: File.read!(@dhis2_tar_path)} - end) - - assert File.ls!(@icons_path) == [] - InstallAdaptorIcons.run([]) - - assert_receive {:mix_shell, :info, [msg]} - assert msg =~ "Adaptor icons installed successfully. Manifest saved at: " - - icons = File.ls!(@icons_path) - assert length(icons) == 2 - assert "dhis2-square.png" in icons - assert "adaptor_icons.json" in icons - - assert File.read!(Path.join(@icons_path, "adaptor_icons.json")) == - Jason.encode!(%{ - dhis2: %{ - square: - Routes.static_path( - LightningWeb.Endpoint, - "/images/adaptors/dhis2-square.png" - ) - } - }) - end - - @tag :capture_log - test "generates both dhis2 and http adaptor icons correctly" do - mock(fn - %{method: :get, url: @adaptors_tar_url} -> - %Tesla.Env{status: 200, body: File.read!(@http_dhis2_tar_path)} - end) - - assert File.ls!(@icons_path) == [] - InstallAdaptorIcons.run([]) - - assert_receive {:mix_shell, :info, [msg]} - assert msg =~ "Adaptor icons installed successfully. Manifest saved at: " - - icons = File.ls!(@icons_path) - assert length(icons) == 3 - assert "dhis2-square.png" in icons - assert "http-square.png" in icons - assert "adaptor_icons.json" in icons - - expected_content = %{ - dhis2: %{ - square: - Routes.static_path( - LightningWeb.Endpoint, - "/images/adaptors/dhis2-square.png" - ) - }, - http: %{ - square: - Routes.static_path( - LightningWeb.Endpoint, - "/images/adaptors/http-square.png" - ) - } - } - - assert File.read!(Path.join(@icons_path, "adaptor_icons.json")) - |> Jason.decode!(keys: :atoms) == expected_content - end -end diff --git a/test/lightning/install_schemas_test.exs b/test/lightning/install_schemas_test.exs deleted file mode 100644 index 81f34a1919c..00000000000 --- a/test/lightning/install_schemas_test.exs +++ /dev/null @@ -1,260 +0,0 @@ -defmodule Lightning.InstallSchemasTest do - use ExUnit.Case, async: false - use Mimic - - import ExUnit.CaptureIO - import ExUnit.CaptureLog - require Logger - - alias Mix.Tasks.Lightning.InstallSchemas - - @registry_url "https://registry.npmjs.org/-/user/openfn/package" - @registry_request_options [recv_timeout: 15_000, pool: :default] - - # Per-package recv_timeouts as the implementation escalates them. - @first_attempt_opts [recv_timeout: 30_000, pool: :default] - @second_attempt_opts [recv_timeout: 15_000, pool: :default] - @third_attempt_opts [recv_timeout: 5_000, pool: :default] - - @schemas_path Application.compile_env(:lightning, :schemas_path) - - # --- helpers ---------------------------------------------------------- - - defp schema_url(package_name) do - "https://cdn.jsdelivr.net/npm/#{package_name}/configuration-schema.json" - end - - defp expect_registry(status_code, body) do - response = {:ok, status_code, "headers", body} - - expect_registry(response) - end - - defp expect_registry(response) do - expect(:hackney, :request, fn - :get, @registry_url, [], "", @registry_request_options -> response - end) - end - - # Stub a single jsdelivr fetch attempt for `package_name` at the given - # timeout-options profile, returning `response`. - defp expect_schema_request(package_name, opts, response) do - url = schema_url(package_name) - - expect(:hackney, :request, fn - :get, ^url, [], "", ^opts -> response - end) - end - - defp expect_schema_request(package_name, opts, status_code, body) do - response = {:ok, status_code, "headers", body} - expect_schema_request(package_name, opts, response) - end - - # Stub all three escalating attempts for `package_name` with the same - # error response (used when verifying retry-exhaustion behaviour). - defp expect_all_attempts_error(package_name, reason) do - error = {:error, reason} - - expect_schema_request(package_name, @first_attempt_opts, error) - expect_schema_request(package_name, @second_attempt_opts, error) - expect_schema_request(package_name, @third_attempt_opts, error) - end - - describe "install_schemas mix task" do - setup do - stub(:hackney) - Mox.stub(Lightning.MockConfig, :adaptor_registry, fn -> [] end) - :ok - end - - test "run reports a tally of installed and skipped packages" do - # Registry returns 3 packages: - # language-common -> excluded by default - # language-asana -> installed on first attempt - # language-primero -> skipped after exhausting all retries - expect_registry( - 200, - ~s({"@openfn/language-common": "write", "@openfn/language-asana": "write", "@openfn/language-primero": "write"}) - ) - - expect_schema_request( - "@openfn/language-asana", - @first_attempt_opts, - 200, - ~s({"name": "language-asana"}) - ) - - expect_all_attempts_error("@openfn/language-primero", :timeout) - - File - |> expect(:rm_rf, fn _ -> nil end) - |> expect(:mkdir_p, fn _ -> nil end) - |> expect(:open!, fn "test/fixtures/schemas/asana.json", [:write] -> - nil - end) - |> expect(:close, fn _ -> nil end) - - expect(IO, :binwrite, fn _, ~s({"name": "language-asana"}) -> nil end) - - {output, log} = - with_log(fn -> - capture_io(fn -> - InstallSchemas.run([]) - end) - end) - - assert output =~ "1 installed, 1 skipped" - - assert log =~ - "Skipping @openfn/language-primero: :timeout after 3 attempt(s)" - end - - test "run raises when the schemas directory cannot be created" do - expect(File, :rm_rf, fn _ -> {:error, "error occured"} end) - expect(File, :mkdir_p, fn _ -> {:error, "error occured"} end) - - assert_raise RuntimeError, - "Couldn't create the schemas directory: test/fixtures/schemas, got :error occured.", - fn -> - InstallSchemas.run([]) - end - end - - test "persist_schema retries transient errors then succeeds" do - # First attempt times out; second succeeds. We also verify that the - # retry escalates to the 15s recv_timeout profile. - expect_schema_request( - "@openfn/language-asana", - @first_attempt_opts, - {:error, :timeout} - ) - - expect_schema_request( - "@openfn/language-asana", - @second_attempt_opts, - 200, - ~s({"name": "language-asana"}) - ) - - File - |> expect(:open!, fn "test/fixtures/schemas/asana.json", [:write] -> - nil - end) - |> expect(:close, fn _ -> nil end) - - expect(IO, :binwrite, fn _, ~s({"name": "language-asana"}) -> nil end) - - {result, log} = - with_log(fn -> - InstallSchemas.persist_schema(@schemas_path, "@openfn/language-asana") - end) - - assert result == {:installed, "@openfn/language-asana"} - - assert log =~ - "Transient error fetching @openfn/language-asana (:timeout); retrying with recv_timeout=15000ms" - end - - test "persist_schema skips after exhausting all retries" do - expect_all_attempts_error("@openfn/language-asana", :timeout) - - {result, log} = - with_log(fn -> - InstallSchemas.persist_schema(@schemas_path, "@openfn/language-asana") - end) - - assert result == {:skipped, "@openfn/language-asana", :timeout} - - assert log =~ - "Skipping @openfn/language-asana: :timeout after 3 attempt(s)" - end - - test "persist_schema skips immediately on non-retriable errors" do - # :nxdomain is not in @retriable_reasons, so we expect exactly one - # attempt (no retry) and an "after 1 attempt(s)" log line. - expect_schema_request( - "@openfn/language-asana", - @first_attempt_opts, - {:error, :nxdomain} - ) - - {result, log} = - with_log(fn -> - InstallSchemas.persist_schema(@schemas_path, "@openfn/language-asana") - end) - - assert result == {:skipped, "@openfn/language-asana", :nxdomain} - - assert log =~ - "Skipping @openfn/language-asana: :nxdomain after 1 attempt(s)" - end - - test "persist_schema skips on non-200 status without retrying" do - expect_schema_request( - "@openfn/language-asana", - @first_attempt_opts, - 400, - "" - ) - - {result, log} = - with_log(fn -> - InstallSchemas.persist_schema(@schemas_path, "@openfn/language-asana") - end) - - assert {:skipped, "@openfn/language-asana", {:http_status, 400}} = result - - assert log =~ - "Unable to fetch @openfn/language-asana configuration schema. status=400" - end - - test "fetch_schemas raises when the registry request errors" do - expect_registry({:error, :some_error}) - - assert_raise RuntimeError, - ~r/Unable to connect to NPM; no adaptors fetched: /, - fn -> InstallSchemas.fetch_schemas([]) end - end - - test "fetch_schemas raises when the registry returns a non-200 status" do - expect_registry(400, "") - - assert_raise RuntimeError, - "Unable to access openfn user packages. status=400", - fn -> InstallSchemas.fetch_schemas([]) end - end - - test "fetch_schemas preserves the package name when a worker crashes" do - # Regression guard: an earlier bug surfaced the {:exit, _} branch with - # the name "unknown" because results weren't zipped against the input - # names. We feed fetch_schemas a worker that always raises and assert - # the skip tuple still carries the package name. - expect_registry(200, ~s({"@openfn/language-boom": "write"})) - - crashing_fun = fn _name -> raise "boom" end - - {results, log} = - with_log(fn -> - InstallSchemas.fetch_schemas([], crashing_fun) |> Enum.to_list() - end) - - assert [{:skipped, "@openfn/language-boom", _reason}] = results - assert log =~ "Schema fetch worker for @openfn/language-boom crashed" - end - - test "parse_excluded merges CLI args with defaults" do - assert [ - "pack1", - "pack2", - "language-common", - "language-devtools", - "language-divoc" - ] == - InstallSchemas.parse_excluded(["--exclude", "pack1", "pack2"]) - - assert ["language-common", "language-devtools", "language-divoc"] == - InstallSchemas.parse_excluded([]) - end - end -end diff --git a/test/mix/tasks/lightning.adaptors.dump_test.exs b/test/mix/tasks/lightning.adaptors.dump_test.exs index 00a82daee52..21af93b079f 100644 --- a/test/mix/tasks/lightning.adaptors.dump_test.exs +++ b/test/mix/tasks/lightning.adaptors.dump_test.exs @@ -20,6 +20,9 @@ defmodule Mix.Tasks.Lightning.Adaptors.DumpTest do deprecated: false, schema_data: ~s({"type":"object"}), schema_sha256: "sha256-schema-http", + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "square-bytes"), + icon_square_etag: "\"square-etag\"", versions: [ %{ version: "2.0.0", @@ -69,7 +72,8 @@ defmodule Mix.Tasks.Lightning.Adaptors.DumpTest do @compared_adaptor_fields ~w(name source description homepage repository license latest_version deprecated schema_data - schema_sha256)a + schema_sha256 icon_square_ext icon_square_sha256 + icon_square_etag)a @compared_version_fields ~w(version integrity tarball_url size_bytes dependencies peer_dependencies published_at @@ -112,6 +116,10 @@ defmodule Mix.Tasks.Lightning.Adaptors.DumpTest do assert http["latest_version"] == "2.1.0" assert http["description"] == "HTTP adaptor" assert http["schema_data"] == ~s({"type":"object"}) + assert http["icon_square_ext"] == "png" + + assert http["icon_square_sha256"] == + Base.encode64(:crypto.hash(:sha256, "square-bytes")) assert http["versions"] |> Enum.map(& &1["version"]) |> Enum.sort() == ["2.0.0", "2.1.0"] diff --git a/test/test_helper.exs b/test/test_helper.exs index 790779e12bb..942b070c13f 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -20,7 +20,6 @@ Mimic.copy(Lightning.Adaptors.Config) Mimic.copy(Lightning.FailureEmail) Mimic.copy(Lightning.Projects.Provisioner) Mimic.copy(Lightning.MetadataService) -Mimic.copy(Mix.Tasks.Lightning.InstallSchemas) # Other ExUnit configuration can be found in `config/runtime.exs`, # for example to change the `assert_receive` timeout, configure it using the diff --git a/tooling/adaptor_cache/README.md b/tooling/adaptor_cache/README.md index 2705626bcba..9d9714594f6 100644 --- a/tooling/adaptor_cache/README.md +++ b/tooling/adaptor_cache/README.md @@ -131,14 +131,10 @@ bin/adaptor_cache scenario restore drill-1 # back to exactly that state ``` Scenarios live under `tooling/adaptor_cache/scenarios//` and stay -untracked (not checked into git) for now. +untracked (not checked into git). ## Caveats -- **The legacy `Lightning.AdaptorRegistry` and `mix lightning.install_schemas` - bypass this entirely.** Both have hardcoded upstream URLs and don't read the - `ADAPTORS_NPM_*` env vars, so they'll always hit the real internet regardless - of whether the cache is up. - **Never set this as your global npm registry in `~/.npmrc`.** The `/npm/` prefix is a transparent GET proxy of registry.npmjs.org, so npm would mostly work, badly: this cache never expires what it records, so `npm install` could From 0e03dab985126722f1fcc58358d67c7a55251fd2 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 3 Sep 2026 15:51:32 +0200 Subject: [PATCH 08/37] Isolate adaptor tests from the shared Lightning.Adaptors instance - Release form added for dumping the adaptor catalogue, refresh via rpc documented - Real-worker tests fail fast on a non-success run and seed the adaptor they need - Adaptor tests isolated from the shared Lightning.Adaptors instance, with a wait for the scheduler to come up and follow-up cleanup - ADAPTORS.md trimmed to reduce reading time without losing any instruction --- .claude/rules/adaptors-otp.md | 14 ++- ADAPTORS.md | 113 +++++++++--------- lib/lightning/adaptors.ex | 51 +++++--- lib/lightning/adaptors/config.ex | 11 ++ lib/lightning/adaptors/dump.ex | 65 ++++++++++ lib/lightning/adaptors/seed.ex | 4 +- lib/lightning/release.ex | 22 ++++ .../controllers/adaptor_icon_controller.ex | 38 ++++-- lib/mix/tasks/lightning.adaptors.dump.ex | 45 +------ test/integration/web_and_worker_test.exs | 18 ++- test/integration/workflow_edge_cases_test.exs | 2 + test/lightning/accounts_test.exs | 3 + test/lightning/adaptors/config_test.exs | 6 + .../adaptors/isolated_adaptors_test.exs | 76 ++++++++++++ test/lightning/adaptors/seed_test.exs | 11 +- test/lightning/adaptors_test.exs | 57 +++++---- .../ai_assistant/ai_assistant_test.exs | 6 +- .../collaboration/no_change_snapshot_test.exs | 5 +- test/lightning/collaboration/session_test.exs | 2 + test/lightning/credentials/schema_test.exs | 9 +- test/lightning/credentials_test.exs | 5 + test/lightning/jobs_test.exs | 4 + test/lightning/metadata_service_test.exs | 6 +- test/lightning/projects/provisioner_test.exs | 9 +- test/lightning/setup_utils_test.exs | 3 + test/lightning/workflows/edge_test.exs | 4 + test/lightning/workflows/job_test.exs | 13 +- test/lightning/workflows/query_test.exs | 3 + test/lightning/workflows_test.exs | 7 +- .../channels/ai_assistant_channel_test.exs | 2 + .../channels/run_channel_test.exs | 4 +- .../channels/run_with_options_test.exs | 6 +- .../workflow_channel_broadcast_test.exs | 2 + .../channels/workflow_channel_test.exs | 20 ++-- .../live/credential_live_test.exs | 2 + test/lightning_web/live/project_live_test.exs | 2 + .../live/workflow_live/collaborate_test.exs | 3 + .../tasks/lightning.adaptors.import_test.exs | 3 + test/support/adaptor_test_helpers.ex | 98 +++++++++++++-- test/support/factories.ex | 2 + 40 files changed, 546 insertions(+), 210 deletions(-) create mode 100644 lib/lightning/adaptors/dump.ex create mode 100644 test/lightning/adaptors/isolated_adaptors_test.exs diff --git a/.claude/rules/adaptors-otp.md b/.claude/rules/adaptors-otp.md index 4345c1ad2f4..3dd5895360c 100644 --- a/.claude/rules/adaptors-otp.md +++ b/.claude/rules/adaptors-otp.md @@ -22,8 +22,18 @@ When adding or changing a process here: collaborators passed in the child spec. Do not add a `Registry`: the child set is fixed and the registered atom already addresses it. - Public functions that talk to a running process lead with the server ref, - defaulted: `def refresh(sup \\ @sup, name)`. `start_link` takes `name:` in - trailing opts. + defaulted: `def refresh(sup \\ Config.default_instance(), name)`. `start_link` + takes `name:` in trailing opts. +- `Lightning.Adaptors.Config.default_instance/0` is what production code and + every public `Lightning.Adaptors` function default their `sup` argument + through, not a hardcoded module attribute. Tests get a private instance with + `setup :isolated_adaptors` (from `Lightning.AdaptorTestHelpers`), which + starts a fresh supervisor and stubs `default_instance/0` to it, instead of + touching the global instance. For an `async: true` module, that stub only + reaches processes reachable via `$callers` (`Task`, `start_supervised!`, + ...); a process spawned outside that chain, or already running, still + sees the real global instance. An `async: false` module gets Mimic's + global mode instead, which reaches every process in the VM. - The `Scheduler` is a cluster singleton behind `HighlanderPG` and registers under `global_scheduler_name/1` (`supervisor.ex:149`). `Process.whereis` will not find it. diff --git a/ADAPTORS.md b/ADAPTORS.md index 5ceb1872642..ae3b2b1fcc6 100644 --- a/ADAPTORS.md +++ b/ADAPTORS.md @@ -1,59 +1,59 @@ # Adaptors -The adaptor registry is the catalogue of adaptors, versions, credential schemas -and icons that the workflow editor shows. Lightning fetches it from npm by -default and keeps a copy in Postgres. For the Elixir side, start at -`Lightning.Adaptors`. +The adaptor registry catalogues adaptors, versions, credential schemas and icons +for the workflow editor. Lightning fetches it from npm by default, cached in +Postgres. For the Elixir side, start at `Lightning.Adaptors`. ## Using local adaptors -Point Lightning at a checkout of the adaptors monorepo instead of npm: +Point Lightning at a checkout of the adaptors monorepo, not npm: ```sh ADAPTORS_STRATEGY=local ADAPTORS_LOCAL_REPO=/path/to/adaptors ``` -The path is the repository root, not its `packages/` directory. Every -subdirectory of `packages/` with a `package.json` becomes an adaptor, named and -versioned from that file. Lightning also reads `configuration-schema.json` for -the credential form, and `assets/square` and `assets/rectangle` as `.png` or -`.svg` for the icons. A package without them still appears, with no credential -form or icon. +The path is the repo root, not `packages/`. Each subdirectory with a +`package.json` becomes an adaptor, named and versioned from it. Lightning also +reads `configuration-schema.json` for the credential form and +`assets/square`/`assets/rectangle` (`.png`/`.svg`) for icons; a package without +them still appears, minus the form or icon. -To layer a private checkout over the public one, give more than one root, comma -separated: +To layer a private checkout over the public one, comma-separate multiple roots: ```sh ADAPTORS_LOCAL_REPO=/path/to/private-adaptors,/path/to/adaptors ``` -A package found in more than one root comes from the first root only. Lightning -logs a warning naming each shadowed package on every catalogue scan, not just at -boot. +A package in multiple roots comes from the first; Lightning logs each shadowed +package on every scan, not just at boot. -Lightning still accepts the old names for these two settings, -`LOCAL_ADAPTORS=true` and `OPENFN_ADAPTORS_REPO`. Each warns at boot only when -Lightning falls back to it: `LOCAL_ADAPTORS=true` when `ADAPTORS_STRATEGY` is -unset, `OPENFN_ADAPTORS_REPO` when the strategy is local and -`ADAPTORS_LOCAL_REPO` is not set. +> #### Note {: .info} +> +> Lightning still accepts the old names `LOCAL_ADAPTORS=true` and +> `OPENFN_ADAPTORS_REPO`, warning at boot only when it falls back to them: +> `LOCAL_ADAPTORS=true` when `ADAPTORS_STRATEGY` is unset, +> `OPENFN_ADAPTORS_REPO` when the strategy is local and `ADAPTORS_LOCAL_REPO` is +> unset. ## Running without internet access -Copy the catalogue from an instance that has internet access and has finished a -refresh. On that instance, dump the catalogue and archive the icons directory. -The dump holds the icon metadata only, so the icon files have to travel with it: +On an internet-connected, refreshed instance, dump the catalogue. The dump holds +icon metadata only, so archive the icons directory too: ```sh mix lightning.adaptors.dump --path snapshot.json tar czf icons.tar.gz -C "$ADAPTORS_ICONS_PATH" . ``` -If `ADAPTORS_ICONS_PATH` is not set, the directory is `lightning/adaptor_icons` -under the system temp directory. +`ADAPTORS_ICONS_PATH` defaults to `lightning/adaptor_icons` under the temp +directory. On a release image (no Mix), dump with: -On the offline instance, unpack the icons where `ADAPTORS_ICONS_PATH` points, -then import the snapshot: +```sh +bin/lightning eval 'Lightning.Release.dump_adaptors("/path/to/snapshot.json")' +``` + +Offline, unpack icons to `ADAPTORS_ICONS_PATH`, then import: ```sh mkdir -p "$ADAPTORS_ICONS_PATH" @@ -61,58 +61,61 @@ tar xzf icons.tar.gz -C "$ADAPTORS_ICONS_PATH" mix lightning.adaptors.import --path snapshot.json --replace ``` -The `mix` commands need a source checkout. A release image has no Mix, so import -there with: +On a release image, import with: ```sh bin/lightning eval 'Lightning.Release.seed_adaptors("/path/to/snapshot.json", replace: true)' ``` -With no populated instance to dump from, build the snapshot straight from npm on -any machine with internet access. This needs no database and carries no icons: +With no populated instance, build the snapshot from npm anywhere online (no +database, no icons): ```sh mix lightning.adaptors.snapshot --path snapshot.json ``` -Import it as above. +Import as above. -If you run internal mirrors instead, any npm-compatible registry works. Set +Internal mirrors: any npm-compatible registry works. Set `ADAPTORS_NPM_REGISTRY_URL`, `ADAPTORS_NPM_JSDELIVR_URL` and -`ADAPTORS_NPM_GITHUB_URL`, and leave the strategy as npm. Set -`ADAPTORS_NPM_GITHUB_REF` too if the mirror serves a branch other than `main`. +`ADAPTORS_NPM_GITHUB_URL`, leave the strategy as npm, and set +`ADAPTORS_NPM_GITHUB_REF` if the mirror serves a branch other than `main`. -An imported catalogue survives the hourly refresh. When the refresh cannot reach -its source it logs a warning and leaves the existing rows alone. +An imported catalogue survives the hourly refresh; a failed one logs a warning +and leaves rows alone. -The worker installs adaptor packages into `ADAPTORS_PATH` on its own. That is a -separate download and none of the above provides it. +The worker installs adaptor packages into `ADAPTORS_PATH` itself, a separate +download not covered here. ## Keeping the catalogue fresh -Lightning refreshes the catalogue from its source every hour. To force a refresh -now, on a source checkout: +Lightning refreshes the catalogue hourly; force one, on a source checkout: ```sh mix lightning.adaptors.refresh mix lightning.adaptors.refresh --name @openfn/language-http ``` -Without `--name` it runs a full refresh and waits for it to finish. With -`--name` it refetches that one adaptor, whether or not its version changed. Exit -codes are in the task's `mix help` output. +Without `--name` it runs a full refresh and waits; with `--name` it refetches +that adaptor regardless of version change. Exit codes are in +`mix help lightning.adaptors.refresh`. + +A release image has no Mix; run the same call against the node: + +```sh +bin/lightning rpc 'Lightning.Adaptors.refresh(await: true)' +bin/lightning rpc 'Lightning.Adaptors.refresh_package("@openfn/language-http")' +``` ## Troubleshooting -- An adaptor is missing from the picker: look for a `fetch_adaptor` warning - naming it in the log, then force a refresh with `--name`. -- A new version is not showing: the hourly refresh has not run yet. Force one, - or wait for the next. -- Icons are missing after an import: the icons directory never reached - `ADAPTORS_ICONS_PATH` on this instance, or the dump came from a Lightning - version that did not write icon metadata. Redo the dump and copy the +- Adaptor missing from the picker: find its `fetch_adaptor` warning in the log, + then force a refresh with `--name`. +- New version not showing: the hourly refresh hasn't run. Force one, or wait. +- Icons missing after import: they never reached `ADAPTORS_ICONS_PATH` on this + instance, or the dump predates icon metadata. Redo the dump and copy the directory. -- A local package is ignored: an earlier root in `ADAPTORS_LOCAL_REPO` has a - package with the same name. The log names each shadowed package. -- A boot warning says a variable is deprecated: rename `LOCAL_ADAPTORS=true` to +- Local package ignored: an earlier `ADAPTORS_LOCAL_REPO` root has a package of + the same name; the log names each shadowed package. +- Deprecated-variable boot warning: rename `LOCAL_ADAPTORS=true` to `ADAPTORS_STRATEGY=local` and `OPENFN_ADAPTORS_REPO` to `ADAPTORS_LOCAL_REPO`. diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index 44b7ecbe3a2..465ef38c8c7 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -32,9 +32,13 @@ defmodule Lightning.Adaptors do ## Testing Every function that talks to a running process takes the supervisor - name as an optional first argument, defaulting to `Lightning.Adaptors`. - Start a `Lightning.Adaptors.Supervisor` under another name and pass - that name to run a test against its own catalogue and cache. + name as an optional first argument, defaulting to + `Lightning.Adaptors.Config.default_instance/0`. Start a + `Lightning.Adaptors.Supervisor` under another name and pass that name + to run a test against its own catalogue and cache, or stub + `default_instance/0` to make it the default for the test. The + recommended way to get a private instance is `setup :isolated_adaptors` + from `Lightning.AdaptorTestHelpers`, which does both. """ alias Lightning.Adaptors.Catalogue @@ -96,13 +100,11 @@ defmodule Lightning.Adaptors do ] end - @sup Lightning.Adaptors - @doc """ Returns every adaptor in the catalogue as `Package` structs. """ @spec packages(atom()) :: {:ok, [Package.t()]} | {:error, :timeout | term()} - def packages(sup \\ @sup) do + def packages(sup \\ Config.default_instance()) do with {:ok, metas} <- Store.packages(sup) do source = AdaptorsSupervisor.source(sup) {:ok, Enum.map(metas, &to_package(&1, source))} @@ -114,7 +116,7 @@ defmodule Lightning.Adaptors do binary. """ @spec schema(atom(), String.t()) :: {:ok, String.t()} | {:error, term()} - def schema(sup \\ @sup, pkg), do: Store.schema(sup, pkg) + def schema(sup \\ Config.default_instance(), pkg), do: Store.schema(sup, pkg) @doc """ Returns the on-disk path of the adaptor's `:square` or `:rectangle` @@ -122,7 +124,8 @@ defmodule Lightning.Adaptors do """ @spec icon(atom(), String.t(), :square | :rectangle) :: {:ok, Path.t()} | {:error, term()} - def icon(sup \\ @sup, pkg, shape), do: Store.icon(sup, pkg, shape) + def icon(sup \\ Config.default_instance(), pkg, shape), + do: Store.icon(sup, pkg, shape) @doc """ Returns the picker catalogue as `{{latest_updated_at, count}, entries}`: @@ -133,7 +136,7 @@ defmodule Lightning.Adaptors do """ @spec catalogue_with_stamp(atom()) :: {{DateTime.t() | nil, non_neg_integer()}, [Store.catalogue_entry()]} - def catalogue_with_stamp(sup \\ @sup) do + def catalogue_with_stamp(sup \\ Config.default_instance()) do {:ok, catalogue} = Store.catalogue(sup) catalogue end @@ -145,7 +148,7 @@ defmodule Lightning.Adaptors do for the catalogue to load; see `fetch_adaptor/2` for that. """ @spec get_adaptor(atom(), String.t()) :: Package.t() | nil - def get_adaptor(sup \\ @sup, name) when is_binary(name), + def get_adaptor(sup \\ Config.default_instance(), name) when is_binary(name), do: lookup(sup, name) @doc """ @@ -163,7 +166,8 @@ defmodule Lightning.Adaptors do @spec fetch_adaptor(atom(), String.t()) :: {:ok, Package.t()} | {:error, :not_found | :timeout | :unavailable | :not_ready} - def fetch_adaptor(sup \\ @sup, name) when is_binary(name) do + def fetch_adaptor(sup \\ Config.default_instance(), name) + when is_binary(name) do case lookup(sup, name) do %Package{} = package -> {:ok, package} @@ -214,7 +218,7 @@ defmodule Lightning.Adaptors do """ @spec ensure_loaded(atom()) :: :ok | {:error, :timeout | :unavailable | :not_ready} - def ensure_loaded(sup \\ @sup) do + def ensure_loaded(sup \\ Config.default_instance()) do if ready?(sup), do: :ok, else: load(sup) end @@ -260,7 +264,7 @@ defmodule Lightning.Adaptors do @spec to_wire(atom(), String.t() | nil) :: {:ok, String.t()} | {:error, :not_found | :timeout | :unavailable | :not_ready} - def to_wire(sup \\ @sup, spec) + def to_wire(sup \\ Config.default_instance(), spec) def to_wire(_sup, nil), do: {:ok, ""} @@ -289,9 +293,10 @@ defmodule Lightning.Adaptors do """ @spec refresh(atom(), keyword()) :: :ok | {:ok, Scheduler.refresh_counts()} | {:error, term()} - def refresh(opts) when is_list(opts), do: refresh(@sup, opts) + def refresh(opts) when is_list(opts), + do: refresh(Config.default_instance(), opts) - def refresh(sup \\ @sup, opts \\ []) do + def refresh(sup \\ Config.default_instance(), opts \\ []) do scheduler = AdaptorsSupervisor.global_scheduler_name(sup) if opts[:await] do @@ -311,7 +316,8 @@ defmodule Lightning.Adaptors do """ @spec refresh_package(atom(), String.t()) :: :ok | {:error, :not_found | term()} - def refresh_package(sup \\ @sup, name) when is_binary(name) do + def refresh_package(sup \\ Config.default_instance(), name) + when is_binary(name) do Scheduler.refresh_package( AdaptorsSupervisor.global_scheduler_name(sup), name @@ -328,7 +334,7 @@ defmodule Lightning.Adaptors do @spec refresh_icons(atom()) :: {:ok, %{updated: non_neg_integer(), unchanged: non_neg_integer()}} | {:error, term()} - def refresh_icons(sup \\ @sup) do + def refresh_icons(sup \\ Config.default_instance()) do Scheduler.refresh_icons(AdaptorsSupervisor.global_scheduler_name(sup)) catch :exit, {:timeout, _} -> {:error, :timeout} @@ -342,7 +348,8 @@ defmodule Lightning.Adaptors do """ @spec icon_meta(atom(), String.t()) :: {:ok, Store.icon_meta()} | {:error, :not_found} - def icon_meta(sup \\ @sup, name), do: Store.icon_meta(sup, name) + def icon_meta(sup \\ Config.default_instance(), name), + do: Store.icon_meta(sup, name) @doc """ Subscribes the calling process to catalogue update broadcasts. @@ -351,7 +358,7 @@ defmodule Lightning.Adaptors do messages. """ @spec subscribe_to_updates(atom()) :: :ok | {:error, term()} - def subscribe_to_updates(sup \\ @sup) do + def subscribe_to_updates(sup \\ Config.default_instance()) do Phoenix.PubSub.subscribe( Lightning.PubSub, AdaptorsSupervisor.client_topic(sup) @@ -363,4 +370,10 @@ defmodule Lightning.Adaptors do `Lightning.Adaptors.Seed.seed_from_file/2`. """ defdelegate seed_from_file(path, opts \\ []), to: Lightning.Adaptors.Seed + + @doc """ + Writes the catalogue to a JSON snapshot file. See + `Lightning.Adaptors.Dump.dump_to_file/2`. + """ + defdelegate dump_to_file(path, opts \\ []), to: Lightning.Adaptors.Dump end diff --git a/lib/lightning/adaptors/config.ex b/lib/lightning/adaptors/config.ex index 5c7101647b7..79de389c7ee 100644 --- a/lib/lightning/adaptors/config.ex +++ b/lib/lightning/adaptors/config.ex @@ -15,6 +15,17 @@ defmodule Lightning.Adaptors.Config do @default_icon_path {:tmp, "lightning/adaptor_icons"} @default_first_load_timeout :timer.seconds(60) + @doc """ + The supervisor instance public `Lightning.Adaptors` functions read + through when called with no explicit instance. Defaults to + `Lightning.Adaptors`, the one started in `application.ex`. + + Tests stub this to isolate reads to a private instance; see + `Lightning.AdaptorTestHelpers.isolated_adaptors/1`. + """ + @spec default_instance() :: atom() + def default_instance, do: get(:default_instance, Lightning.Adaptors) + @doc """ The active strategy module. Defaults to `Lightning.Adaptors.NPM`. """ diff --git a/lib/lightning/adaptors/dump.ex b/lib/lightning/adaptors/dump.ex new file mode 100644 index 00000000000..b211ba702f5 --- /dev/null +++ b/lib/lightning/adaptors/dump.ex @@ -0,0 +1,65 @@ +defmodule Lightning.Adaptors.Dump do + @moduledoc """ + Writes the adaptor catalogue to a JSON snapshot file, in the shape + `Lightning.Adaptors.Seed.seed_from_file/2` reads back. + """ + + alias Lightning.Adaptors.Catalogue + + @adaptor_fields ~w(name source description homepage repository license + latest_version deprecated schema_data schema_sha256 + icon_square_ext icon_rectangle_ext + icon_square_sha256 icon_rectangle_sha256 + icon_square_etag icon_rectangle_etag)a + + @version_fields ~w(version integrity tarball_url size_bytes dependencies + peer_dependencies published_at deprecated)a + + @icon_sha256_fields ~w(icon_square_sha256 icon_rectangle_sha256)a + + @doc """ + Writes every `source` adaptor and its versions to `path` as JSON, and + returns the number of records written. + + Options: + + * `:source` - `:npm` (default) or `:local` + """ + @spec dump_to_file(Path.t(), keyword()) :: {:ok, non_neg_integer()} + def dump_to_file(path, opts \\ []) do + source = Keyword.get(opts, :source, :npm) + + records = + source + |> Catalogue.list_adaptors() + |> Enum.map(&dump_record(&1, source)) + + File.write!(path, Jason.encode_to_iodata!(records)) + + {:ok, length(records)} + end + + # ponytail: one version query per adaptor; join them if a catalogue ever + # grows past a few hundred rows. + defp dump_record(adaptor, source) do + versions = + adaptor.name + |> Catalogue.list_versions(source) + |> Enum.map(&(&1 |> Map.from_struct() |> Map.take(@version_fields))) + + adaptor + |> Map.from_struct() + |> Map.take(@adaptor_fields) + |> encode_icon_sha256s() + |> Map.put(:versions, versions) + end + + # icon_*_sha256 columns hold raw hash bytes, not valid JSON text; + # `Lightning.Adaptors.Seed.normalize_snapshot_record/2` decodes on the + # way back in. + defp encode_icon_sha256s(record) do + Enum.reduce(@icon_sha256_fields, record, fn field, acc -> + Map.update!(acc, field, &(&1 && Base.encode64(&1))) + end) + end +end diff --git a/lib/lightning/adaptors/seed.ex b/lib/lightning/adaptors/seed.ex index 37c912065ea..4df67aade9b 100644 --- a/lib/lightning/adaptors/seed.ex +++ b/lib/lightning/adaptors/seed.ex @@ -25,13 +25,13 @@ defmodule Lightning.Adaptors.Seed do * `:replace` - when `true`, deletes every existing row for the source first, in the same transaction as the upserts * `:sup` - supervisor instance whose topic the broadcasts go to, - defaulting to `Lightning.Adaptors` + defaulting to `Lightning.Adaptors.Config.default_instance/0` """ @spec seed_from_file(Path.t(), keyword()) :: {:ok, non_neg_integer()} def seed_from_file(path, opts \\ []) do source = Keyword.get(opts, :source, :npm) replace? = Keyword.get(opts, :replace, false) - sup = Keyword.get(opts, :sup, Lightning.Adaptors) + sup = Keyword.get(opts, :sup, Lightning.Adaptors.Config.default_instance()) records = path diff --git a/lib/lightning/release.ex b/lib/lightning/release.ex index 06ba2cf7aa3..0b9534e18cf 100644 --- a/lib/lightning/release.ex +++ b/lib/lightning/release.ex @@ -58,6 +58,28 @@ defmodule Lightning.Release do result end + @doc """ + Write the adaptor catalogue to a JSON snapshot file, without the rest + of the app running. This is the release equivalent of + `mix lightning.adaptors.dump` — a release has no Mix, so run this + through `bin/lightning eval` instead. + + ## Usage + + bin/lightning eval 'Lightning.Release.dump_adaptors("/path/to/snapshot.json")' + bin/lightning eval 'Lightning.Release.dump_adaptors("/path/to/snapshot.json", source: :local)' + """ + def dump_adaptors(path, opts \\ []) do + load_app() + + {:ok, result, _apps} = + Ecto.Migrator.with_repo(@repo, fn _repo -> + Lightning.Adaptors.dump_to_file(path, opts) + end) + + result + end + def rollback(repo, version) do load_app() diff --git a/lib/lightning_web/controllers/adaptor_icon_controller.ex b/lib/lightning_web/controllers/adaptor_icon_controller.ex index c78a8f7512d..ca89a949a97 100644 --- a/lib/lightning_web/controllers/adaptor_icon_controller.ex +++ b/lib/lightning_web/controllers/adaptor_icon_controller.ex @@ -15,8 +15,8 @@ defmodule LightningWeb.AdaptorIconURL do """ @spec build(String.t(), map(), :square | :rectangle) :: String.t() | nil def build(name, meta, shape) do - with ext when not is_nil(ext) <- Map.get(meta, :"icon_#{shape}_ext"), - sha when not is_nil(sha) <- Map.get(meta, :"icon_#{shape}_sha256") do + with ext when not is_nil(ext) <- ext_for_shape(meta, shape), + sha when not is_nil(sha) <- sha_for_shape(meta, shape) do sha8 = sha |> binary_part(0, 4) |> Base.encode16(case: :lower) "/adaptors/icons/#{URI.encode(name, &URI.char_unreserved?/1)}/" <> @@ -25,6 +25,14 @@ defmodule LightningWeb.AdaptorIconURL do _ -> nil end end + + defp ext_for_shape(meta, :square), do: Map.get(meta, :icon_square_ext) + defp ext_for_shape(meta, :rectangle), do: Map.get(meta, :icon_rectangle_ext) + + defp sha_for_shape(meta, :square), do: Map.get(meta, :icon_square_sha256) + + defp sha_for_shape(meta, :rectangle), + do: Map.get(meta, :icon_rectangle_sha256) end defmodule LightningWeb.AdaptorIconController do @@ -83,7 +91,7 @@ defmodule LightningWeb.AdaptorIconController do {:ok, meta} -> cond do - ext_for_shape(meta, shape) != ext -> + ext_for_shape_param(meta, shape) != ext -> send_resp(conn, 404, "") not has_icon?(meta, shape) -> @@ -127,19 +135,23 @@ defmodule LightningWeb.AdaptorIconController do |> send_resp(302, "") end - defp has_icon?(meta, shape), do: not is_nil(ext_for_shape(meta, shape)) + defp has_icon?(meta, shape), do: not is_nil(ext_for_shape_param(meta, shape)) - defp ext_for_shape(meta, shape), do: Map.get(meta, :"icon_#{shape}_ext") + defp ext_for_shape_param(meta, "square"), do: Map.get(meta, :icon_square_ext) - defp sha_matches?(meta, shape, sha8) do - case Map.get(meta, :"icon_#{shape}_sha256") do - <> -> - Base.encode16(prefix, case: :lower) == String.downcase(sha8) + defp ext_for_shape_param(meta, "rectangle"), + do: Map.get(meta, :icon_rectangle_ext) - _ -> - false - end - end + defp sha_matches?(meta, "square", sha8), + do: sha_prefix_matches?(Map.get(meta, :icon_square_sha256), sha8) + + defp sha_matches?(meta, "rectangle", sha8), + do: sha_prefix_matches?(Map.get(meta, :icon_rectangle_sha256), sha8) + + defp sha_prefix_matches?(<>, sha8), + do: Base.encode16(prefix, case: :lower) == String.downcase(sha8) + + defp sha_prefix_matches?(_, _sha8), do: false defp content_type_for("png"), do: "image/png" defp content_type_for("svg"), do: "image/svg+xml" diff --git a/lib/mix/tasks/lightning.adaptors.dump.ex b/lib/mix/tasks/lightning.adaptors.dump.ex index 188d22792f3..26b34dcfdf8 100644 --- a/lib/mix/tasks/lightning.adaptors.dump.ex +++ b/lib/mix/tasks/lightning.adaptors.dump.ex @@ -29,18 +29,7 @@ defmodule Mix.Tasks.Lightning.Adaptors.Dump do use Mix.Task - alias Lightning.Adaptors.Catalogue - - @adaptor_fields ~w(name source description homepage repository license - latest_version deprecated schema_data schema_sha256 - icon_square_ext icon_rectangle_ext - icon_square_sha256 icon_rectangle_sha256 - icon_square_etag icon_rectangle_etag)a - - @version_fields ~w(version integrity tarball_url size_bytes dependencies - peer_dependencies published_at deprecated)a - - @icon_sha256_fields ~w(icon_square_sha256 icon_rectangle_sha256)a + alias Lightning.Adaptors @impl Mix.Task def run(argv) do @@ -54,37 +43,9 @@ defmodule Mix.Tasks.Lightning.Adaptors.Dump do source = parse_source(opts[:source]) - records = - source - |> Catalogue.list_adaptors() - |> Enum.map(&dump_record(&1, source)) - - File.write!(path, Jason.encode_to_iodata!(records)) - - Mix.shell().info("Dumped #{length(records)} adaptor(s) to #{path}.") - end - - # ponytail: one version query per adaptor; join them if a catalogue ever - # grows past a few hundred rows. - defp dump_record(adaptor, source) do - versions = - adaptor.name - |> Catalogue.list_versions(source) - |> Enum.map(&(&1 |> Map.from_struct() |> Map.take(@version_fields))) - - adaptor - |> Map.from_struct() - |> Map.take(@adaptor_fields) - |> encode_icon_sha256s() - |> Map.put(:versions, versions) - end + {:ok, count} = Adaptors.dump_to_file(path, source: source) - # icon_*_sha256 columns hold raw hash bytes, not valid JSON text; encode - # them here, `Seed.normalize_snapshot_record/2` decodes on the way back in. - defp encode_icon_sha256s(record) do - Enum.reduce(@icon_sha256_fields, record, fn field, acc -> - Map.update!(acc, field, &(&1 && Base.encode64(&1))) - end) + Mix.shell().info("Dumped #{count} adaptor(s) to #{path}.") end defp parse_source(nil), do: :npm diff --git a/test/integration/web_and_worker_test.exs b/test/integration/web_and_worker_test.exs index cb3eeb08d2a..de6c99ea1dd 100644 --- a/test/integration/web_and_worker_test.exs +++ b/test/integration/web_and_worker_test.exs @@ -2,6 +2,7 @@ defmodule Lightning.WebAndWorkerTest do use LightningWeb.ConnCase, async: false import Ecto.Query + import Lightning.AdaptorTestHelpers import Lightning.Factories import Mox @@ -41,7 +42,12 @@ defmodule Lightning.WebAndWorkerTest do end describe "webhook triggered runs" do - setup [:register_and_log_in_superuser, :stub_rate_limiter_ok] + setup [ + :isolated_adaptors, + :register_and_log_in_superuser, + :stub_rate_limiter_ok, + :seed_default_adaptor + ] @tag :integration @tag timeout: 120_000 @@ -119,7 +125,7 @@ defmodule Lightning.WebAndWorkerTest do end @tag :integration - @tag timeout: 20_000 + @tag timeout: 120_000 test "the whole thing", %{conn: conn, user: user} do Lightning.AdaptorTestHelpers.seed_adaptor_package( "@openfn/language-http", @@ -407,6 +413,7 @@ defmodule Lightning.WebAndWorkerTest do describe "webhook with delayed response (after_completion)" do setup [ + :isolated_adaptors, :register_and_log_in_superuser, :stub_rate_limiter_ok, :seed_default_adaptor @@ -1103,6 +1110,8 @@ defmodule Lightning.WebAndWorkerTest do start_supervised!({RuntimeManager, opts}, restart: :temporary) end + @failure_states Lightning.Run.final_states() -- [:success] + # A dead RuntimeManager means the Node worker never started (or exited), so # fail immediately with its reason instead of blocking for the full window. defp await_run_success(run_id, runtime_manager) do @@ -1112,6 +1121,11 @@ defmodule Lightning.WebAndWorkerTest do %Events.RunUpdated{run: %{id: ^run_id, state: :success}} -> Process.demonitor(ref, [:flush]) + %Events.RunUpdated{run: %{id: ^run_id, state: state} = run} + when state in @failure_states -> + Process.demonitor(ref, [:flush]) + flunk("run #{run_id} finished with state #{state}: #{inspect(run)}") + {:DOWN, ^ref, :process, _pid, :noproc} -> flunk( "runtime manager had already exited before the run started - " <> diff --git a/test/integration/workflow_edge_cases_test.exs b/test/integration/workflow_edge_cases_test.exs index 66933c136cc..813f5e81fab 100644 --- a/test/integration/workflow_edge_cases_test.exs +++ b/test/integration/workflow_edge_cases_test.exs @@ -10,6 +10,7 @@ defmodule Lightning.WorkflowEdgeCasesTest do use LightningWeb.ConnCase, async: false import Ecto.Query + import Lightning.AdaptorTestHelpers import Lightning.Factories import Mox @@ -41,6 +42,7 @@ defmodule Lightning.WorkflowEdgeCasesTest do end setup [ + :isolated_adaptors, :register_and_log_in_superuser, :stub_rate_limiter_ok, :seed_default_adaptor diff --git a/test/lightning/accounts_test.exs b/test/lightning/accounts_test.exs index 539e3d4a574..c9fa7edfc54 100644 --- a/test/lightning/accounts_test.exs +++ b/test/lightning/accounts_test.exs @@ -16,6 +16,7 @@ defmodule Lightning.AccountsTest do alias LightningWeb.UserAuth import Lightning.AccountsFixtures + import Lightning.AdaptorTestHelpers import Lightning.Factories import Swoosh.TestAssertions @@ -790,6 +791,8 @@ defmodule Lightning.AccountsTest do end describe "purge user" do + setup :isolated_adaptors + test "purging a user removes that user from projects they are members of and deletes them from the system" do %{project_users: [proj_user1]} = insert(:project, diff --git a/test/lightning/adaptors/config_test.exs b/test/lightning/adaptors/config_test.exs index 382a2dd23fb..aee7705188b 100644 --- a/test/lightning/adaptors/config_test.exs +++ b/test/lightning/adaptors/config_test.exs @@ -5,6 +5,12 @@ defmodule Lightning.Adaptors.ConfigTest do @parent_key Lightning.Adaptors + describe "default_instance/0" do + test "defaults to the global Lightning.Adaptors instance" do + assert Config.default_instance() == Lightning.Adaptors + end + end + describe "source_for/1" do test "returns :local for Lightning.Adaptors.Local" do assert Config.source_for(Lightning.Adaptors.Local) == :local diff --git a/test/lightning/adaptors/isolated_adaptors_test.exs b/test/lightning/adaptors/isolated_adaptors_test.exs new file mode 100644 index 00000000000..bbd20c0222e --- /dev/null +++ b/test/lightning/adaptors/isolated_adaptors_test.exs @@ -0,0 +1,76 @@ +defmodule Lightning.Adaptors.IsolatedAdaptorsTest do + @moduledoc """ + `Lightning.AdaptorTestHelpers.isolated_adaptors/1` gives a test its own + `Lightning.Adaptors.Supervisor` instance and makes it the default one, + for the test process and its descendants. + """ + + use Lightning.DataCase, async: true + + import Lightning.AdaptorTestHelpers + + alias Lightning.Adaptors + alias Lightning.Adaptors.Config + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + + describe "isolated" do + setup :isolated_adaptors + + test "the default instance is the isolated one for this test and its descendants", + %{sup: sup} do + assert Config.default_instance() == sup + + assert Task.async(fn -> Config.default_instance() end) |> Task.await() == + sup + + source = AdaptorsSupervisor.source(sup) + + fake_meta = %{ + name: "@openfn/language-isolated-fixture", + latest_version: "1.2.3", + description: nil, + deprecated: false, + icon_square_ext: nil, + icon_rectangle_ext: nil, + icon_square_sha256: nil, + icon_rectangle_sha256: nil + } + + Cachex.put( + AdaptorsSupervisor.cache_name(sup), + {:packages, source}, + {:ok, [fake_meta]} + ) + + assert {:ok, + [%Adaptors.Package{name: "@openfn/language-isolated-fixture"}]} = + Adaptors.packages() + end + + test "seed_credential_schema writes into the isolated cache, not the global one", + %{sup: sup} do + seed_credential_schema("http") + + source = AdaptorsSupervisor.source(sup) + + assert {:ok, {:ok, _schema_body}} = + Cachex.get( + AdaptorsSupervisor.cache_name(sup), + {:schema, "http", source} + ) + + assert Cachex.get( + AdaptorsSupervisor.cache_name(Lightning.Adaptors), + {:schema, "http", source} + ) == {:ok, nil} + end + end + + describe "not isolated" do + test "ensure_adaptor raises without an isolated instance" do + assert_raise RuntimeError, ~r/setup :isolated_adaptors/, fn -> + ensure_adaptor("@openfn/language-common") + end + end + end +end diff --git a/test/lightning/adaptors/seed_test.exs b/test/lightning/adaptors/seed_test.exs index 38bf84e5142..f0a368cabb2 100644 --- a/test/lightning/adaptors/seed_test.exs +++ b/test/lightning/adaptors/seed_test.exs @@ -1,18 +1,16 @@ defmodule Lightning.Adaptors.SeedTest do use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers + alias Lightning.Adaptors.Seed alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor @moduletag :tmp_dir - setup %{tmp_dir: tmp_dir} do - sup = :"seed_test_#{System.unique_integer([:positive])}" - - start_supervised!( - {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} - ) + setup :isolated_adaptors + setup %{sup: sup, tmp_dir: tmp_dir} do :ok = Phoenix.PubSub.subscribe( Lightning.PubSub, @@ -20,7 +18,6 @@ defmodule Lightning.Adaptors.SeedTest do ) {:ok, - sup: sup, source: AdaptorsSupervisor.source(sup), cache: AdaptorsSupervisor.cache_name(sup), tmp_dir: tmp_dir} diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index e9f237b8fef..783835c6cc4 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -2,6 +2,7 @@ defmodule Lightning.AdaptorsTest do use Lightning.DataCase, async: false import Eventually + import Lightning.AdaptorTestHelpers import Mox alias Lightning.Adaptors @@ -11,18 +12,7 @@ defmodule Lightning.AdaptorsTest do setup :set_mox_global setup :verify_on_exit! - - setup do - sup = :"adaptors_test_#{System.unique_integer([:positive])}" - - start_supervised!( - {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} - ) - - Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() - - {:ok, sup: sup} - end + setup :isolated_adaptors defp adaptor_record(overrides \\ []) do overrides = Map.new(overrides) @@ -112,15 +102,31 @@ defmodule Lightning.AdaptorsTest do end end - describe "packages/0 delegates to packages(Lightning.Adaptors)" do - test "packages/0 and packages(Lightning.Adaptors) return identical results" do - # The production `Lightning.Adaptors.Supervisor` is started under the - # name `Lightning.Adaptors` in `application.ex`; in test it uses - # `Lightning.Adaptors.StrategyMock` per `config/test.exs`. Both forms - # resolve to `Store.packages(Lightning.Adaptors)`; equality is always - # guaranteed regardless of cache state. - assert Adaptors.packages() == - Adaptors.packages(Lightning.Adaptors) + describe "default instance resolution" do + test "resolves through the stubbed default instance when one is set", %{ + sup: sup + } do + source = AdaptorsSupervisor.source(sup) + + fake_meta = %{ + name: "@openfn/language-stub-fixture", + latest_version: "9.9.9", + description: nil, + deprecated: false, + icon_square_ext: nil, + icon_rectangle_ext: nil, + icon_square_sha256: nil, + icon_rectangle_sha256: nil + } + + Cachex.put( + AdaptorsSupervisor.cache_name(sup), + {:packages, source}, + {:ok, [fake_meta]} + ) + + assert {:ok, [%Adaptors.Package{name: "@openfn/language-stub-fixture"}]} = + Adaptors.packages() end end @@ -333,16 +339,19 @@ defmodule Lightning.AdaptorsTest do end describe "refresh/1 with a bare keyword list" do - test "defaults the supervisor when given only opts" do + test "defaults the supervisor when given only opts", %{sup: sup} do stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> {:ok, []} end) stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> {:ok, %{}} end) - assert {:ok, _counts} = Adaptors.refresh(await: true, timeout: 2_000) + # The isolated Scheduler sits behind HighlanderPG and only registers + # once it holds the advisory lock. + {:global, gname} = AdaptorsSupervisor.global_scheduler_name(sup) + assert_eventually(is_pid(:global.whereis_name(gname)), 2000) - Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() + assert {:ok, _counts} = Adaptors.refresh(await: true, timeout: 2_000) end end diff --git a/test/lightning/ai_assistant/ai_assistant_test.exs b/test/lightning/ai_assistant/ai_assistant_test.exs index 4edadd9bf17..f2d2ea89bcf 100644 --- a/test/lightning/ai_assistant/ai_assistant_test.exs +++ b/test/lightning/ai_assistant/ai_assistant_test.exs @@ -1,18 +1,20 @@ defmodule Lightning.AiAssistantTest do use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers import Mox alias Lightning.AiAssistant alias Lightning.AiAssistant.ChatMessage setup :verify_on_exit! + setup :isolated_adaptors setup do user = insert(:user) project = insert(:project, project_users: [%{user: user, role: :owner}]) workflow = insert(:simple_workflow, project: project) - insert(:adaptor, name: "@openfn/language-common") - insert(:adaptor, name: "@openfn/language-http") + ensure_adaptor("@openfn/language-common") + ensure_adaptor("@openfn/language-http") [user: user, project: project, workflow: workflow] end diff --git a/test/lightning/collaboration/no_change_snapshot_test.exs b/test/lightning/collaboration/no_change_snapshot_test.exs index 7f3d213952b..117632b0029 100644 --- a/test/lightning/collaboration/no_change_snapshot_test.exs +++ b/test/lightning/collaboration/no_change_snapshot_test.exs @@ -5,6 +5,7 @@ defmodule Lightning.Collaboration.NoChangeSnapshotTest do """ use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers import Lightning.Factories import Lightning.CollaborationHelpers @@ -12,6 +13,8 @@ defmodule Lightning.Collaboration.NoChangeSnapshotTest do alias Lightning.Workflows describe "saving without changes" do + setup :isolated_adaptors + setup do # Each test drives its own isolated collaboration tree (Registry, # DynamicSupervisor, and `:pg` scope), so concurrent tests can't see each @@ -23,7 +26,7 @@ defmodule Lightning.Collaboration.NoChangeSnapshotTest do # granted access by the owner-anchored startup hook via `owner: self()`. Mox.stub(LightningMock, :broadcast, fn _topic, _message -> :ok end) - Lightning.AdaptorTestHelpers.seed_ready_catalogue() + seed_ready_catalogue() instance = start_collaboration_instance() diff --git a/test/lightning/collaboration/session_test.exs b/test/lightning/collaboration/session_test.exs index ef3d9302add..54cc9f60153 100644 --- a/test/lightning/collaboration/session_test.exs +++ b/test/lightning/collaboration/session_test.exs @@ -23,6 +23,8 @@ defmodule Lightning.SessionTest do # and `start_supervised!`, so the DB-writing SharedDoc/PersistenceWriter # children are flushed and stopped — via DocumentSupervisor.terminate/2 — before # this test process (the sandbox owner) exits, even if an assertion raises. + setup :isolated_adaptors + setup do instance = start_collaboration_instance() user = insert(:user) diff --git a/test/lightning/credentials/schema_test.exs b/test/lightning/credentials/schema_test.exs index 9dec9144f5e..eaf4f23acb2 100644 --- a/test/lightning/credentials/schema_test.exs +++ b/test/lightning/credentials/schema_test.exs @@ -2,6 +2,7 @@ defmodule Lightning.Credentials.SchemaTest do use Lightning.DataCase, async: true import ExUnit.CaptureLog + import Lightning.AdaptorTestHelpers import Lightning.Factories import Mox @@ -10,6 +11,7 @@ defmodule Lightning.Credentials.SchemaTest do alias Lightning.Credentials.SchemaDocument setup :verify_on_exit! + setup :isolated_adaptors setup do Mox.stub(Lightning.MockConfig, :sentry, fn -> Lightning.MockSentry end) @@ -318,11 +320,6 @@ defmodule Lightning.Credentials.SchemaTest do end describe "Credentials.get_schema/1" do - setup do - Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() - :ok - end - test "preserves JSON property order from the persisted schema body" do ordered_body = ~s({ "properties": { @@ -339,8 +336,6 @@ defmodule Lightning.Credentials.SchemaTest do schema_data: ordered_body ) - Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() - stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> {:error, :unreachable} end) diff --git a/test/lightning/credentials_test.exs b/test/lightning/credentials_test.exs index 3a0df68b516..1a550673c2e 100644 --- a/test/lightning/credentials_test.exs +++ b/test/lightning/credentials_test.exs @@ -7,6 +7,7 @@ defmodule Lightning.CredentialsTest do alias Lightning.Credentials.Credential alias Lightning.Repo + import Lightning.AdaptorTestHelpers import Lightning.Factories import Ecto.Query import Mox @@ -334,6 +335,8 @@ defmodule Lightning.CredentialsTest do end describe "create_credential/1" do + setup :isolated_adaptors + setup do # create_credential/1 needs a schema on file for body casting to work. Lightning.AdaptorTestHelpers.seed_credential_schema("postgresql") @@ -497,6 +500,8 @@ defmodule Lightning.CredentialsTest do end describe "update_credential/2" do + setup :isolated_adaptors + setup do Lightning.AdaptorTestHelpers.seed_credential_schema("postgresql") :ok diff --git a/test/lightning/jobs_test.exs b/test/lightning/jobs_test.exs index 72a7ce010b6..526e6626caf 100644 --- a/test/lightning/jobs_test.exs +++ b/test/lightning/jobs_test.exs @@ -1,6 +1,8 @@ defmodule Lightning.JobsTest do use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers + alias Lightning.Auditing.Audit alias Lightning.Jobs alias Lightning.Workflows.Job @@ -191,6 +193,8 @@ defmodule Lightning.JobsTest do end describe "create_job/2" do + setup :isolated_adaptors + setup do Lightning.AdaptorTestHelpers.ensure_adaptor("@openfn/language-common") diff --git a/test/lightning/metadata_service_test.exs b/test/lightning/metadata_service_test.exs index be8e3646c06..e607d584a87 100644 --- a/test/lightning/metadata_service_test.exs +++ b/test/lightning/metadata_service_test.exs @@ -1,12 +1,16 @@ defmodule Lightning.MetadataServiceTest do use Lightning.DataCase, async: false + import Lightning.AdaptorTestHelpers + alias Lightning.MetadataService + setup :isolated_adaptors + # Seeds the one name the "succeeds" cases below use; the "not in the # registry" cases rely on their name staying unseeded. setup do - insert(:adaptor, name: "@openfn/language-common") + ensure_adaptor("@openfn/language-common") :ok end diff --git a/test/lightning/projects/provisioner_test.exs b/test/lightning/projects/provisioner_test.exs index ab07b0433cb..ccf96abd3e0 100644 --- a/test/lightning/projects/provisioner_test.exs +++ b/test/lightning/projects/provisioner_test.exs @@ -8,10 +8,13 @@ defmodule Lightning.Projects.ProvisionerTest do alias Lightning.Workflows.Snapshot import Ecto.Query + import Lightning.AdaptorTestHelpers import Lightning.Factories import LightningWeb.CoreComponents, only: [translate_error: 1] describe "parse_document/2 with a new project" do + setup :isolated_adaptors + test "with invalid data" do Mox.verify_on_exit!() @@ -169,7 +172,7 @@ defmodule Lightning.Projects.ProvisionerTest do end test "rejects a job with an adaptor that is not in the registry" do - insert(:adaptor, name: "@openfn/language-common") + ensure_adaptor("@openfn/language-common") %{body: body} = valid_document() @@ -270,9 +273,11 @@ defmodule Lightning.Projects.ProvisionerTest do end describe "import_document/2 adaptor validation" do + setup :isolated_adaptors + test "allows the import when the adaptor is known" do user = insert(:user) - insert(:adaptor, name: "@openfn/language-foo") + ensure_adaptor("@openfn/language-foo") %{body: body} = valid_document() body = diff --git a/test/lightning/setup_utils_test.exs b/test/lightning/setup_utils_test.exs index 5b1fbf110b5..f0145509612 100644 --- a/test/lightning/setup_utils_test.exs +++ b/test/lightning/setup_utils_test.exs @@ -1,6 +1,7 @@ defmodule Lightning.SetupUtilsTest do alias Lightning.Invocation use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers import Swoosh.TestAssertions alias Lightning.{Accounts, Projects, Workflows, Jobs, SetupUtils} @@ -8,6 +9,8 @@ defmodule Lightning.SetupUtilsTest do alias Lightning.Accounts.{User, UserToken} alias Lightning.Credentials.{Credential} + setup :isolated_adaptors + # The demo projects' jobs are built through `Job.changeset/2`, which only # accepts adaptors present in the catalogue. setup do diff --git a/test/lightning/workflows/edge_test.exs b/test/lightning/workflows/edge_test.exs index a161e03351e..3735559f887 100644 --- a/test/lightning/workflows/edge_test.exs +++ b/test/lightning/workflows/edge_test.exs @@ -1,6 +1,8 @@ defmodule Lightning.Workflows.EdgeTest do use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers + alias Lightning.Workflows.Edge describe "condition_label and condition_expression" do @@ -181,6 +183,8 @@ defmodule Lightning.Workflows.EdgeTest do end describe "changeset/2" do + setup :isolated_adaptors + test "valid changeset" do changeset = Edge.changeset(%Edge{source_job_id: Ecto.UUID.generate()}, %{ diff --git a/test/lightning/workflows/job_test.exs b/test/lightning/workflows/job_test.exs index c5861dd6ffb..d144c5d5d42 100644 --- a/test/lightning/workflows/job_test.exs +++ b/test/lightning/workflows/job_test.exs @@ -4,6 +4,7 @@ defmodule Lightning.Workflows.JobTest do alias Lightning.Workflows.Job alias Lightning.Repo + import Lightning.AdaptorTestHelpers import Lightning.Factories # No space in the alphabet on purpose: the changeset trims before it measures, @@ -17,6 +18,8 @@ defmodule Lightning.Workflows.JobTest do end describe "changeset/2" do + setup :isolated_adaptors + test "a malformed id is a changeset error, not an Ecto.ChangeError on save" do # An unsubstituted import placeholder reaching :id (a :binary_id field) # passes cast/3 and would only raise when dumped on insert. validate_uuid @@ -494,8 +497,8 @@ defmodule Lightning.Workflows.JobTest do end test "accepts well-formed, registry-listed adaptor strings" do - insert(:adaptor, name: "@openfn/language-common") - insert(:adaptor, name: "@openfn/language-http") + ensure_adaptor("@openfn/language-common") + ensure_adaptor("@openfn/language-http") [ "@openfn/language-common@latest", @@ -518,7 +521,7 @@ defmodule Lightning.Workflows.JobTest do end test "accepts an adaptor the catalogue listing excludes" do - insert(:adaptor, name: "@openfn/language-collections") + ensure_adaptor("@openfn/language-collections") errors = Job.changeset(%Job{}, %{ @@ -543,14 +546,14 @@ defmodule Lightning.Workflows.JobTest do assert Job.changeset(%Job{}, params) |> errors_on() |> Map.get(:adaptor) == ["adaptor catalogue is not ready yet, try again shortly"] - insert(:adaptor, name: "@openfn/language-http") + ensure_adaptor("@openfn/language-http") assert Job.changeset(%Job{}, params) |> errors_on() |> Map.get(:adaptor) == ["is not a recognised adaptor"] end test "rejects a well-formed adaptor that is not in the registry" do - insert(:adaptor, name: "@openfn/language-http") + ensure_adaptor("@openfn/language-http") # The registry membership check only runs on an otherwise-valid changeset, # so name and body are supplied here. diff --git a/test/lightning/workflows/query_test.exs b/test/lightning/workflows/query_test.exs index 04581621c69..d82be7e9782 100644 --- a/test/lightning/workflows/query_test.exs +++ b/test/lightning/workflows/query_test.exs @@ -5,11 +5,14 @@ defmodule Lightning.Workflows.QueryTest do alias Lightning.Workflows.Workflow alias Lightning.Workflows.WorkflowReleases import Ecto.Query + import Lightning.AdaptorTestHelpers import Lightning.JobsFixtures import Lightning.AccountsFixtures import Lightning.ProjectsFixtures import Lightning.Factories + setup :isolated_adaptors + test "jobs_for/1 with user" do user = user_fixture() project = project_fixture(project_users: [%{user_id: user.id}]) diff --git a/test/lightning/workflows_test.exs b/test/lightning/workflows_test.exs index 13911a962bd..a5d6dab7feb 100644 --- a/test/lightning/workflows_test.exs +++ b/test/lightning/workflows_test.exs @@ -3,6 +3,7 @@ defmodule Lightning.WorkflowsTest do use Mimic import ExUnit.CaptureLog + import Lightning.AdaptorTestHelpers import Lightning.Factories alias Lightning.Auditing.Audit @@ -1469,10 +1470,12 @@ defmodule Lightning.WorkflowsTest do end describe "save_workflow/3 adaptor validation" do + setup :isolated_adaptors + test "allows a job adaptor change to a known adaptor" do user = insert(:user) project = insert(:project) - insert(:adaptor, name: "@openfn/language-common") + ensure_adaptor("@openfn/language-common") changeset = Lightning.Workflows.Workflow.changeset( @@ -1492,7 +1495,7 @@ defmodule Lightning.WorkflowsTest do test "refuses an adaptor the catalogue does not list" do user = insert(:user) project = insert(:project) - insert(:adaptor, name: "@openfn/language-common") + ensure_adaptor("@openfn/language-common") changeset = Lightning.Workflows.Workflow.changeset( diff --git a/test/lightning_web/channels/ai_assistant_channel_test.exs b/test/lightning_web/channels/ai_assistant_channel_test.exs index 6218163d366..1f193cb6e50 100644 --- a/test/lightning_web/channels/ai_assistant_channel_test.exs +++ b/test/lightning_web/channels/ai_assistant_channel_test.exs @@ -3,6 +3,7 @@ defmodule LightningWeb.AiAssistantChannelTest do @moduletag :capture_log import Mox + import Lightning.AdaptorTestHelpers import Lightning.Factories import Lightning.{ @@ -17,6 +18,7 @@ defmodule LightningWeb.AiAssistantChannelTest do alias LightningWeb.AiAssistantChannel setup :verify_on_exit! + setup :isolated_adaptors setup do Process.put(:oban_testing, :manual) diff --git a/test/lightning_web/channels/run_channel_test.exs b/test/lightning_web/channels/run_channel_test.exs index 071f432a05d..480cdf9ff00 100644 --- a/test/lightning_web/channels/run_channel_test.exs +++ b/test/lightning_web/channels/run_channel_test.exs @@ -8,6 +8,7 @@ defmodule LightningWeb.RunChannelTest do alias Lightning.Workflows import Ecto.Query + import Lightning.AdaptorTestHelpers import Lightning.Factories import Lightning.TestUtils import Lightning.TokenHelpers @@ -244,6 +245,7 @@ defmodule LightningWeb.RunChannelTest do end describe "fetching run data" do + setup :isolated_adaptors setup :set_google_credential setup :create_socket_and_run @@ -313,7 +315,7 @@ defmodule LightningWeb.RunChannelTest do test "fetch:plan replies with an error when a job adaptor cannot be resolved", %{project: project} = context do - insert(:adaptor, name: "@openfn/language-readiness-fixture") + seed_ready_catalogue() trigger = build(:trigger, type: :webhook, enabled: true) job = build(:job, adaptor: "@openfn/language-never-published-zzz@latest") diff --git a/test/lightning_web/channels/run_with_options_test.exs b/test/lightning_web/channels/run_with_options_test.exs index ab45b06e83c..a5c797e4a1f 100644 --- a/test/lightning_web/channels/run_with_options_test.exs +++ b/test/lightning_web/channels/run_with_options_test.exs @@ -1,6 +1,7 @@ defmodule LightningWeb.RunWithOptionsTest do use Lightning.DataCase, async: false + import Lightning.AdaptorTestHelpers import Lightning.Factories alias Lightning.Runs @@ -9,10 +10,9 @@ defmodule LightningWeb.RunWithOptionsTest do alias LightningWeb.RunWithOptions describe "rendering a run" do - setup do - cache = Lightning.Adaptors.Supervisor.cache_name(Lightning.Adaptors) - Cachex.clear(cache) + setup :isolated_adaptors + setup do insert(:adaptor, name: "@openfn/language-common", source: :npm, diff --git a/test/lightning_web/channels/workflow_channel_broadcast_test.exs b/test/lightning_web/channels/workflow_channel_broadcast_test.exs index 873cffce595..4857acc2d4c 100644 --- a/test/lightning_web/channels/workflow_channel_broadcast_test.exs +++ b/test/lightning_web/channels/workflow_channel_broadcast_test.exs @@ -8,11 +8,13 @@ defmodule LightningWeb.WorkflowChannelBroadcastTest do """ use LightningWeb.ChannelCase + import Lightning.AdaptorTestHelpers import Lightning.CollaborationHelpers import Lightning.Factories import Mox setup :verify_on_exit! + setup :isolated_adaptors setup do Mox.stub(Lightning.MockConfig, :check_flag?, fn diff --git a/test/lightning_web/channels/workflow_channel_test.exs b/test/lightning_web/channels/workflow_channel_test.exs index ec6c957b8e9..df0d2dafb5b 100644 --- a/test/lightning_web/channels/workflow_channel_test.exs +++ b/test/lightning_web/channels/workflow_channel_test.exs @@ -9,6 +9,7 @@ defmodule LightningWeb.WorkflowChannelTest do import ExUnit.CaptureLog setup :verify_on_exit! + setup :isolated_adaptors setup do Mox.stub(Lightning.MockConfig, :check_flag?, fn @@ -2704,9 +2705,6 @@ defmodule LightningWeb.WorkflowChannelTest do describe "request_adaptors and request_credentials" do setup do - cache = Lightning.Adaptors.Supervisor.cache_name(Lightning.Adaptors) - Cachex.clear(cache) - insert(:adaptor, name: "@openfn/language-salesforce", source: :npm) insert(:adaptor, name: "@openfn/language-http", source: :npm) :ok @@ -3364,10 +3362,11 @@ defmodule LightningWeb.WorkflowChannelTest do test "handles an adaptor catalogue that is not ready", %{ socket: socket, - workflow: workflow + workflow: workflow, + sup: _sup } do - # Global mode: the refresh runs in a Task owned by the production - # Scheduler. + # Global mode: the refresh runs in a Task owned by the isolated + # instance's Scheduler. Lightning.Adaptors.Catalogue.delete_all_for_source(:npm) Mox.set_mox_global(Lightning.Adaptors.StrategyMock) @@ -4746,13 +4745,13 @@ defmodule LightningWeb.WorkflowChannelTest do describe "PubSub subscription and adaptors broadcasting" do test "forwards adaptors_updated envelope from client topic to socket", %{ - socket: _socket + sup: sup } do payload = %{adaptors: [%{name: "a"}]} Phoenix.PubSub.broadcast( Lightning.PubSub, - Lightning.Adaptors.Supervisor.client_topic(Lightning.Adaptors), + Lightning.Adaptors.Supervisor.client_topic(sup), %{event: "adaptors_updated", payload: payload} ) @@ -4778,11 +4777,12 @@ defmodule LightningWeb.WorkflowChannelTest do } end - test "does not push adaptors_updated for unrelated events on client topic" do + test "does not push adaptors_updated for unrelated events on client topic", + %{sup: sup} do capture_log(fn -> Phoenix.PubSub.broadcast( Lightning.PubSub, - Lightning.Adaptors.Supervisor.client_topic(Lightning.Adaptors), + Lightning.Adaptors.Supervisor.client_topic(sup), %{event: "something_else", payload: %{}} ) diff --git a/test/lightning_web/live/credential_live_test.exs b/test/lightning_web/live/credential_live_test.exs index 227d1a7d0d3..b2c9c80da78 100644 --- a/test/lightning_web/live/credential_live_test.exs +++ b/test/lightning_web/live/credential_live_test.exs @@ -4,6 +4,7 @@ defmodule LightningWeb.CredentialLiveTest do import Phoenix.LiveViewTest import LightningWeb.CredentialLiveHelpers + import Lightning.AdaptorTestHelpers import Lightning.Factories import Ecto.Query @@ -42,6 +43,7 @@ defmodule LightningWeb.CredentialLiveTest do setup :register_and_log_in_user setup :create_project_for_current_user + setup :isolated_adaptors setup do Lightning.AdaptorTestHelpers.seed_all_credential_schemas() diff --git a/test/lightning_web/live/project_live_test.exs b/test/lightning_web/live/project_live_test.exs index bea72e682af..7e183670167 100644 --- a/test/lightning_web/live/project_live_test.exs +++ b/test/lightning_web/live/project_live_test.exs @@ -5,6 +5,7 @@ defmodule LightningWeb.ProjectLiveTest do import Phoenix.Component import Lightning.ProjectsFixtures import Lightning.AccountsFixtures + import Lightning.AdaptorTestHelpers import Lightning.Factories import LightningWeb.CredentialLiveHelpers @@ -879,6 +880,7 @@ defmodule LightningWeb.ProjectLiveTest do describe "projects settings page" do setup :register_and_log_in_user setup :create_project_for_current_user + setup :isolated_adaptors setup do Lightning.AdaptorTestHelpers.seed_credential_schema("http") diff --git a/test/lightning_web/live/workflow_live/collaborate_test.exs b/test/lightning_web/live/workflow_live/collaborate_test.exs index 1a16c64c42e..f2c2746ef7a 100644 --- a/test/lightning_web/live/workflow_live/collaborate_test.exs +++ b/test/lightning_web/live/workflow_live/collaborate_test.exs @@ -1,6 +1,7 @@ defmodule LightningWeb.WorkflowLive.CollaborateTest do use LightningWeb.ConnCase, async: false + import Lightning.AdaptorTestHelpers import Lightning.Factories import Lightning.WorkflowsFixtures import Phoenix.LiveViewTest @@ -1028,6 +1029,8 @@ defmodule LightningWeb.WorkflowLive.CollaborateTest do end describe "credential modal interactions" do + setup :isolated_adaptors + setup do Lightning.AdaptorTestHelpers.seed_credential_schema("http") :ok diff --git a/test/mix/tasks/lightning.adaptors.import_test.exs b/test/mix/tasks/lightning.adaptors.import_test.exs index 88b1be9fe35..df20825e589 100644 --- a/test/mix/tasks/lightning.adaptors.import_test.exs +++ b/test/mix/tasks/lightning.adaptors.import_test.exs @@ -2,12 +2,15 @@ defmodule Mix.Tasks.Lightning.Adaptors.ImportTest do use Lightning.DataCase import ExUnit.CaptureIO + import Lightning.AdaptorTestHelpers alias Lightning.Adaptors.Catalogue alias Mix.Tasks.Lightning.Adaptors.Import @moduletag :tmp_dir + setup :isolated_adaptors + defp write_snapshot(tmp_dir, records) do path = Path.join(tmp_dir, "snapshot.json") File.write!(path, Jason.encode_to_iodata!(records)) diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex index 7ec94d79a63..e5edce68c35 100644 --- a/test/support/adaptor_test_helpers.ex +++ b/test/support/adaptor_test_helpers.ex @@ -1,22 +1,71 @@ defmodule Lightning.AdaptorTestHelpers do @moduledoc """ - Seeds `Lightning.Adaptors.Catalogue` rows and manages the production - `Lightning.Adaptors` cache for tests that read through it. - - The production cache outlives the SQL sandbox, so a test that seeds - rows and reads them through the cache must clear it first. + Starts isolated `Lightning.Adaptors.Supervisor` instances for tests (see + `isolated_adaptors/1`) and seeds `Lightning.Adaptors.Catalogue` rows and + their cache into whichever instance is current. + + The production `Lightning.Adaptors` cache outlives the SQL sandbox, so + every seeding helper except `isolated_adaptors/1` refuses to run without + it (`ensure_isolated!/0`) — otherwise a seeded row's cache fill leaks into + later tests. """ + import Eventually import Lightning.Factories + alias Lightning.Adaptors.Config alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + @doc """ + Starts an isolated `Lightning.Adaptors.Supervisor` instance under a + fresh name, backed by `Lightning.Adaptors.StrategyMock`, and stubs + `Lightning.Adaptors.Config.default_instance/0` to it, for the calling + test process and any process it starts (`Task`, `start_supervised!`, + ...) via `$callers`. + + For an `async: true` module, the stub only reaches processes in that + `$callers` chain — a process spawned outside it, or already running + before this setup, still resolves `default_instance/0` to the real + global `Lightning.Adaptors` instance. An `async: false` module gets + Mimic's global mode instead, which reaches every process in the VM. + + Use as `setup :isolated_adaptors`. Returns `%{sup: sup}`. + """ + @spec isolated_adaptors(map()) :: %{sup: atom()} + def isolated_adaptors(context) do + sup = :"isolated_adaptors_#{System.unique_integer([:positive])}" + + ExUnit.Callbacks.start_supervised!( + Supervisor.child_spec( + {AdaptorsSupervisor, + name: sup, strategy: Lightning.Adaptors.StrategyMock}, + id: sup + ) + ) + + Mimic.set_mimic_from_context(context) + Mimic.stub(Config, :default_instance, fn -> sup end) + + await_scheduler(sup) + + %{sup: sup} + end + + # The instance's Scheduler only registers once HighlanderPG holds its + # advisory lock, which it acquires after `start_supervised!` returns. + defp await_scheduler(sup) do + {:global, gname} = AdaptorsSupervisor.global_scheduler_name(sup) + assert_eventually(is_pid(:global.whereis_name(gname)), 2000) + end + @doc """ Seeds a throwaway adaptor row so the catalogue counts as loaded and saves do not wait on the production Scheduler. """ @spec seed_ready_catalogue() :: :ok def seed_ready_catalogue do + ensure_isolated!() + {:ok, _} = Lightning.Adaptors.Catalogue.upsert_adaptor(%{ name: "@openfn/language-readiness-fixture", @@ -45,15 +94,38 @@ defmodule Lightning.AdaptorTestHelpers do :ok end + @doc """ + Raises unless the calling test has opted into an isolated instance via + `setup :isolated_adaptors`. Without it, a seeded row's cache fill lands in + the shared `Lightning.Adaptors` cache and outlives the test's DB rollback. + """ + @spec ensure_isolated!() :: :ok + def ensure_isolated! do + if Config.default_instance() == Lightning.Adaptors do + raise """ + This seeds the adaptor catalogue against the global Lightning.Adaptors \ + instance. Its cache fill outlives this test's DB rollback and leaks \ + into later tests. + + Add `import Lightning.AdaptorTestHelpers` and `setup :isolated_adaptors` \ + to this test module. + """ + end + + :ok + end + @doc """ Seeds the catalogue row an adaptor spec needs to pass `Lightning.Workflows.Job` validation, unless it is already there. """ @spec ensure_adaptor(String.t()) :: :ok def ensure_adaptor(spec) when is_binary(spec) do + ensure_isolated!() + case Lightning.Adaptors.parse_spec(spec) do {name, _version} when is_binary(name) -> - source = AdaptorsSupervisor.source(Lightning.Adaptors) + source = AdaptorsSupervisor.source(Config.default_instance()) if is_nil(Lightning.Adaptors.Catalogue.get_adaptor(name, source)), do: insert(:adaptor, name: name) @@ -72,6 +144,8 @@ defmodule Lightning.AdaptorTestHelpers do @spec seed_credential_schema(String.t()) :: Lightning.Adaptors.Catalogue.Adaptor.t() def seed_credential_schema(short_name) when is_binary(short_name) do + ensure_isolated!() + # Raw JSON binary, not a decoded map: `Credentials.Schema.new/2` decodes # it with ordered objects. schema_body = @@ -83,8 +157,8 @@ defmodule Lightning.AdaptorTestHelpers do # Cachex fills run in its Courier process, which cannot see the sandbox # connection, so populate the cache directly. - cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) - source = AdaptorsSupervisor.source(Lightning.Adaptors) + cache = AdaptorsSupervisor.cache_name(Config.default_instance()) + source = AdaptorsSupervisor.source(Config.default_instance()) Cachex.put(cache, {:schema, short_name, source}, {:ok, schema_body}) row @@ -95,6 +169,8 @@ defmodule Lightning.AdaptorTestHelpers do """ @spec seed_all_credential_schemas() :: :ok def seed_all_credential_schemas do + ensure_isolated!() + metas = Path.wildcard("test/fixtures/schemas/*.json") |> Enum.reject(fn path -> File.stat!(path).size == 0 end) @@ -115,8 +191,8 @@ defmodule Lightning.AdaptorTestHelpers do } end) - cache = AdaptorsSupervisor.cache_name(Lightning.Adaptors) - source = AdaptorsSupervisor.source(Lightning.Adaptors) + cache = AdaptorsSupervisor.cache_name(Config.default_instance()) + source = AdaptorsSupervisor.source(Config.default_instance()) Cachex.put(cache, {:packages, source}, {:ok, metas}) :ok @@ -129,6 +205,8 @@ defmodule Lightning.AdaptorTestHelpers do Lightning.Adaptors.Catalogue.Adaptor.t() def seed_adaptor_package(name, latest_version) when is_binary(name) and is_binary(latest_version) do + ensure_isolated!() + {:ok, row} = Lightning.Adaptors.Catalogue.upsert_adaptor(%{ name: name, diff --git a/test/support/factories.ex b/test/support/factories.ex index 0d363445b21..49a4855c260 100644 --- a/test/support/factories.ex +++ b/test/support/factories.ex @@ -5,6 +5,8 @@ defmodule Lightning.Factories do alias Lightning.Workflows.Snapshot def adaptor_factory do + Lightning.AdaptorTestHelpers.ensure_isolated!() + %Lightning.Adaptors.Catalogue.Adaptor{ name: sequence(:adaptor_name, &"@openfn/language-test-#{&1}"), source: :npm, From 128dd56b141987e158a52de9e57b8f3017d9363f Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Fri, 4 Sep 2026 16:32:31 +0200 Subject: [PATCH 09/37] Condense the adaptors changelog entry, fix ExDoc warnings - ExDoc warnings introduced on this branch fixed - Adaptors changelog entries condensed into one succinct note --- CHANGELOG.md | 5 +++++ lib/lightning/adaptor_service.ex | 5 ++++- lib/lightning/adaptors/npm.ex | 27 +++++++++++++++------------ 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39f5e78fe34..2a8ea7a438c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,11 @@ and this project adheres to does not normalise anything today, but #4577 adds it on every name, so the runtime moves first. +- Lightning now keeps its own adaptor registry instead of fetching the list from + npm at startup, so new adaptors and versions show up without a rebuild or + redeploy. See [ADAPTORS.md](ADAPTORS.md). + [#4801](https://github.com/OpenFn/lightning/pull/4801) + ### Removed - The AI assistant's "Send code" tickbox. The assistant reads your workflow to diff --git a/lib/lightning/adaptor_service.ex b/lib/lightning/adaptor_service.ex index 8372311bb54..86974a560b6 100644 --- a/lib/lightning/adaptor_service.ex +++ b/lib/lightning/adaptor_service.ex @@ -63,7 +63,10 @@ defmodule Lightning.AdaptorService do require Logger defmodule InstalledAdaptor do - @moduledoc false + @moduledoc """ + An adaptor installed on disk, as returned by `Lightning.AdaptorService.find_adaptor/2` + and `Lightning.AdaptorService.install/2`. + """ @type install_status :: :present | :installing @type t :: %__MODULE__{ diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index 156b35715c9..9964f4fe664 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -5,18 +5,21 @@ defmodule Lightning.Adaptors.NPM do Implements the four `Lightning.Adaptors.Strategy` callbacks: - * `c:list_adaptors/0` — single search-API call returning - `name + latest_version` for every `@openfn/language-*` package. - * `c:fetch_adaptor/1` — packument fetch + per-version decode and - latest-version schema retrieval via jsDelivr. Icon fields are - **not** stamped here; the Scheduler joins them on after a bulk - `c:fetch_icons/1` pass. - * `c:fetch_icon/2` — single icon raw GET against - `raw.githubusercontent.com`, used by the Store's rare lazy-miss - fallback. - * `c:fetch_icons/1` — bulk fan-out over the search listing, one - HTTP request per `(name, shape)`. Threads `:prior_etags` from - the caller down into the per-request `If-None-Match` headers. + * `c:Lightning.Adaptors.Strategy.list_adaptors/0` — single search-API + call returning `name + latest_version` for every + `@openfn/language-*` package. + * `c:Lightning.Adaptors.Strategy.fetch_adaptor/1` — packument fetch + + per-version decode and latest-version schema retrieval via + jsDelivr. Icon fields are **not** stamped here; the Scheduler + joins them on after a bulk + `c:Lightning.Adaptors.Strategy.fetch_icons/1` pass. + * `c:Lightning.Adaptors.Strategy.fetch_icon/2` — single icon raw GET + against `raw.githubusercontent.com`, used by the Store's rare + lazy-miss fallback. + * `c:Lightning.Adaptors.Strategy.fetch_icons/1` — bulk fan-out over + the search listing, one HTTP request per `(name, shape)`. Threads + `:prior_etags` from the caller down into the per-request + `If-None-Match` headers. ## HTTP From 686c29ae6c2cb30f52d8966a44d79e0eabb28002 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Fri, 4 Sep 2026 17:02:44 +0200 Subject: [PATCH 10/37] Make npm listing authoritative, hide deprecated adaptors from pickers - Feature-specific adaptors rule docs retired, durable content folded into code comments and guidelines - Stale fetch/2 call in the metadata service test fixed - Legacy short adaptor names resolved to full package names - npm registry listing made authoritative rather than search-ranked - Deprecated adaptors and versions stop appearing as new choices in pickers - Stale/inaccurate comments and docs surfaced during registry work fixed --- .../guidelines/testable-supervision-trees.md | 15 +- .claude/rules/adaptors-docs.md | 52 ------ .claude/rules/adaptors-otp.md | 52 ------ CHANGELOG.md | 5 - .../components/ConfigureAdaptorModal.tsx | 7 +- .../components/ConfigureAdaptorModal.test.tsx | 8 +- config/test.exs | 4 + lib/lightning/adaptors.ex | 24 +++ lib/lightning/adaptors/catalogue.ex | 35 +++-- lib/lightning/adaptors/node_monitor.ex | 17 +- lib/lightning/adaptors/npm.ex | 29 ++-- lib/lightning/adaptors/npm/github.ex | 8 +- lib/lightning/adaptors/npm/registry.ex | 124 ++++++++++++--- lib/lightning/adaptors/package_name.ex | 24 ++- lib/lightning/adaptors/scheduler.ex | 7 +- lib/lightning/adaptors/supervisor.ex | 2 + lib/lightning/ai_assistant/ai_assistant.ex | 3 +- lib/lightning/application.ex | 11 ++ lib/lightning/channels/destination_auth.ex | 8 +- lib/lightning/collaboration/session.ex | 1 - lib/lightning/config/bootstrap.ex | 10 +- lib/lightning/credentials.ex | 44 +++++- lib/lightning/credentials/credential.ex | 8 + .../credentials/schema_reconciler.ex | 91 +++++++++++ lib/lightning/setup_utils.ex | 2 +- .../channels/workflow_channel.ex | 7 +- .../live/components/data_tables.ex | 3 +- .../credential_form_component.ex | 26 +-- .../live/maintenance_live/index.ex | 6 +- lib/mix/tasks/lightning.adaptors.snapshot.ex | 8 +- ...0260907112954_widen_credentials_schema.exs | 9 ++ test/lightning/adaptors/catalogue_test.exs | 79 +++++++++- .../adaptors/isolated_adaptors_test.exs | 5 +- test/lightning/adaptors/npm/registry_test.exs | 148 +++++++++++++++++- test/lightning/adaptors/npm_test.exs | 11 +- test/lightning/adaptors/package_name_test.exs | 24 +++ test/lightning/adaptors/scheduler_test.exs | 6 +- test/lightning/adaptors_test.exs | 34 ++++ .../channels/destination_auth_test.exs | 20 +-- .../credentials/schema_reconciler_test.exs | 74 +++++++++ test/lightning/credentials_test.exs | 40 +++++ test/lightning/metadata_service_test.exs | 13 +- .../live/credential_live_test.exs | 50 ++++-- test/lightning_web/live/project_live_test.exs | 6 +- .../plugs/channel_proxy_plug_test.exs | 62 ++++++-- .../lightning.adaptors.snapshot_test.exs | 8 + test/support/adaptor_test_helpers.ex | 10 +- test/support/credential_live_helpers.ex | 2 +- tooling/adaptor_cache/README.md | 26 +-- 49 files changed, 980 insertions(+), 288 deletions(-) delete mode 100644 .claude/rules/adaptors-docs.md delete mode 100644 .claude/rules/adaptors-otp.md create mode 100644 lib/lightning/credentials/schema_reconciler.ex create mode 100644 priv/repo/migrations/20260907112954_widen_credentials_schema.exs create mode 100644 test/lightning/credentials/schema_reconciler_test.exs diff --git a/.claude/guidelines/testable-supervision-trees.md b/.claude/guidelines/testable-supervision-trees.md index cd525c9acd6..d6a60beee25 100644 --- a/.claude/guidelines/testable-supervision-trees.md +++ b/.claude/guidelines/testable-supervision-trees.md @@ -157,6 +157,15 @@ The tell to watch for is global storage smuggling a value past a structural boundary that was already open two lines away and already carrying its siblings across. +`Lightning.Adaptors.Supervisor` shows both sides. `strategy` and `source` go +into `:persistent_term` keyed per instance (`lib/lightning/adaptors/supervisor.ex:42-45`), +which is the acceptable shape for the stateless `Store` functions: a caller +holding only the instance name has nowhere else to read boot-fixed config from. +It is the tell for `Scheduler`, whose child spec already carries `cache`, +`tasks` and `source_topic` (`supervisor.ex:58-65`) but not `strategy`, so the +process re-reads it from the global store on every refresh (`scheduler.ex:259`, +`:271`, `:313`). New children take config from the child spec, as `lock_key` does. + ### Scoping mocks without going global `Mox.allow/3` and `Ecto.Adapters.SQL.Sandbox.allow/3` are one concept with one @@ -168,10 +177,8 @@ Lightning's collaboration suite is the in-repo example, at and each per-test mock to a directly-started collaboration process, guarding every `Mox.allow` so a test that never stubbed a given mock is unaffected. Its comment records the migration: `set_mox_global` previously made every mock visible -cross-process, and under private Mox they are allowed explicitly. Private Mox via -`set_mox_from_context` is the house default; the two remaining `set_mox_global` -call sites are `test/lightning_web/channels/workflow_channel_test.exs:18` and -`workflow_channel_broadcast_test.exs:24`. +cross-process, and under private Mox they are allowed explicitly. Private Mox via `set_mox_from_context` is the house default; +`grep -rn set_mox_global test/` lists the exceptions. When the pid does not exist yet at setup time, because of leader election or a lazy start, `Mox.allow/3` accepts a `(-> pid())` resolved lazily at first dispatch diff --git a/.claude/rules/adaptors-docs.md b/.claude/rules/adaptors-docs.md deleted file mode 100644 index fe9c98d96da..00000000000 --- a/.claude/rules/adaptors-docs.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -paths: - - "lib/lightning/adaptors.ex" - - "lib/lightning/adaptors/**" - - "lib/lightning_web/controllers/adaptor_icon_controller.ex" - - "lib/mix/tasks/lightning.adaptors.refresh.ex" - - "lib/mix/tasks/lightning.adaptors.import.ex" - - "lib/mix/tasks/lightning.adaptors.dump.ex" - - "lib/mix/tasks/lightning.adaptors.snapshot.ex" - - "test/lightning/adaptors_test.exs" - - "test/lightning/adaptors/**" - - "test/mix/tasks/lightning.adaptors.*.exs" - - "assets/js/collaborative-editor/stores/createAdaptorStore.ts" - - "assets/js/collaborative-editor/types/adaptor.ts" - - ".context/adaptors/**" ---- - -# Adaptors: which documents to trust - -Most of `.context/adaptors/` is archaeology from designs that were abandoned before they -shipped. Grep will find it and it reads convincingly. Everything at the top level of that -folder is current, and there are six things: - -- `README.md` — the entry point, and the shortest thing to read. -- `ATLAS.md` — the architecture in eight diagrams, stamped with the commit it describes. - Start here to understand the shape of the subsystem. -- `REWRITE-2026-05.md` — the canonical spec. Per-callback contracts and the reasoning behind - each decision. Grep it, don't read it end to end. -- `BEHAVIOURS.md` — the subsystem's promises, one section per behaviour, each stating what - holds today and what observation would settle it. Read it before treating something as a - gap nobody noticed. -- `07-channel-live-update-findings-2026-06-03.md` — two decisions still open, still blocking. -- `NOTES.md` — a running log of open questions and irregularities hit while working on the - subsystem, newest entry on top. Dated entries, none of them acted on yet. Read it before - concluding you have found a new bug. - -Everything under `.context/adaptors/archive/` is superseded, and every file there carries an -ARCHIVED banner saying why. Do not cite it, follow it, or use it to answer a question about -how the subsystem works. Two traps worth naming: `archive/ARCHITECTURE.md` diagrams the -abandoned PR #4473 design (blob table plus Oban) in convincing detail and shares almost -nothing with what shipped, and `archive/NOTES.md` is a dead namesake of the live `NOTES.md` -above — check which one you opened. - -Live status is the PR, not the folder: `gh pr view 4801 --json body -q .body`. Don't -reconstruct that checklist anywhere else, and don't infer completion state from the archived -phase-A/phase-B PRDs — both phases shipped, so those describe code that already exists. - -If you change the subsystem's shape, update `ATLAS.md` and move its commit stamp. A stale -architecture diagram is worse than none, because it misleads with confidence. - -Process and naming conventions for this subsystem are a separate rule: -`.claude/rules/adaptors-otp.md`. diff --git a/.claude/rules/adaptors-otp.md b/.claude/rules/adaptors-otp.md deleted file mode 100644 index 3dd5895360c..00000000000 --- a/.claude/rules/adaptors-otp.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -paths: - - "lib/lightning/adaptors.ex" - - "lib/lightning/adaptors/**/*.ex" - - "test/lightning/adaptors_test.exs" - - "test/lightning/adaptors/**/*.exs" ---- - -# Adaptors: naming and dependency injection - -Every process in this subsystem derives its name from the single `:name` opt passed -to `Lightning.Adaptors.Supervisor`. Nothing is hardcoded, which is what lets the -integration suite run isolated instances in one BEAM under `async: true`. Adding a -process that breaks this forces the whole suite serial. - -When adding or changing a process here: - -- Take `:name` from opts (`Keyword.fetch!(opts, :name)`) and derive any child, - cache, topic or lock name from it. Follow the helpers at - `lib/lightning/adaptors/supervisor.ex:116-173`. -- Add it to the fixed child list in `init/1` (`supervisor.ex:67-85`) with its - collaborators passed in the child spec. Do not add a `Registry`: the child set is - fixed and the registered atom already addresses it. -- Public functions that talk to a running process lead with the server ref, - defaulted: `def refresh(sup \\ Config.default_instance(), name)`. `start_link` - takes `name:` in trailing opts. -- `Lightning.Adaptors.Config.default_instance/0` is what production code and - every public `Lightning.Adaptors` function default their `sup` argument - through, not a hardcoded module attribute. Tests get a private instance with - `setup :isolated_adaptors` (from `Lightning.AdaptorTestHelpers`), which - starts a fresh supervisor and stubs `default_instance/0` to it, instead of - touching the global instance. For an `async: true` module, that stub only - reaches processes reachable via `$callers` (`Task`, `start_supervised!`, - ...); a process spawned outside that chain, or already running, still - sees the real global instance. An `async: false` module gets Mimic's - global mode instead, which reaches every process in the VM. -- The `Scheduler` is a cluster singleton behind `HighlanderPG` and registers under - `global_scheduler_name/1` (`supervisor.ex:149`). `Process.whereis` will not find - it. - -Known wart, do not copy it: `strategy` and `source` are published to -`:persistent_term` in `init/1` (`supervisor.ex:40-43`) and re-read at call time by -`Scheduler` (`scheduler.ex:259`, `:271`, `:313`) and `Store`. New code should -take them from process state or the child spec instead. `Scheduler` already does -this correctly for `source` (`scheduler.ex:113`, `:132`). - -In tests, prefer `Mox.allow(StrategyMock, self(), pid)` and keep `async: true`, as -`test/lightning/adaptors/store_test.exs:135` does. `set_mox_global` costs the file -its async, and is only justified where the hop graph is genuinely dynamic, as in -`highlander_integration_test.exs:27`. - -Full reasoning: `.claude/guidelines/testable-supervision-trees.md`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a8ea7a438c..39f5e78fe34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,11 +67,6 @@ and this project adheres to does not normalise anything today, but #4577 adds it on every name, so the runtime moves first. -- Lightning now keeps its own adaptor registry instead of fetching the list from - npm at startup, so new adaptors and versions show up without a rebuild or - redeploy. See [ADAPTORS.md](ADAPTORS.md). - [#4801](https://github.com/OpenFn/lightning/pull/4801) - ### Removed - The AI assistant's "Send code" tickbox. The assistant reads your workflow to diff --git a/assets/js/collaborative-editor/components/ConfigureAdaptorModal.tsx b/assets/js/collaborative-editor/components/ConfigureAdaptorModal.tsx index 46cbdc2a673..df8066904a9 100644 --- a/assets/js/collaborative-editor/components/ConfigureAdaptorModal.tsx +++ b/assets/js/collaborative-editor/components/ConfigureAdaptorModal.tsx @@ -285,6 +285,7 @@ export function ConfigureAdaptorModal({ // Filter credentials into sections const credentialSections = useMemo(() => { const adaptorName = extractAdaptorName(currentAdaptor); + const packageName = extractPackageName(currentAdaptor); if (!adaptorName) { return { schemaMatched: [], @@ -300,7 +301,7 @@ export function ConfigureAdaptorModal({ const schemaMatched: CredentialWithType[] = projectCredentials .filter(c => { // Exact schema match - if (c.schema === adaptorName) return true; + if (c.schema === packageName) return true; // For HTTP adaptor, all OAuth credentials are considered matching if (adaptorName === 'http' && c.schema === 'oauth') return true; @@ -324,7 +325,9 @@ export function ConfigureAdaptorModal({ const universal: CredentialWithType[] = projectCredentials .filter(c => { const isUniversal = - c.schema === 'http' || c.schema === 'raw' || c.schema === 'oauth'; + c.schema === '@openfn/language-http' || + c.schema === 'raw' || + c.schema === 'oauth'; const alreadyInSchemaMatched = schemaMatched.some( matched => matched.id === c.id ); diff --git a/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx b/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx index 37614f3be7b..da53b1565a4 100644 --- a/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx +++ b/assets/test/collaborative-editor/components/ConfigureAdaptorModal.test.tsx @@ -60,7 +60,7 @@ const mockProjectCredentials: ProjectCredential[] = [ id: 'cred-1', project_credential_id: 'proj-cred-1', name: 'Salesforce Production', - schema: 'salesforce', + schema: '@openfn/language-salesforce', external_id: 'ext-1', inserted_at: '2024-01-01T00:00:00Z', updated_at: '2024-01-01T00:00:00Z', @@ -71,7 +71,7 @@ const mockProjectCredentials: ProjectCredential[] = [ id: 'cred-2', project_credential_id: 'proj-cred-2', name: 'Salesforce Testing', - schema: 'salesforce', + schema: '@openfn/language-salesforce', external_id: 'ext-2', inserted_at: '2024-01-01T00:00:00Z', updated_at: '2024-01-01T00:00:00Z', @@ -82,7 +82,7 @@ const mockProjectCredentials: ProjectCredential[] = [ id: 'cred-3', project_credential_id: 'proj-cred-3', name: 'HTTP API Key', - schema: 'http', + schema: '@openfn/language-http', external_id: 'ext-3', inserted_at: '2024-01-01T00:00:00Z', updated_at: '2024-01-01T00:00:00Z', @@ -1344,7 +1344,7 @@ describe('ConfigureAdaptorModal', () => { id: 'cred-other', project_credential_id: 'proj-cred-other', name: 'Other User Credential', - schema: 'salesforce', + schema: '@openfn/language-salesforce', external_id: 'ext-other', inserted_at: '2024-01-01T00:00:00Z', updated_at: '2024-01-01T00:00:00Z', diff --git a/config/test.exs b/config/test.exs index 574a8ab96d4..43c8a014885 100644 --- a/config/test.exs +++ b/config/test.exs @@ -110,6 +110,10 @@ config :lightning, Lightning.Adaptors, strategy: Lightning.Adaptors.StrategyMock, refresh_interval: 0 +# The reconciler runs against the shared production catalogue table, which +# tests seed freely; each test that needs it starts its own named instance. +config :lightning, Lightning.Credentials.SchemaReconciler, enabled: false + # `Config.source_for/1` only maps the two real strategies; the mock has to # declare its catalogue source like any other third-party strategy would. config :lightning, Lightning.Adaptors.StrategyMock, source: :npm diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index 465ef38c8c7..e0375f1e250 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -118,6 +118,30 @@ defmodule Lightning.Adaptors do @spec schema(atom(), String.t()) :: {:ok, String.t()} | {:error, term()} def schema(sup \\ Config.default_instance(), pkg), do: Store.schema(sup, pkg) + @doc """ + Resolves a possibly-legacy short adaptor name (e.g. `"http"`) to its full + npm package name (`"@openfn/language-http"`), if the full name resolves in + the catalogue. Returns `name` unchanged if it already resolves, or if + neither form does. + + `"raw"` and `"oauth"` are sentinels, not adaptor names, and are returned + unchanged without consulting the catalogue. + """ + @spec resolve_name(atom(), String.t()) :: String.t() + def resolve_name(sup \\ Config.default_instance(), name) + + def resolve_name(_sup, name) when name in ["raw", "oauth"], do: name + + def resolve_name(sup, name) do + full = PackageName.full_name(name) + + cond do + get_adaptor(sup, name) -> name + not String.starts_with?(name, "@") and get_adaptor(sup, full) -> full + true -> name + end + end + @doc """ Returns the on-disk path of the adaptor's `:square` or `:rectangle` icon, fetching it on the first request. diff --git a/lib/lightning/adaptors/catalogue.ex b/lib/lightning/adaptors/catalogue.ex index fceced9fe22..8a349aed3db 100644 --- a/lib/lightning/adaptors/catalogue.ex +++ b/lib/lightning/adaptors/catalogue.ex @@ -53,16 +53,16 @@ defmodule Lightning.Adaptors.Catalogue do @openfn/language-collections) @doc """ - Picker-facing lean projection for a source. Avoids the heavy JSONB - columns (`schema_data`, `dependencies`, `peer_dependencies`). + Picker-facing lean projection for a source. Avoids the heavy + `schema_data` JSONB column and skips the version join entirely. - Excludes the packages listed in `@excluded_names`. + Excludes the packages listed in `@excluded_names` and any deprecated + adaptor. """ @spec list_package_metas(source()) :: [package_meta()] def list_package_metas(source) do Repo.all( - from a in Adaptor, - where: a.source == ^source and a.name not in ^@excluded_names, + from a in active_adaptors(source), select: %{ name: a.name, latest_version: a.latest_version, @@ -78,8 +78,9 @@ defmodule Lightning.Adaptors.Catalogue do end @doc """ - Full structs for a source. Rare — used by debug tools and admin - views. Picker traffic goes through `list_package_metas/1`. + Full structs for a source. Heavier than `list_package_metas/1`, which is + what picker traffic uses — this one is for callers that need the whole + row, like the Scheduler's diffing and the dump/seed tooling. """ @spec list_adaptors(source()) :: [Adaptor.t()] def list_adaptors(source) do @@ -276,14 +277,14 @@ defmodule Lightning.Adaptors.Catalogue do Full catalogue projection for a source: every adaptor's `name`, `latest_version`, `repository`, icon fields, and full version list. - Excludes the packages listed in `@excluded_names`. + Excludes the packages listed in `@excluded_names`, any deprecated + adaptor, and — for an otherwise-listed adaptor — any deprecated version. """ @spec catalogue(source()) :: [catalogue_entry()] def catalogue(source) do adaptors = Repo.all( - from a in Adaptor, - where: a.source == ^source and a.name not in ^@excluded_names, + from a in active_adaptors(source), order_by: [asc: a.name], select: %{ name: a.name, @@ -299,9 +300,9 @@ defmodule Lightning.Adaptors.Catalogue do versions_by_name = Repo.all( from v in AdaptorVersion, - join: a in Adaptor, + join: a in subquery(active_adaptors(source)), on: v.adaptor_id == a.id, - where: a.source == ^source and a.name not in ^@excluded_names, + where: v.deprecated == false, order_by: [asc: v.inserted_at, asc: v.version], select: {a.name, v.version} ) @@ -337,6 +338,16 @@ defmodule Lightning.Adaptors.Catalogue do ) end + # The adaptor-level predicate shared by every listing query: never a + # hard-excluded name, never a deprecated adaptor. Resolve paths + # (`get_adaptor/2`, `list_versions/2`) deliberately skip this. + defp active_adaptors(source) do + from a in Adaptor, + where: + a.source == ^source and a.name not in ^@excluded_names and + a.deprecated == false + end + defp upsert_adaptor_row(repo, nil, attrs, _now) do %Adaptor{} |> Adaptor.changeset(attrs) diff --git a/lib/lightning/adaptors/node_monitor.ex b/lib/lightning/adaptors/node_monitor.ex index 3ecc0d3ab8f..1d7f935aed1 100644 --- a/lib/lightning/adaptors/node_monitor.ex +++ b/lib/lightning/adaptors/node_monitor.ex @@ -2,12 +2,15 @@ defmodule Lightning.Adaptors.NodeMonitor do @moduledoc """ Partition-recovery companion to `Lightning.Adaptors.Invalidator`. - On `:nodeup`, re-warms the Cachex table from Postgres so a reconnecting - peer never serves stale data until the 24-hour TTL expires. Steady-state + Cache entries have no TTL, so a node that misses a `:changed` broadcast + while partitioned would otherwise serve it forever. On `:nodeup`, + re-warms the Cachex table from Postgres to close that gap. Steady-state invalidation belongs to `Lightning.Adaptors.Invalidator`. - `:nodedown` is a deliberate no-op. The worst case on a silent departure is - one stale-URL redirect per client, backstopped by 302-on-stale-sha. + `:nodedown` is a deliberate no-op: there's nothing to invalidate on this + end when the connection drops. The worst case while partitioned is a + stale icon URL, which `LightningWeb.AdaptorIconController`'s + 302-on-stale-sha handles regardless of which node serves the request. """ use GenServer @@ -18,7 +21,7 @@ defmodule Lightning.Adaptors.NodeMonitor do Start a NodeMonitor for the given supervisor instance. Required opts: - * `:name` — registered GenServer name (§6.11 async-test rule). + * `:name` — registered GenServer name. * `:sup` — supervisor instance name, forwarded to `Store.warm_from_repo/1`. """ @spec start_link(keyword()) :: GenServer.on_start() @@ -40,8 +43,8 @@ defmodule Lightning.Adaptors.NodeMonitor do {:noreply, state} end - # Deliberate no-op: nodedown does not trigger a re-warm. The 24h Cachex TTL - # backstops any staleness; 302-on-stale-sha handles already-issued URLs. + # Deliberate no-op: nodedown does not trigger a re-warm. 302-on-stale-sha + # handles already-issued icon URLs; other reads stay stale until nodeup. def handle_info({:nodedown, _node, _info}, state) do {:noreply, state} end diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index 9964f4fe664..a5e99dd7295 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -5,9 +5,12 @@ defmodule Lightning.Adaptors.NPM do Implements the four `Lightning.Adaptors.Strategy` callbacks: - * `c:Lightning.Adaptors.Strategy.list_adaptors/0` — single search-API - call returning `name + latest_version` for every - `@openfn/language-*` package. + * `c:Lightning.Adaptors.Strategy.list_adaptors/0` — merges the + `@openfn` org's authoritative package listing with the search API's + cheap version lookup, returning `name + latest_version` for every + `@openfn/language-*` package. See + `Lightning.Adaptors.NPM.Registry` for why this is two calls, not + one. * `c:Lightning.Adaptors.Strategy.fetch_adaptor/1` — packument fetch + per-version decode and latest-version schema retrieval via jsDelivr. Icon fields are **not** stamped here; the Scheduler @@ -17,7 +20,7 @@ defmodule Lightning.Adaptors.NPM do against `raw.githubusercontent.com`, used by the Store's rare lazy-miss fallback. * `c:Lightning.Adaptors.Strategy.fetch_icons/1` — bulk fan-out over - the search listing, one HTTP request per `(name, shape)`. Threads + the adaptor listing, one HTTP request per `(name, shape)`. Threads `:prior_etags` from the caller down into the per-request `If-None-Match` headers. @@ -35,19 +38,21 @@ defmodule Lightning.Adaptors.NPM do Each sub-module issues at most a handful of single-shot Tesla requests bounded by `http_timeout`. No retry, no backoff, no circuit-breaker — transient failures (5xx, timeout, nxdomain) of the *primary* request - (`packument` for `fetch_adaptor/1`, `search` for `list_adaptors/0` and - `fetch_icons/1`) surface as `{:error, term()}` unchanged. Schema and - icon fetches inside `fetch_adaptor/1` and `fetch_icons/1` are - best-effort: a single icon miss degrades that entry to absence rather - than failing the whole record. + (`packument` for `fetch_adaptor/1`, the org package listing for + `list_adaptors/0` and `fetch_icons/1`) surface as `{:error, term()}` + unchanged. The schema fetch inside `fetch_adaptor/1` and each icon + fetch inside `fetch_icons/1` are best-effort instead: a miss there + degrades to a nil schema or an absent icon shape, rather than failing + the whole record or batch. ## Configuration Each sub-module reads `:registry_url`, `:jsdelivr_url`, `:github_url`, `:github_ref`, and `:http_timeout` via - `Lightning.Adaptors.Config.strategy_opts(__MODULE__)`, with defaults - baked in so the module works even when no Application env block is - set. + `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)` — all + three share this module's own config key rather than each having their + own — with defaults baked in so the module works even when no + Application env block is set. """ @behaviour Lightning.Adaptors.Strategy diff --git a/lib/lightning/adaptors/npm/github.ex b/lib/lightning/adaptors/npm/github.ex index 30f7fb5678f..013616335c6 100644 --- a/lib/lightning/adaptors/npm/github.ex +++ b/lib/lightning/adaptors/npm/github.ex @@ -10,9 +10,11 @@ defmodule Lightning.Adaptors.NPM.GitHub do /OpenFn/adaptors//packages//assets/. - where `` strips the `@openfn/` scope from the package name. - Each `(name, shape)` is probed `png` first then `svg` — matching the - ext order used by `Lightning.Adaptors.Local`. + where `` strips the `@openfn/` scope and, when present, the + `language-` prefix too — `@openfn/language-common` becomes `common`, + matching the monorepo's `packages/` directory names. Each `(name, + shape)` is probed `png` first then `svg` — matching the ext order used + by `Lightning.Adaptors.Local`. ## Configuration diff --git a/lib/lightning/adaptors/npm/registry.ex b/lib/lightning/adaptors/npm/registry.ex index 8d6199e1206..7b30e1461e9 100644 --- a/lib/lightning/adaptors/npm/registry.ex +++ b/lib/lightning/adaptors/npm/registry.ex @@ -2,16 +2,33 @@ defmodule Lightning.Adaptors.NPM.Registry do @moduledoc """ NPM registry HTTP client for `Lightning.Adaptors.NPM`. - Talks to `registry.npmjs.org`. Responsible for the `search` endpoint - used by `c:Lightning.Adaptors.Strategy.list_adaptors/0` and the - `packument` endpoint used by `fetch_adaptor/1` and `fetch_icon/2`. + Talks to `registry.npmjs.org`. Responsible for the `list_adaptors/0` + scope listing, and the `packument` endpoint used by `fetch_adaptor/1` + and `fetch_icon/2`. + + `list_adaptors/0` deliberately merges two endpoints rather than calling + one: + + * `/-/user/openfn/package` is the authoritative name list for the + `@openfn` org — the actual trust boundary, since it can't return a + name Lightning doesn't already trust. It has no version data. + * `/-/v1/search` is used only as a cheap version lookup for whichever + of those names it happens to cover. npm's relevance ranking demotes + or excludes deprecated packages from search results even on an + exact-name query, so search alone silently drops names — it is not + a safe source of *scope membership*, only of version data for names + already known to be in scope. + + Any authoritative name missing from the search results falls back to a + per-name `get_packument/1` + `latest_version/1` call, bounded to the + handful of names search doesn't cover. Do not "simplify" this back to a + single search call or a pagination bump — search's result count is + capped by npm's relevance ranking regardless of `size`/`from`, and + deprecated packages are excluded from ranking entirely, so no amount of + paging recovers them. Base URL via `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)[:registry_url]`, default `https://registry.npmjs.org`. - - Search results are filtered down to `@openfn/language-*` packages; - non-language packages in the `@openfn/` scope (e.g. `@openfn/cli`) are - rejected. """ alias Lightning.Adaptors.Config @@ -25,30 +42,20 @@ defmodule Lightning.Adaptors.NPM.Registry do @language_prefix "@openfn/language-" @doc """ - Single `/-/v1/search` call returning `name + latest_version` for every - `@openfn/language-*` package. + Full `@openfn/language-*` scope listing, each with a real `latest_version`. + + Merges `/-/user/openfn/package` (authoritative name list) with + `/-/v1/search` (cheap version lookup), falling back to a per-name + packument fetch for any name search doesn't cover. See the moduledoc for + why this isn't a single call. """ @spec list_adaptors() :: {:ok, [%{name: String.t(), latest_version: String.t()}]} | {:error, term()} def list_adaptors do - case Tesla.get(json_client(), "/-/v1/search", - query: [text: "@" <> @search_scope, size: @search_size] - ) do - {:ok, %Tesla.Env{status: 200, body: body}} when is_map(body) -> - listing = - body - |> Map.get("objects", []) - |> Enum.map(&extract_listing_entry/1) - |> Enum.reject(&is_nil/1) - - {:ok, listing} - - {:ok, %Tesla.Env{status: status}} -> - {:error, {:http_status, status}} - - {:error, reason} -> - {:error, reason} + with {:ok, names} <- scoped_package_names(), + {:ok, version_by_name} <- search_versions() do + resolve_versions(names, version_by_name) end end @@ -135,6 +142,71 @@ defmodule Lightning.Adaptors.NPM.Registry do def repository_url(url) when is_binary(url), do: url def repository_url(_), do: nil + defp scoped_package_names do + case Tesla.get(json_client(), "/-/user/openfn/package") do + {:ok, %Tesla.Env{status: 200, body: body}} when is_map(body) -> + names = + body + |> Map.keys() + |> Enum.filter(&String.starts_with?(&1, @language_prefix)) + + {:ok, names} + + {:ok, %Tesla.Env{status: status}} -> + {:error, {:http_status, status}} + + {:error, reason} -> + {:error, reason} + end + end + + defp search_versions do + case Tesla.get(json_client(), "/-/v1/search", + query: [text: "@" <> @search_scope, size: @search_size] + ) do + {:ok, %Tesla.Env{status: 200, body: body}} when is_map(body) -> + version_by_name = + body + |> Map.get("objects", []) + |> Enum.map(&extract_listing_entry/1) + |> Enum.reject(&is_nil/1) + |> Map.new(&{&1.name, &1.latest_version}) + + {:ok, version_by_name} + + _error_or_non_200 -> + {:ok, %{}} + end + end + + defp resolve_versions(names, version_by_name) do + Enum.reduce_while(names, {:ok, []}, fn name, {:ok, acc} -> + case Map.fetch(version_by_name, name) do + {:ok, version} -> + {:cont, {:ok, [%{name: name, latest_version: version} | acc]}} + + :error -> + case fetch_latest_version(name) do + {:ok, version} -> + {:cont, {:ok, [%{name: name, latest_version: version} | acc]}} + + {:error, reason} -> + {:halt, {:error, reason}} + end + end + end) + |> case do + {:ok, entries} -> {:ok, Enum.reverse(entries)} + {:error, reason} -> {:error, reason} + end + end + + defp fetch_latest_version(name) do + with {:ok, packument} <- get_packument(name) do + latest_version(packument) + end + end + defp extract_listing_entry(%{ "package" => %{"name" => name, "version" => version} }) diff --git a/lib/lightning/adaptors/package_name.ex b/lib/lightning/adaptors/package_name.ex index a55e0533442..c98569f09bc 100644 --- a/lib/lightning/adaptors/package_name.ex +++ b/lib/lightning/adaptors/package_name.ex @@ -8,6 +8,8 @@ defmodule Lightning.Adaptors.PackageName do @name_format ~r{\A@?[\w.-]+(?:/[\w.-]+)?\z} + @language_prefix "@openfn/language-" + @doc """ Returns the spec format: a package name plus optional `@version`, with no newlines or shell metacharacters. @@ -40,9 +42,9 @@ defmodule Lightning.Adaptors.PackageName do @doc """ Renders a spec for the worker. - `opts[:source]` of `:local` forces `name@local`. `opts[:latest]` is the - concrete version for a `latest` spec, and is required for one under - any other source. A `name@local` spec is always kept as is. + `opts[:source]` of `:local` forces `name@local`. Otherwise, a `latest` + spec is resolved to `opts[:latest]`, which must be given in that case. + A `name@local` spec is always kept as is. """ @spec to_wire(String.t() | nil, keyword()) :: String.t() def to_wire(adaptor, opts \\ []) do @@ -61,4 +63,20 @@ defmodule Lightning.Adaptors.PackageName do end end end + + @doc """ + Strips the `@openfn/language-` prefix from a full package name, e.g. + `"@openfn/language-http"` -> `"http"`. Any other value, including `nil` + (credential schemas are nullable), passes through unchanged. + """ + @spec short_name(String.t() | nil) :: String.t() | nil + def short_name(@language_prefix <> short), do: short + def short_name(other), do: other + + @doc """ + Prepends the `@openfn/language-` prefix to a short adaptor name, e.g. + `"http"` -> `"@openfn/language-http"`. + """ + @spec full_name(String.t()) :: String.t() + def full_name(short) when is_binary(short), do: @language_prefix <> short end diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index f06df454e3e..9ec6b8f5b80 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -658,10 +658,9 @@ defmodule Lightning.Adaptors.Scheduler do {:error, {:upsert_failed, Exception.message(e)}} end - # Builds the prior-etag map for `Strategy.fetch_icons/1`. A row or shape - # with no etag is left out rather than kept as an empty entry — the - # strategy already treats an absent entry as "no prior etag, don't send - # If-None-Match". + # A row or shape with no etag is left out rather than kept as an empty + # entry — the strategy already treats an absent entry as "no prior etag, + # don't send If-None-Match". @spec prior_etags_from_rows([map()]) :: %{ String.t() => %{optional(:square | :rectangle) => String.t()} } diff --git a/lib/lightning/adaptors/supervisor.ex b/lib/lightning/adaptors/supervisor.ex index caeb934d765..1d98d662e96 100644 --- a/lib/lightning/adaptors/supervisor.ex +++ b/lib/lightning/adaptors/supervisor.ex @@ -37,6 +37,8 @@ defmodule Lightning.Adaptors.Supervisor do strategy = Keyword.get(opts, :strategy, Config.strategy()) lock_key = Keyword.get(opts, :lock_key, lock_key(name)) + # Per-instance config for stateless callers that hold only the name. + # Children take theirs from the child spec (see lock_key), not from here. :persistent_term.put(meta_key(name), %{ strategy: strategy, source: Config.source_for(strategy) diff --git a/lib/lightning/ai_assistant/ai_assistant.ex b/lib/lightning/ai_assistant/ai_assistant.ex index 55f0323aa07..32a668e24b6 100644 --- a/lib/lightning/ai_assistant/ai_assistant.ex +++ b/lib/lightning/ai_assistant/ai_assistant.ex @@ -545,7 +545,8 @@ defmodule Lightning.AiAssistant do ## Returns An updated `ChatSession` struct with `:expression` and `:adaptor` fields populated. - The adaptor is resolved through `Lightning.Adaptors.to_wire/1`. + The adaptor is resolved through `Lightning.Adaptors.to_wire/1`, falling back + to the adaptor as given if that fails. """ @spec put_expression_and_adaptor(ChatSession.t(), String.t(), String.t()) :: ChatSession.t() diff --git a/lib/lightning/application.ex b/lib/lightning/application.ex index d84aae759ef..1523c1fec9f 100644 --- a/lib/lightning/application.ex +++ b/lib/lightning/application.ex @@ -123,6 +123,16 @@ defmodule Lightning.Application do ) ) + schema_reconciler_childspec = + if Application.get_env( + :lightning, + Lightning.Credentials.SchemaReconciler, + enabled: true + )[:enabled] do + {Lightning.Credentials.SchemaReconciler, + name: Lightning.Credentials.SchemaReconciler, sup: Lightning.Adaptors} + end + goth = Application.get_env(:lightning, Lightning.Google, []) |> then(fn config -> @@ -166,6 +176,7 @@ defmodule Lightning.Application do LightningWeb.WorkerPresence, adaptor_service_childspec, {Lightning.Adaptors.Supervisor, name: Lightning.Adaptors}, + schema_reconciler_childspec, {Lightning.TaskWorker, name: :cli_task_worker}, {Lightning.Runtime.RuntimeManager, worker_secret: Lightning.Config.worker_secret(), diff --git a/lib/lightning/channels/destination_auth.ex b/lib/lightning/channels/destination_auth.ex index f44fc8754c2..a157e703910 100644 --- a/lib/lightning/channels/destination_auth.ex +++ b/lib/lightning/channels/destination_auth.ex @@ -4,8 +4,8 @@ defmodule Lightning.Channels.DestinationAuth do ## Supported Schemas - - `"http"` — Bearer token (`access_token`) or Basic auth (`username`+`password`) - - `"dhis2"` — DHIS2 ApiToken (`pat`) or Basic auth (`username`+`password`) + - `"@openfn/language-http"` — Bearer token (`access_token`) or Basic auth (`username`+`password`) + - `"@openfn/language-dhis2"` — DHIS2 ApiToken (`pat`) or Basic auth (`username`+`password`) - `"oauth"` — Bearer token (`access_token`, with auto-refresh via `resolve_credential_body`) Schemas not in this list are rejected at config time. If an unsupported schema @@ -24,7 +24,7 @@ defmodule Lightning.Channels.DestinationAuth do @spec build_auth_header(String.t(), map()) :: {:ok, String.t()} | {:error, :no_auth_fields | {:unsupported_schema, String.t()}} - def build_auth_header("http", body) do + def build_auth_header("@openfn/language-http", body) do cond do token = body["access_token"] -> {:ok, "Bearer #{token}"} @@ -38,7 +38,7 @@ defmodule Lightning.Channels.DestinationAuth do end end - def build_auth_header("dhis2", body) do + def build_auth_header("@openfn/language-dhis2", body) do cond do token = body["pat"] -> {:ok, "ApiToken #{token}"} diff --git a/lib/lightning/collaboration/session.ex b/lib/lightning/collaboration/session.ex index 5a645acf742..39a967f9549 100644 --- a/lib/lightning/collaboration/session.ex +++ b/lib/lightning/collaboration/session.ex @@ -578,7 +578,6 @@ defmodule Lightning.Collaboration.Session do """ end) - # Write validation errors to Y.Doc write_validation_errors_to_ydoc(state, changeset) {:reply, {:error, changeset}, state} diff --git a/lib/lightning/config/bootstrap.ex b/lib/lightning/config/bootstrap.ex index ca3a30f8372..9e86b967270 100644 --- a/lib/lightning/config/bootstrap.ex +++ b/lib/lightning/config/bootstrap.ex @@ -1066,11 +1066,11 @@ defmodule Lightning.Config.Bootstrap do end end - # ADAPTORS_LOCAL_REPO wins outright when set. When unset, fall back to - # the (ungated) OPENFN_ADAPTORS_REPO parse above, warning only when - # Lightning.Adaptors is actually running the Local strategy — an operator - # running the npm strategy with OPENFN_ADAPTORS_REPO still set for the - # ws-worker shouldn't be warned about a var they legitimately need. + # ADAPTORS_LOCAL_REPO wins outright when set. When it's unset, fall back + # to the OPENFN_ADAPTORS_REPO value parsed above, but only warn about it + # when Lightning.Adaptors is actually running the Local strategy — an + # operator running the npm strategy can leave OPENFN_ADAPTORS_REPO set for + # the ws-worker without being warned about a var they still need. defp resolve_local_strategy_paths(local_adaptors_repos, adaptors_strategy) do case env!("ADAPTORS_LOCAL_REPO", :string, nil) |> parse_repo_list() do [] -> diff --git a/lib/lightning/credentials.ex b/lib/lightning/credentials.ex index f9c49b17fe8..c4d6dfa7f47 100644 --- a/lib/lightning/credentials.ex +++ b/lib/lightning/credentials.ex @@ -587,15 +587,55 @@ defmodule Lightning.Credentials do """ @spec get_schema(String.t()) :: Credentials.Schema.t() def get_schema(schema_name) do - case Lightning.Adaptors.schema(schema_name) do + resolved = Lightning.Adaptors.resolve_name(schema_name) + + case Lightning.Adaptors.schema(resolved) do {:ok, schema_body} -> - Credentials.Schema.new(schema_body, schema_name) + Credentials.Schema.new(schema_body, resolved) {:error, reason} -> raise "Error reading credential schema. Got: #{inspect(reason)}" end end + @doc """ + Resolves every credential's legacy short-form `schema` to its full npm + package name, in place. Idempotent; safe to call repeatedly. + + Does not touch `updated_at` or emit audit events; this is a + storage-format fix, not an edit. + """ + @spec reconcile_legacy_schema_names(atom()) :: non_neg_integer() + def reconcile_legacy_schema_names(sup) do + updated = + from(c in Credential, + where: not like(c.schema, "@%"), + select: c.schema, + distinct: true + ) + |> Repo.all() + |> Enum.reduce(0, fn short, count -> + case Lightning.Adaptors.resolve_name(sup, short) do + ^short -> + count + + full -> + {n, _} = + Repo.update_all(from(c in Credential, where: c.schema == ^short), + set: [schema: full] + ) + + count + n + end + end) + + if updated > 0 do + Logger.info("Reconciled #{updated} legacy credential schema name(s)") + end + + updated + end + defp cast_credential_body_change( %Ecto.Changeset{valid?: true, changes: %{body: body}} = changeset, schema_name diff --git a/lib/lightning/credentials/credential.ex b/lib/lightning/credentials/credential.ex index 04eff7de7cb..bb95bef8524 100644 --- a/lib/lightning/credentials/credential.ex +++ b/lib/lightning/credentials/credential.ex @@ -70,6 +70,7 @@ defmodule Lightning.Credentials.Credential do defp shared_validations(changeset) do changeset + |> resolve_schema_name() |> normalize_external_id() |> cast_assoc(:project_credentials) |> validate_required([:name, :user_id]) @@ -107,4 +108,11 @@ defmodule Lightning.Credentials.Credential do _ -> changeset end end + + defp resolve_schema_name(changeset) do + update_change(changeset, :schema, fn + schema when is_binary(schema) -> Lightning.Adaptors.resolve_name(schema) + schema -> schema + end) + end end diff --git a/lib/lightning/credentials/schema_reconciler.ex b/lib/lightning/credentials/schema_reconciler.ex new file mode 100644 index 00000000000..659c558aea1 --- /dev/null +++ b/lib/lightning/credentials/schema_reconciler.ex @@ -0,0 +1,91 @@ +defmodule Lightning.Credentials.SchemaReconciler do + @moduledoc """ + Sweeps legacy short-form credential `schema` names to their full npm + package names whenever the adaptor catalogue might have grown enough to + resolve one, via `Lightning.Credentials.reconcile_legacy_schema_names/1`. + + Runs once on start **and** on every `adaptors_updated` broadcast. The + broadcast (`Lightning.Adaptors.ChannelBroadcaster`) only fires when an + adaptor row actually changes, so a catalogue that is already warm (e.g. + after a restart, backed by a persistent store) may complete its first + refresh without changing a single row and would never emit anything — a + subscriber that waited only for the broadcast would then never sweep. The + on-start run covers that case. + + The sweep is idempotent (each pass only touches rows still on a short + name), so running it twice, from two triggers, or on every node in a + cluster (the PubSub topic is cluster-wide) is safe. There is deliberately + no "done" flag gating it — that flag was the bug this module replaces: it + could get set after a failed run and then never retry. + + Each sweep issues a `SELECT DISTINCT schema` over `credentials` (no index + on that column) plus one catalogue lookup per distinct legacy name, and + every node runs it on every broadcast. Cheap at current table sizes; + revisit if `credentials` or broadcast frequency grow enough to matter. + """ + + use GenServer + + alias Lightning.Credentials + + require Logger + + @default_retry_ms :timer.minutes(5) + + @doc """ + Starts the reconciler. Required opts: `:name`, `:sup`. Optional: + `:reconcile` (1-arity fn, default + `&Lightning.Credentials.reconcile_legacy_schema_names/1`) and `:retry_ms` + (default 5 minutes) controlling the retry delay after a failed sweep. + """ + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts) do + name = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts) do + sup = Keyword.fetch!(opts, :sup) + + reconcile = + Keyword.get(opts, :reconcile, &Credentials.reconcile_legacy_schema_names/1) + + retry_ms = Keyword.get(opts, :retry_ms, @default_retry_ms) + + :ok = Lightning.Adaptors.subscribe_to_updates(sup) + send(self(), :reconcile) + + {:ok, %{sup: sup, reconcile: reconcile, retry_ms: retry_ms, retry_ref: nil}} + end + + @impl true + def handle_info(:reconcile, state) do + {:noreply, run(state)} + end + + def handle_info(%{event: "adaptors_updated"}, state) do + {:noreply, run(state)} + end + + def handle_info(_msg, state), do: {:noreply, state} + + # Cancels any pending retry so a run triggered by a broadcast (or a + # concurrent :reconcile) never leaves two retry chains ticking. + defp run(state) do + if state.retry_ref, do: Process.cancel_timer(state.retry_ref) + state.reconcile.(state.sup) + %{state | retry_ref: nil} + rescue + error -> + Logger.warning( + "SchemaReconciler: sweep failed, retrying in #{state.retry_ms}ms: " <> + Exception.format(:error, error, __STACKTRACE__) + ) + + %{ + state + | retry_ref: Process.send_after(self(), :reconcile, state.retry_ms) + } + end +end diff --git a/lib/lightning/setup_utils.ex b/lib/lightning/setup_utils.ex index 374c803ba5f..3313dbf91e6 100644 --- a/lib/lightning/setup_utils.ex +++ b/lib/lightning/setup_utils.ex @@ -110,7 +110,7 @@ defmodule Lightning.SetupUtils do }, name: "DHIS2 play", user_id: user_id, - schema: "dhis2" + schema: "@openfn/language-dhis2" }, user ) diff --git a/lib/lightning_web/channels/workflow_channel.ex b/lib/lightning_web/channels/workflow_channel.ex index 64ab99fe977..3fb230867df 100644 --- a/lib/lightning_web/channels/workflow_channel.ex +++ b/lib/lightning_web/channels/workflow_channel.ex @@ -897,7 +897,7 @@ defmodule LightningWeb.WorkflowChannel do # Catch-all for any event this channel doesn't recognise (e.g. a stale # client tab still sending an event removed in a later deploy). Replies # with an error instead of raising FunctionClauseError, which would kill - # the channel process and disconnect every collaborator in the room. + # this client's channel process and drop its connection. @impl true def handle_in(event, _payload, socket) do warn_unhandled_message("handle_in", event) @@ -918,7 +918,8 @@ defmodule LightningWeb.WorkflowChannel do @impl true def handle_info({:save_workflow_reply, ref, {:ok, workflow}}, socket) do - # Broadcast the new lock_version to all users in the channel so they can + # broadcast_from! skips the saving client, which already gets its + # lock_version through the reply below; everyone else needs this to # update their latestSnapshotLockVersion in SessionContextStore. broadcast_from!(socket, "workflow_saved", %{ latest_snapshot_lock_version: workflow.lock_version, @@ -1218,7 +1219,7 @@ defmodule LightningWeb.WorkflowChannel do # Catch-all for any internal message this channel doesn't recognise (e.g. a # PubSub broadcast for an event type removed in a later deploy). Logs and # keeps the channel alive instead of raising FunctionClauseError, which - # would kill the process and disconnect every collaborator in the room. + # would kill this client's channel process and drop its connection. @impl true def handle_info(message, socket) do warn_unhandled_message("handle_info", unhandled_message_type(message)) diff --git a/lib/lightning_web/live/components/data_tables.ex b/lib/lightning_web/live/components/data_tables.ex index 777b2b6be8e..71d3e3e4f21 100644 --- a/lib/lightning_web/live/components/data_tables.ex +++ b/lib/lightning_web/live/components/data_tables.ex @@ -3,6 +3,7 @@ defmodule LightningWeb.Components.DataTables do use LightningWeb, :component alias Lightning.Accounts.User + alias Lightning.Adaptors.PackageName alias Lightning.Credentials.Credential alias Lightning.Policies.Permissions alias Lightning.Policies.ProjectUsers @@ -272,7 +273,7 @@ defmodule LightningWeb.Components.DataTables do end defp credential_type(%Credential{schema: schema}) do - schema + PackageName.short_name(schema) end defp missing_oauth_client?(credential) do diff --git a/lib/lightning_web/live/credential_live/credential_form_component.ex b/lib/lightning_web/live/credential_live/credential_form_component.ex index 7da547dd3ae..cda03aa3af5 100644 --- a/lib/lightning_web/live/credential_live/credential_form_component.ex +++ b/lib/lightning_web/live/credential_live/credential_form_component.ex @@ -5,6 +5,7 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do use LightningWeb, :live_component alias Lightning.Adaptors + alias Lightning.Adaptors.PackageName alias Lightning.Credentials alias Lightning.OauthClients alias LightningWeb.AdaptorIconURL @@ -1181,7 +1182,7 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do adaptor_options |> Enum.reject(fn {_, name, _, _} -> - name in ["googlesheets", "gmail", "collections"] + name in ["@openfn/language-googlesheets", "@openfn/language-gmail"] end) |> Enum.concat([ {"Raw JSON", "raw", @@ -1194,7 +1195,8 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do end defp adaptor_type_option(%Adaptors.Package{name: name} = pkg) do - {name, name, AdaptorIconURL.build(name, pkg, :square), nil} + {PackageName.short_name(name), name, + AdaptorIconURL.build(name, pkg, :square), nil} end defp list_users do @@ -1350,14 +1352,20 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do assign(socket, oauth_clients: oauth_clients, type_options: type_options) end - defp format_schema_name("raw"), do: "Raw JSON" - defp format_schema_name("oauth"), do: "OAuth" - defp format_schema_name("http"), do: "HTTP" - defp format_schema_name(schema) when is_binary(schema) do - schema - |> String.split("_") - |> Enum.map_join(" ", &String.capitalize/1) + case PackageName.short_name(schema) do + "raw" -> + "Raw JSON" + + "oauth" -> + "OAuth" + + "http" -> + "HTTP" + + short -> + short |> String.split("_") |> Enum.map_join(" ", &String.capitalize/1) + end end defp get_credential_description("raw", _type), diff --git a/lib/lightning_web/live/maintenance_live/index.ex b/lib/lightning_web/live/maintenance_live/index.ex index ae9959fbe26..4102b86f642 100644 --- a/lib/lightning_web/live/maintenance_live/index.ex +++ b/lib/lightning_web/live/maintenance_live/index.ex @@ -5,9 +5,9 @@ defmodule LightningWeb.MaintenanceLive.Index do Exposes two actions: "Refresh Adaptor Registry" (`refresh/0`) and "Refresh Adaptor Icons" (`refresh_icons/0`). Neither blocks the LiveView: - the registry refresh is fire-and-forget on the leader node, while the icon - refresh runs under `start_async` (the underlying call can take up to two - minutes) and flashes its result when it completes. + the registry refresh is fire-and-forget on the cluster-singleton scheduler, + while the icon refresh runs under `start_async` (the underlying call can + take up to two minutes) and flashes its result when it completes. """ use LightningWeb, :live_view diff --git a/lib/mix/tasks/lightning.adaptors.snapshot.ex b/lib/mix/tasks/lightning.adaptors.snapshot.ex index 8b2e5cc5470..3de2fdf6c42 100644 --- a/lib/mix/tasks/lightning.adaptors.snapshot.ex +++ b/lib/mix/tasks/lightning.adaptors.snapshot.ex @@ -12,7 +12,13 @@ defmodule Mix.Tasks.Lightning.Adaptors.Snapshot do catalogue is already populated. Either file can be read back by `mix lightning.adaptors.import`. - Use --path to specify the location + ## Usage + + mix lightning.adaptors.snapshot + mix lightning.adaptors.snapshot --path snapshot.json + + Without `--path`, writes to `adaptor_registry_cache.json` in this + instance's `priv` directory. """ use Mix.Task diff --git a/priv/repo/migrations/20260907112954_widen_credentials_schema.exs b/priv/repo/migrations/20260907112954_widen_credentials_schema.exs new file mode 100644 index 00000000000..b11f79a44d3 --- /dev/null +++ b/priv/repo/migrations/20260907112954_widen_credentials_schema.exs @@ -0,0 +1,9 @@ +defmodule Lightning.Repo.Migrations.WidenCredentialsSchema do + use Ecto.Migration + + def change do + alter table(:credentials) do + modify :schema, :string, size: 100, from: {:string, size: 40} + end + end +end diff --git a/test/lightning/adaptors/catalogue_test.exs b/test/lightning/adaptors/catalogue_test.exs index 53d97448581..105a51be6d8 100644 --- a/test/lightning/adaptors/catalogue_test.exs +++ b/test/lightning/adaptors/catalogue_test.exs @@ -317,6 +317,82 @@ defmodule Lightning.Adaptors.CatalogueTest do assert [%{name: "@openfn/language-http"}] = Catalogue.list_package_metas(:npm) end + + test "omits a deprecated adaptor" do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record(name: "@openfn/language-deprecated", deprecated: true) + ) + + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert [%{name: "@openfn/language-http"}] = + Catalogue.list_package_metas(:npm) + end + end + + describe "catalogue/1 — deprecated filtering" do + test "omits a deprecated adaptor entirely" do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record(name: "@openfn/language-deprecated", deprecated: true) + ) + + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert [%{name: "@openfn/language-http"}] = Catalogue.catalogue(:npm) + end + + test "omits only the deprecated version from a still-listed adaptor" do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + versions: [ + version_record("1.0.0", deprecated: true), + version_record("1.1.0"), + version_record("1.2.0") + ] + ) + ) + + assert [%{name: "@openfn/language-http", versions: versions}] = + Catalogue.catalogue(:npm) + + assert Enum.sort(versions) == ["1.1.0", "1.2.0"] + end + end + + describe "get_adaptor/2 and list_versions/2 — resolve paths stay unfiltered" do + test "still return a fully-deprecated adaptor and its versions" do + {:ok, adaptor} = + Catalogue.upsert_adaptor( + adaptor_record(name: "@openfn/language-deprecated", deprecated: true) + ) + + assert %Adaptor{name: "@openfn/language-deprecated"} = + Catalogue.get_adaptor("@openfn/language-deprecated", :npm) + + assert [%AdaptorVersion{version: "1.0.0"}] = + Catalogue.list_versions("@openfn/language-deprecated", :npm) + + refute adaptor.id == nil + end + + test "still return a deprecated version of an otherwise-normal adaptor" do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + versions: [ + version_record("1.0.0", deprecated: true), + version_record("1.1.0") + ] + ) + ) + + assert Catalogue.list_versions("@openfn/language-http", :npm) + |> Enum.map(& &1.version) + |> Enum.sort() == ["1.0.0", "1.1.0"] + end end describe "list_adaptors/1" do @@ -470,7 +546,7 @@ defmodule Lightning.Adaptors.CatalogueTest do |> Map.merge(overrides) end - defp version_record(version) do + defp version_record(version, overrides \\ []) do %{ version: version, integrity: "sha512-#{version}", @@ -481,6 +557,7 @@ defmodule Lightning.Adaptors.CatalogueTest do published_at: nil, deprecated: false } + |> Map.merge(Map.new(overrides)) end defp seed_adaptor(opts) do diff --git a/test/lightning/adaptors/isolated_adaptors_test.exs b/test/lightning/adaptors/isolated_adaptors_test.exs index bbd20c0222e..e334ab45880 100644 --- a/test/lightning/adaptors/isolated_adaptors_test.exs +++ b/test/lightning/adaptors/isolated_adaptors_test.exs @@ -52,16 +52,17 @@ defmodule Lightning.Adaptors.IsolatedAdaptorsTest do seed_credential_schema("http") source = AdaptorsSupervisor.source(sup) + full_name = "@openfn/language-http" assert {:ok, {:ok, _schema_body}} = Cachex.get( AdaptorsSupervisor.cache_name(sup), - {:schema, "http", source} + {:schema, full_name, source} ) assert Cachex.get( AdaptorsSupervisor.cache_name(Lightning.Adaptors), - {:schema, "http", source} + {:schema, full_name, source} ) == {:ok, nil} end end diff --git a/test/lightning/adaptors/npm/registry_test.exs b/test/lightning/adaptors/npm/registry_test.exs index 2132eb14d47..e8b701955f8 100644 --- a/test/lightning/adaptors/npm/registry_test.exs +++ b/test/lightning/adaptors/npm/registry_test.exs @@ -35,9 +35,13 @@ defmodule Lightning.Adaptors.NPM.RegistryTest do end describe "list_adaptors/0" do - test "returns an empty list when the search has no results", %{ + test "returns an empty list when the org has no packages", %{ bypass: bypass } do + Bypass.expect(bypass, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{}) + end) + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> conn = Plug.Conn.fetch_query_params(conn) assert conn.query_params["text"] == "@openfn" @@ -49,7 +53,16 @@ defmodule Lightning.Adaptors.NPM.RegistryTest do assert {:ok, []} = Registry.list_adaptors() end - test "returns name + latest_version for each search hit", %{bypass: bypass} do + test "returns name + latest_version for each authoritative name", %{ + bypass: bypass + } do + Bypass.expect(bypass, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{ + "@openfn/language-http" => "write", + "@openfn/language-salesforce" => "write" + }) + end) + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> body = %{ "objects" => [ @@ -81,6 +94,10 @@ defmodule Lightning.Adaptors.NPM.RegistryTest do test "filters out @openfn/* packages that aren't language-* adaptors and other scopes", %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{"@openfn/language-http" => "write"}) + end) + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> body = %{ "objects" => [ @@ -122,6 +139,10 @@ defmodule Lightning.Adaptors.NPM.RegistryTest do end test "skips malformed entries that lack name or version", %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{"@openfn/language-http" => "write"}) + end) + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> body = %{ "objects" => [ @@ -143,20 +164,141 @@ defmodule Lightning.Adaptors.NPM.RegistryTest do Registry.list_adaptors() end - test "surfaces 5xx responses as {:error, _}", %{bypass: bypass} do + test "degrades a failed search to a full packument fallback", %{ + bypass: bypass + } do + Bypass.expect(bypass, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{ + "@openfn/language-http" => "write", + "@openfn/language-salesforce" => "write" + }) + end) + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> Plug.Conn.resp(conn, 503, "") end) + Bypass.expect(bypass, "GET", "/@openfn/language-http", fn conn -> + json_resp(conn, 200, %{"dist-tags" => %{"latest" => "2.1.0"}}) + end) + + Bypass.expect(bypass, "GET", "/@openfn/language-salesforce", fn conn -> + json_resp(conn, 200, %{"dist-tags" => %{"latest" => "4.6.3"}}) + end) + + {:ok, listing} = Registry.list_adaptors() + + assert Enum.sort_by(listing, & &1.name) == [ + %{name: "@openfn/language-http", latest_version: "2.1.0"}, + %{name: "@openfn/language-salesforce", latest_version: "4.6.3"} + ] + end + + test "surfaces 5xx responses from the org package listing as {:error, _}", + %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/-/user/openfn/package", fn conn -> + Plug.Conn.resp(conn, 503, "") + end) + assert {:error, {:http_status, 503}} = Registry.list_adaptors() end + test "propagates a packument-fallback failure for the whole call", %{ + bypass: bypass + } do + Bypass.expect(bypass, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{"@openfn/language-eapts" => "write"}) + end) + + Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> + json_resp(conn, 200, %{"objects" => []}) + end) + + Bypass.expect(bypass, "GET", "/@openfn/language-eapts", fn conn -> + Plug.Conn.resp(conn, 502, "") + end) + + assert {:error, {:http_status, 502}} = Registry.list_adaptors() + end + test "surfaces network failure as {:error, _}", %{bypass: bypass} do Bypass.down(bypass) assert {:error, _reason} = Registry.list_adaptors() end end + describe "list_adaptors/0 registry completeness" do + test "includes authoritative names missing from search, via packument fallback", + %{bypass: bypass} do + authoritative_names = [ + "@openfn/language-http", + "@openfn/language-salesforce", + "@openfn/language-dhis2", + "@openfn/language-fhir", + "@openfn/language-mysql", + "@openfn/language-postgresql", + "@openfn/language-common", + "@openfn/language-asana", + "@openfn/language-openmrs", + "@openfn/language-eapts" + ] + + Bypass.expect_once(bypass, "GET", "/-/user/openfn/package", fn conn -> + body = Map.new(authoritative_names, &{&1, "write"}) + json_resp(conn, 200, body) + end) + + Bypass.expect_once(bypass, "GET", "/-/v1/search", fn conn -> + objects = + authoritative_names + |> List.delete("@openfn/language-eapts") + |> Enum.map(fn name -> + %{"package" => %{"name" => name, "version" => "1.0.0"}} + end) + + json_resp(conn, 200, %{"objects" => objects}) + end) + + Bypass.expect_once(bypass, "GET", "/@openfn/language-eapts", fn conn -> + json_resp(conn, 200, %{ + "name" => "@openfn/language-eapts", + "dist-tags" => %{"latest" => "3.2.1"} + }) + end) + + {:ok, listing} = Registry.list_adaptors() + + assert Enum.sort_by(listing, & &1.name) == + Enum.sort_by( + [ + %{name: "@openfn/language-http", latest_version: "1.0.0"}, + %{ + name: "@openfn/language-salesforce", + latest_version: "1.0.0" + }, + %{name: "@openfn/language-dhis2", latest_version: "1.0.0"}, + %{name: "@openfn/language-fhir", latest_version: "1.0.0"}, + %{name: "@openfn/language-mysql", latest_version: "1.0.0"}, + %{ + name: "@openfn/language-postgresql", + latest_version: "1.0.0" + }, + %{name: "@openfn/language-common", latest_version: "1.0.0"}, + %{name: "@openfn/language-asana", latest_version: "1.0.0"}, + %{ + name: "@openfn/language-openmrs", + latest_version: "1.0.0" + }, + %{ + name: "@openfn/language-eapts", + latest_version: "3.2.1" + } + ], + & &1.name + ) + end + end + describe "get_packument/1" do test "returns the decoded map on 200", %{bypass: bypass} do packument = %{ diff --git a/test/lightning/adaptors/npm_test.exs b/test/lightning/adaptors/npm_test.exs index 4088fc33520..53cb4547f86 100644 --- a/test/lightning/adaptors/npm_test.exs +++ b/test/lightning/adaptors/npm_test.exs @@ -6,8 +6,6 @@ defmodule Lightning.Adaptors.NPMTest do @package "@openfn/language-http" @latest_version "2.1.0" - # One Bypass server each for the npm registry, jsDelivr, and - # raw.githubusercontent.com; each URL is installed onto the strategy_opts block. setup do registry = Bypass.open() jsdelivr = Bypass.open() @@ -167,6 +165,13 @@ defmodule Lightning.Adaptors.NPMTest do registry: registry, github: github } do + Bypass.expect(registry, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{ + "@openfn/language-http" => "write", + "@openfn/language-salesforce" => "write" + }) + end) + Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> body = %{ "objects" => [ @@ -217,7 +222,7 @@ defmodule Lightning.Adaptors.NPMTest do end test "surfaces list_adaptors errors as {:error, _}", %{registry: registry} do - Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> + Bypass.expect(registry, "GET", "/-/user/openfn/package", fn conn -> Plug.Conn.resp(conn, 503, "") end) diff --git a/test/lightning/adaptors/package_name_test.exs b/test/lightning/adaptors/package_name_test.exs index 19958ab9140..db7e807bed4 100644 --- a/test/lightning/adaptors/package_name_test.exs +++ b/test/lightning/adaptors/package_name_test.exs @@ -80,4 +80,28 @@ defmodule Lightning.Adaptors.PackageNameTest do ) == "@openfn/language-common@local" end end + + describe "short_name/1" do + test "strips the @openfn/language- prefix" do + assert PackageName.short_name("@openfn/language-http") == "http" + end + + test "passes through a name with no prefix unchanged" do + assert PackageName.short_name("raw") == "raw" + end + + test "passes through an unrelated scoped name unchanged" do + assert PackageName.short_name("@other/scope") == "@other/scope" + end + + test "passes through nil unchanged" do + assert PackageName.short_name(nil) == nil + end + end + + describe "full_name/1" do + test "prepends the @openfn/language- prefix" do + assert PackageName.full_name("http") == "@openfn/language-http" + end + end end diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index 77b64c1a29b..416182a164e 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -1,7 +1,7 @@ defmodule Lightning.Adaptors.SchedulerTest do - # async: false because: - # 1. DataCase uses shared sandbox mode (all processes access DB without allow/3) - # 2. set_mox_global is safe only when tests run serially + # async: false — DataCase's shared sandbox mode means every process can + # reach the DB without an allow/3 call, and set_mox_global is only safe + # when tests run serially. use Lightning.DataCase, async: false import Eventually diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index 783835c6cc4..a76a4ef15f8 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -206,6 +206,40 @@ defmodule Lightning.AdaptorsTest do end end + describe "resolve_name/2" do + test "resolves a short name to the full name when the full name is in the catalogue", + %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert Adaptors.resolve_name(sup, "http") == "@openfn/language-http" + end + + test "leaves a full name that is already in the catalogue unchanged", %{ + sup: sup + } do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert Adaptors.resolve_name(sup, "@openfn/language-http") == + "@openfn/language-http" + end + + test "leaves an unknown short name unchanged", %{sup: sup} do + assert Adaptors.resolve_name(sup, "unknownish") == "unknownish" + end + + test "never resolves the raw and oauth sentinels, even if shadowed in the catalogue", + %{sup: sup} do + {:ok, _} = + Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/language-raw")) + + {:ok, _} = + Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/language-oauth")) + + assert Adaptors.resolve_name(sup, "raw") == "raw" + assert Adaptors.resolve_name(sup, "oauth") == "oauth" + end + end + describe "to_wire/1" do test "resolves @latest against the catalogue and passes semver through" do {:ok, _} = diff --git a/test/lightning/channels/destination_auth_test.exs b/test/lightning/channels/destination_auth_test.exs index 90cb4dad95e..24172ecec4e 100644 --- a/test/lightning/channels/destination_auth_test.exs +++ b/test/lightning/channels/destination_auth_test.exs @@ -6,7 +6,7 @@ defmodule Lightning.Channels.DestinationAuthTest do describe "build_auth_header/2 with http schema" do test "Bearer token from access_token" do assert {:ok, "Bearer tok-123"} = - DestinationAuth.build_auth_header("http", %{ + DestinationAuth.build_auth_header("@openfn/language-http", %{ "access_token" => "tok-123" }) end @@ -15,7 +15,7 @@ defmodule Lightning.Channels.DestinationAuthTest do expected = "Basic #{Base.encode64("user:pass")}" assert {:ok, ^expected} = - DestinationAuth.build_auth_header("http", %{ + DestinationAuth.build_auth_header("@openfn/language-http", %{ "username" => "user", "password" => "pass" }) @@ -23,7 +23,7 @@ defmodule Lightning.Channels.DestinationAuthTest do test "access_token takes priority over username/password" do assert {:ok, "Bearer tok-priority"} = - DestinationAuth.build_auth_header("http", %{ + DestinationAuth.build_auth_header("@openfn/language-http", %{ "access_token" => "tok-priority", "username" => "user", "password" => "pass" @@ -32,28 +32,30 @@ defmodule Lightning.Channels.DestinationAuthTest do test "error when no auth fields present" do assert {:error, :no_auth_fields} = - DestinationAuth.build_auth_header("http", %{ + DestinationAuth.build_auth_header("@openfn/language-http", %{ "baseUrl" => "https://example.com" }) end test "error when body is empty" do assert {:error, :no_auth_fields} = - DestinationAuth.build_auth_header("http", %{}) + DestinationAuth.build_auth_header("@openfn/language-http", %{}) end end describe "build_auth_header/2 with dhis2 schema" do test "ApiToken from pat" do assert {:ok, "ApiToken d2pat_abc"} = - DestinationAuth.build_auth_header("dhis2", %{"pat" => "d2pat_abc"}) + DestinationAuth.build_auth_header("@openfn/language-dhis2", %{ + "pat" => "d2pat_abc" + }) end test "Basic auth from username and password" do expected = "Basic #{Base.encode64("admin:secret")}" assert {:ok, ^expected} = - DestinationAuth.build_auth_header("dhis2", %{ + DestinationAuth.build_auth_header("@openfn/language-dhis2", %{ "username" => "admin", "password" => "secret" }) @@ -61,7 +63,7 @@ defmodule Lightning.Channels.DestinationAuthTest do test "pat takes priority over username/password" do assert {:ok, "ApiToken my-pat"} = - DestinationAuth.build_auth_header("dhis2", %{ + DestinationAuth.build_auth_header("@openfn/language-dhis2", %{ "pat" => "my-pat", "username" => "admin", "password" => "secret" @@ -70,7 +72,7 @@ defmodule Lightning.Channels.DestinationAuthTest do test "error when no auth fields present" do assert {:error, :no_auth_fields} = - DestinationAuth.build_auth_header("dhis2", %{ + DestinationAuth.build_auth_header("@openfn/language-dhis2", %{ "hostUrl" => "https://play.dhis2.org" }) end diff --git a/test/lightning/credentials/schema_reconciler_test.exs b/test/lightning/credentials/schema_reconciler_test.exs new file mode 100644 index 00000000000..716ba0305fd --- /dev/null +++ b/test/lightning/credentials/schema_reconciler_test.exs @@ -0,0 +1,74 @@ +defmodule Lightning.Credentials.SchemaReconcilerTest do + # async: false because: + # 1. DataCase uses shared sandbox mode (all processes access DB without allow/3) + # 2. isolated_adaptors stubs Config.default_instance/0 via Mimic globally + use Lightning.DataCase, async: false + + import Eventually + import Lightning.AdaptorTestHelpers + + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor + alias Lightning.Credentials.Credential + alias Lightning.Credentials.SchemaReconciler + + setup :isolated_adaptors + + defp start_reconciler(opts) do + name = :"reconciler_#{System.unique_integer([:positive])}" + start_supervised!({SchemaReconciler, Keyword.put(opts, :name, name)}) + end + + test "sweeps a warm catalogue on start", %{sup: sup} do + seed_adaptor_package("@openfn/language-postgresql", "1.0.0") + credential = insert(:credential, schema: "postgresql") + + start_reconciler(sup: sup) + + assert_eventually( + Repo.get!(Credential, credential.id).schema == + "@openfn/language-postgresql" + ) + end + + test "sweeps again when the catalogue changes after start", %{sup: sup} do + start_reconciler(sup: sup) + + credential = insert(:credential, schema: "postgresql") + + refute Repo.get!(Credential, credential.id).schema == + "@openfn/language-postgresql" + + seed_adaptor_package("@openfn/language-postgresql", "1.0.0") + + Phoenix.PubSub.broadcast!( + Lightning.PubSub, + AdaptorsSupervisor.source_topic(sup), + {:changed, "@openfn/language-postgresql", :npm} + ) + + assert_eventually( + Repo.get!(Credential, credential.id).schema == + "@openfn/language-postgresql", + Lightning.Adaptors.ChannelBroadcaster.debounce_ms() * 4 + ) + end + + test "retries after the sweep raises, without crashing", %{sup: sup} do + test_pid = self() + + pid = + start_reconciler( + sup: sup, + reconcile: fn _sup -> + send(test_pid, :attempt) + raise "boom" + end, + retry_ms: 20 + ) + + assert_receive :attempt + assert_receive :attempt + assert_receive :attempt + assert Process.alive?(pid) + end +end diff --git a/test/lightning/credentials_test.exs b/test/lightning/credentials_test.exs index 1a550673c2e..1564874510c 100644 --- a/test/lightning/credentials_test.exs +++ b/test/lightning/credentials_test.exs @@ -2834,4 +2834,44 @@ defmodule Lightning.CredentialsTest do end end end + + describe "reconcile_legacy_schema_names/1" do + setup :isolated_adaptors + + test "promotes a resolvable short-name row to the full npm name", %{ + sup: sup + } do + Lightning.AdaptorTestHelpers.seed_adaptor_package( + "@openfn/language-postgresql", + "1.0.0" + ) + + credential = insert(:credential, schema: "postgresql") + other = insert(:credential, schema: "postgresql") + + assert Credentials.reconcile_legacy_schema_names(sup) == 2 + + assert Repo.get!(Credential, credential.id).schema == + "@openfn/language-postgresql" + + assert Repo.get!(Credential, other.id).schema == + "@openfn/language-postgresql" + + assert Repo.get!(Credential, credential.id).updated_at == + credential.updated_at + end + + test "leaves raw, oauth, and an unresolvable custom short name untouched", + %{sup: sup} do + raw = insert(:credential, schema: "raw") + oauth = insert(:credential, schema: "oauth") + custom = insert(:credential, schema: "totally-custom") + + assert Credentials.reconcile_legacy_schema_names(sup) == 0 + + assert Repo.get!(Credential, raw.id).schema == "raw" + assert Repo.get!(Credential, oauth.id).schema == "oauth" + assert Repo.get!(Credential, custom.id).schema == "totally-custom" + end + end end diff --git a/test/lightning/metadata_service_test.exs b/test/lightning/metadata_service_test.exs index e607d584a87..3d2125c9a5d 100644 --- a/test/lightning/metadata_service_test.exs +++ b/test/lightning/metadata_service_test.exs @@ -190,13 +190,14 @@ defmodule Lightning.MetadataServiceTest do insert(:credential) |> with_body(%{name: "main", body: %{"username" => "user"}}) - assert MetadataService.fetch("not a valid package!!", credential) == { - :error, - %Lightning.MetadataService.Error{ - type: "no_matching_adaptor", - __exception__: true + assert MetadataService.fetch("not a valid package!!", credential, "main") == + { + :error, + %Lightning.MetadataService.Error{ + type: "no_matching_adaptor", + __exception__: true + } } - } end test "refuses a well-formed adaptor that is not in the registry (whitelist)" do diff --git a/test/lightning_web/live/credential_live_test.exs b/test/lightning_web/live/credential_live_test.exs index b2c9c80da78..ebdd9491352 100644 --- a/test/lightning_web/live/credential_live_test.exs +++ b/test/lightning_web/live/credential_live_test.exs @@ -11,6 +11,8 @@ defmodule LightningWeb.CredentialLiveTest do import Swoosh.TestAssertions alias Lightning.Accounts.User + alias Lightning.Adaptors.Config + alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor alias Lightning.Credentials alias Lightning.Credentials.Credential @@ -837,7 +839,7 @@ defmodule LightningWeb.CredentialLiveTest do open_create_credential_modal(index_live) # Pick a type - index_live |> select_credential_type("dhis2") + index_live |> select_credential_type("@openfn/language-dhis2") index_live |> click_continue() refute index_live |> has_element?("#credential-type-picker") @@ -902,7 +904,7 @@ defmodule LightningWeb.CredentialLiveTest do open_create_credential_modal(index_live) - index_live |> select_credential_type("postgresql") + index_live |> select_credential_type("@openfn/language-postgresql") index_live |> click_continue() refute index_live |> has_element?("#credential-type-picker") @@ -998,7 +1000,7 @@ defmodule LightningWeb.CredentialLiveTest do open_create_credential_modal(index_live) - index_live |> select_credential_type("http") + index_live |> select_credential_type("@openfn/language-http") index_live |> click_continue() assert index_live @@ -1065,7 +1067,7 @@ defmodule LightningWeb.CredentialLiveTest do open_create_credential_modal(view) - select_credential_type(view, "godata") + select_credential_type(view, "@openfn/language-godata") click_continue(view) assert fill_credential(view, %{body: %{email: ""}}) =~ "can't be blank" @@ -1119,7 +1121,7 @@ defmodule LightningWeb.CredentialLiveTest do {:ok, view, _html} = live(conn, ~p"/credentials", on_error: :raise) open_create_credential_modal(view) - select_credential_type(view, "unknownish") + select_credential_type(view, "@openfn/language-unknownish") click_continue(view) refute view |> has_element?("#credential-type-picker") @@ -1137,7 +1139,7 @@ defmodule LightningWeb.CredentialLiveTest do {:ok, view, _html} = live(conn, ~p"/credentials", on_error: :raise) open_create_credential_modal(view) - select_credential_type(view, "dhis2") + select_credential_type(view, "@openfn/language-dhis2") click_continue(view) html = view |> element("#credential-form-new") |> render() @@ -2761,18 +2763,48 @@ defmodule LightningWeb.CredentialLiveTest do html_tree = Floki.parse_document!(html) for adaptor <- ["postgresql", "dhis2", "http"] do + full_name = "@openfn/language-#{adaptor}" + adaptor_label = Floki.find( html_tree, - "label[for='credential-schema-picker_selected_#{adaptor}']" + "label[for='credential-schema-picker_selected_#{full_name}']" ) adaptor_icon = Floki.find(adaptor_label, "object") assert length(adaptor_icon) > 0 img_src = adaptor_icon |> Floki.attribute("data") |> List.first() - assert img_src =~ "/adaptors/icons/#{adaptor}/square-" + + assert img_src =~ + "/adaptors/icons/#{URI.encode(full_name, &URI.char_unreserved?/1)}/square-" end end + + test "omits a deprecated adaptor from the type options", %{conn: conn} do + insert(:adaptor, name: "deprecated-adaptor", deprecated: true) + + # `seed_all_credential_schemas/0` primes the packages cache by hand + # (bypassing `Catalogue.list_package_metas/1`), so drop it here to + # force a fresh DB-backed read that can see the row above. + cache = AdaptorsSupervisor.cache_name(Config.default_instance()) + source = AdaptorsSupervisor.source(Config.default_instance()) + Cachex.del(cache, {:packages, source}) + + {:ok, view, _html} = live(conn, ~p"/credentials") + + html = open_create_credential_modal(view) + html_tree = Floki.parse_document!(html) + + assert Floki.find( + html_tree, + "label[for='credential-schema-picker_selected_http']" + ) != [] + + assert Floki.find( + html_tree, + "label[for='credential-schema-picker_selected_deprecated-adaptor']" + ) == [] + end end describe "generic oauth credential" do @@ -3792,7 +3824,7 @@ defmodule LightningWeb.CredentialLiveTest do {:ok, view, _html} = live(conn, ~p"/credentials", on_error: :raise) open_create_credential_modal(view) - select_credential_type(view, "dhis2") + select_credential_type(view, "@openfn/language-dhis2") click_continue(view) # Fill in values for the main environment diff --git a/test/lightning_web/live/project_live_test.exs b/test/lightning_web/live/project_live_test.exs index 7e183670167..d528431d146 100644 --- a/test/lightning_web/live/project_live_test.exs +++ b/test/lightning_web/live/project_live_test.exs @@ -1005,7 +1005,7 @@ defmodule LightningWeb.ProjectLiveTest do view |> element("#new-credential-option-menu-item") |> render_click() - view |> select_credential_type("http") + view |> select_credential_type("@openfn/language-http") view |> click_continue() assert view @@ -1053,7 +1053,7 @@ defmodule LightningWeb.ProjectLiveTest do ) view |> element("#new-credential-option-menu-item") |> render_click() - view |> select_credential_type("http") + view |> select_credential_type("@openfn/language-http") view |> click_continue() # Only the active sandbox is pre-selected. Ancestors are attached at @@ -1103,7 +1103,7 @@ defmodule LightningWeb.ProjectLiveTest do view |> element("#new-credential-option-menu-item") |> render_click() - view |> select_credential_type("http") + view |> select_credential_type("@openfn/language-http") view |> click_continue() assert view diff --git a/test/lightning_web/plugs/channel_proxy_plug_test.exs b/test/lightning_web/plugs/channel_proxy_plug_test.exs index 06675921c9c..43d081963f0 100644 --- a/test/lightning_web/plugs/channel_proxy_plug_test.exs +++ b/test/lightning_web/plugs/channel_proxy_plug_test.exs @@ -1002,7 +1002,11 @@ defmodule LightningWeb.ChannelProxyPlugTest do user = insert(:user) credential = - insert(:credential, schema: "http", name: "destination-cred", user: user) + insert(:credential, + schema: "@openfn/language-http", + name: "destination-cred", + user: user + ) |> with_body(%{body: %{"access_token" => "dest-token-xyz"}}) project_credential = @@ -1122,7 +1126,11 @@ defmodule LightningWeb.ChannelProxyPlugTest do user = insert(:user) credential = - insert(:credential, schema: "http", name: "destination-cred", user: user) + insert(:credential, + schema: "@openfn/language-http", + name: "destination-cred", + user: user + ) |> with_body(%{body: %{"access_token" => "dest-token-xyz"}}) project_credential = @@ -1220,7 +1228,7 @@ defmodule LightningWeb.ChannelProxyPlugTest do test "Bearer token sent to upstream when channel has http credential with access_token", %{bypass: bypass} do channel = - create_destination_auth_channel(bypass, "http", %{ + create_destination_auth_channel(bypass, "@openfn/language-http", %{ "access_token" => "tok-123" }) @@ -1240,7 +1248,7 @@ defmodule LightningWeb.ChannelProxyPlugTest do test "Basic auth sent when channel has http credential with username/password", %{bypass: bypass} do channel = - create_destination_auth_channel(bypass, "http", %{ + create_destination_auth_channel(bypass, "@openfn/language-http", %{ "username" => "u", "password" => "p" }) @@ -1263,7 +1271,7 @@ defmodule LightningWeb.ChannelProxyPlugTest do test "ApiToken sent when channel has dhis2 credential with pat", %{bypass: bypass} do channel = - create_destination_auth_channel(bypass, "dhis2", %{ + create_destination_auth_channel(bypass, "@openfn/language-dhis2", %{ "pat" => "d2pat_abc" }) @@ -1298,7 +1306,7 @@ defmodule LightningWeb.ChannelProxyPlugTest do test "authorization header redacted in persisted ChannelEvent", %{bypass: bypass} do channel = - create_destination_auth_channel(bypass, "http", %{ + create_destination_auth_channel(bypass, "@openfn/language-http", %{ "access_token" => "secret-token" }) @@ -1336,7 +1344,11 @@ defmodule LightningWeb.ChannelProxyPlugTest do user = insert(:user) credential = - insert(:credential, schema: "http", name: "no-body", user: user) + insert(:credential, + schema: "@openfn/language-http", + name: "no-body", + user: user + ) # Don't call with_body — no CredentialBody exists @@ -1392,7 +1404,11 @@ defmodule LightningWeb.ChannelProxyPlugTest do user = insert(:user) credential = - insert(:credential, schema: "http", name: "shared", user: user) + insert(:credential, + schema: "@openfn/language-http", + name: "shared", + user: user + ) |> with_body(%{ name: "main", body: %{"username" => "prod", "password" => "prod-secret"} @@ -1446,7 +1462,11 @@ defmodule LightningWeb.ChannelProxyPlugTest do user = insert(:user) credential = - insert(:credential, schema: "http", name: "parent", user: user) + insert(:credential, + schema: "@openfn/language-http", + name: "parent", + user: user + ) project_credential = insert(:project_credential, project: sandbox, credential: credential) @@ -1490,7 +1510,7 @@ defmodule LightningWeb.ChannelProxyPlugTest do test "credential with missing auth fields returns 502 with observable error", %{bypass: bypass} do channel = - create_destination_auth_channel(bypass, "http", %{ + create_destination_auth_channel(bypass, "@openfn/language-http", %{ "baseUrl" => "https://example.com" }) @@ -1519,7 +1539,7 @@ defmodule LightningWeb.ChannelProxyPlugTest do test "proxy headers (x-forwarded-*) still forwarded alongside auth header", %{bypass: bypass} do channel = - create_destination_auth_channel(bypass, "http", %{ + create_destination_auth_channel(bypass, "@openfn/language-http", %{ "access_token" => "tok-with-proxy" }) @@ -1690,7 +1710,7 @@ defmodule LightningWeb.ChannelProxyPlugTest do test "persists destination_credential_id on successful proxy with destination auth", %{bypass: bypass} do channel = - create_destination_auth_channel(bypass, "http", %{ + create_destination_auth_channel(bypass, "@openfn/language-http", %{ "access_token" => "tok-123" }) @@ -1727,7 +1747,11 @@ defmodule LightningWeb.ChannelProxyPlugTest do user = insert(:user) credential = - insert(:credential, schema: "http", name: "bad-cred", user: user) + insert(:credential, + schema: "@openfn/language-http", + name: "bad-cred", + user: user + ) |> with_body(%{body: %{"baseUrl" => "https://example.com"}}) project_credential = @@ -1944,7 +1968,11 @@ defmodule LightningWeb.ChannelProxyPlugTest do user = insert(:user) credential = - insert(:credential, schema: "http", name: "no-body", user: user) + insert(:credential, + schema: "@openfn/language-http", + name: "no-body", + user: user + ) # Don't call with_body — no CredentialBody exists, so credential # resolution will fail and `record_credential_error/3` is invoked. @@ -2005,7 +2033,11 @@ defmodule LightningWeb.ChannelProxyPlugTest do user = insert(:user) credential = - insert(:credential, schema: "http", name: "no-body", user: user) + insert(:credential, + schema: "@openfn/language-http", + name: "no-body", + user: user + ) project_credential = insert(:project_credential, diff --git a/test/mix/tasks/lightning.adaptors.snapshot_test.exs b/test/mix/tasks/lightning.adaptors.snapshot_test.exs index 9686649a002..15c4a6f5d28 100644 --- a/test/mix/tasks/lightning.adaptors.snapshot_test.exs +++ b/test/mix/tasks/lightning.adaptors.snapshot_test.exs @@ -48,6 +48,10 @@ defmodule Mix.Tasks.Lightning.Adaptors.SnapshotTest do tmp_dir: tmp_dir, registry: registry } do + Bypass.expect(registry, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{}) + end) + Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> json_resp(conn, 200, %{"objects" => []}) end) @@ -67,6 +71,10 @@ defmodule Mix.Tasks.Lightning.Adaptors.SnapshotTest do registry: registry, jsdelivr: jsdelivr } do + Bypass.expect(registry, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{@package => "write"}) + end) + Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> json_resp(conn, 200, %{ "objects" => [ diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex index e5edce68c35..17d89b7ec83 100644 --- a/test/support/adaptor_test_helpers.ex +++ b/test/support/adaptor_test_helpers.ex @@ -153,13 +153,17 @@ defmodule Lightning.AdaptorTestHelpers do |> File.read!() row = - insert(:adaptor, name: short_name, source: :npm, schema_data: schema_body) + insert(:adaptor, + name: "@openfn/language-#{short_name}", + source: :npm, + schema_data: schema_body + ) # Cachex fills run in its Courier process, which cannot see the sandbox # connection, so populate the cache directly. cache = AdaptorsSupervisor.cache_name(Config.default_instance()) source = AdaptorsSupervisor.source(Config.default_instance()) - Cachex.put(cache, {:schema, short_name, source}, {:ok, schema_body}) + Cachex.put(cache, {:schema, row.name, source}, {:ok, schema_body}) row end @@ -179,7 +183,7 @@ defmodule Lightning.AdaptorTestHelpers do row = seed_credential_schema(short_name) %{ - name: short_name, + name: row.name, latest_version: row.latest_version, description: nil, deprecated: false, diff --git a/test/support/credential_live_helpers.ex b/test/support/credential_live_helpers.ex index 09e1edef642..d2ca1b5f573 100644 --- a/test/support/credential_live_helpers.ex +++ b/test/support/credential_live_helpers.ex @@ -41,7 +41,7 @@ defmodule LightningWeb.CredentialLiveHelpers do assert html |> Floki.parse_fragment!() - |> Floki.find("input[type=radio][value=#{type}]") + |> Floki.find(~s(input[type=radio][value="#{type}"])) |> Enum.any?(), "Expected #{type} to be selected" end diff --git a/tooling/adaptor_cache/README.md b/tooling/adaptor_cache/README.md index 9d9714594f6..18cae53d266 100644 --- a/tooling/adaptor_cache/README.md +++ b/tooling/adaptor_cache/README.md @@ -24,7 +24,7 @@ Bandit + Req the first time it runs). ## Recorded responses have no TTL -A recorded response is authoritative until you `purge` it — there is no expiry. +A recorded response is authoritative until you `purge` it: there is no expiry. That's deliberate: the recorded files double as hand-editable fixtures, so the cache is both the everyday dev cache and the mechanism for driving the `publish` scenarios below, by editing exactly the files it already wrote. @@ -42,12 +42,12 @@ export ADAPTORS_NPM_GITHUB_URL=http://localhost:4874/github `bin/adaptor_cache up` prints these for you with the right port baked in, so you don't have to remember them. -- `ADAPTOR_CACHE_PORT` — host port to bind (default: `4874`). Set it before any +- `ADAPTOR_CACHE_PORT`: host port to bind (default: `4874`). Set it before any `bin/adaptor_cache` command if `4874` is taken, and update the three exports above to match. -- `ADAPTOR_CACHE_DIR` — where recorded responses live (default: +- `ADAPTOR_CACHE_DIR`: where recorded responses live (default: `/tmp/adaptor_cache`). Several distros age-clean `/tmp` (systemd-tmpfiles: 10 - days on Fedora/Arch) — if a fixture goes missing for no obvious reason, that's + days on Fedora/Arch). If a fixture goes missing for no obvious reason, that's likely it. Set this to somewhere outside `/tmp` if you want the cache to survive indefinitely. @@ -78,17 +78,17 @@ Each line is one proxied request: 2026-08-31T10:00:00Z status=200 cache=HIT GET /npm/-/v1/search?text=%40openfn&size=250 ``` -- `cache=HIT` — served entirely from disk, no upstream request made. -- `cache=MISS` — not recorded yet, fetched from the real upstream and saved. -- `cache=ERROR` — the live fetch itself failed (offline, upstream down); nothing +- `cache=HIT`: served entirely from disk, no upstream request made. +- `cache=MISS`: not recorded yet, fetched from the real upstream and saved. +- `cache=ERROR`: the live fetch itself failed (offline, upstream down); nothing is recorded, so the next attempt tries live again. On a warm cache, MISS should only appear for packages the cache has never seen. -## How the URL mapping works +## URL mapping Lightning's `NPM` strategy already builds full paths under each of the three -base URLs — the proxy fetches the same path from the real upstream and caches it +base URLs. The proxy fetches the same path from the real upstream and caches it under the _original_ request path, following any redirect itself first, so the cached entry reflects the final resolved resource, not an intermediate redirect: @@ -101,12 +101,12 @@ cached entry reflects the final resolved resource, not an intermediate redirect: ## Recorded files as fixtures A recorded response is two files: the raw body, plus a `.meta` sidecar with its -status and content type. The path mirrors the request, so — for example — the +status and content type. The path mirrors the request. For example, the `@openfn/language-http` packument lands at `/tmp/adaptor_cache/npm/@openfn/language-http`, and the search response (the one query Lightning ever sends) lands at `/tmp/adaptor_cache/npm/-/v1/search?text=%40openfn&size=250`. Both are plain -JSON — open and edit them directly to hand-craft a scenario. +JSON, so open and edit them directly to hand-craft a scenario. ## Driving both `publish` scenarios @@ -116,7 +116,7 @@ bin/adaptor_cache publish @openfn/language-http 9.9.9 # new version of a ``` Either form updates the packument _and_ the search response's `latest_version` -together in one call — `scheduler.ex`'s change-detection compares the search +together in one call. `scheduler.ex`'s change-detection compares the search response against the DB to decide whether to bother fetching the packument at all, so updating only one is a silent no-op. Run `mix lightning.adaptors.refresh` (or reopen the picker) afterwards to see it @@ -145,7 +145,7 @@ untracked (not checked into git). ## Troubleshooting **`bin/adaptor_cache check` fails on one prefix.** Run `bin/adaptor_cache logs` -and look for the failing request — a `cache=MISS` on the _second_ identical +and look for the failing request. A `cache=MISS` on the _second_ identical request usually means the upstream is refusing the request outright (check status code) rather than a caching problem. From cd7ee6a8d3951011d888860aab3b2db5d046a5fd Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Tue, 8 Sep 2026 08:35:53 +0200 Subject: [PATCH 11/37] Stop the adaptors supervisor crash-looping on a bad boot-time DB read - Scheduler's boot-time DB read no longer crash-loops the adaptors supervisor; boot-tick rescue narrowed to connection errors only - Sandboxed CSP headers added to adaptor icon responses - Duplicate adaptor version rows deduped before insert - 503 returned instead of crashing when the adaptor store is unavailable - Adaptors.catalogue_with_stamp/1 renamed to catalogue/1 - 503 controller test stubs Adaptors instead of Store --- lib/lightning/adaptors.ex | 20 +++--- lib/lightning/adaptors/catalogue.ex | 28 ++++++-- lib/lightning/adaptors/scheduler.ex | 67 +++++++++++++------ .../controllers/adaptor_controller.ex | 40 +++++++---- .../controllers/adaptor_icon_controller.ex | 1 + lib/lightning_web/plugs/channel_proxy_plug.ex | 2 +- lib/lightning_web/utils.ex | 20 ++++++ test/lightning/adaptors/catalogue_test.exs | 17 +++++ test/lightning/adaptors/local_test.exs | 11 +++ test/lightning/adaptors/scheduler_test.exs | 48 +++++++++++-- .../controllers/adaptor_controller_test.exs | 18 ++++- .../adaptor_icon_controller_test.exs | 33 +++++++++ test/test_helper.exs | 1 + 13 files changed, 253 insertions(+), 53 deletions(-) diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index e0375f1e250..3aed31cd9fb 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -152,17 +152,21 @@ defmodule Lightning.Adaptors do do: Store.icon(sup, pkg, shape) @doc """ - Returns the picker catalogue as `{{latest_updated_at, count}, entries}`: - every adaptor with its full version list and icon URLs, rendered once - per change rather than per request, alongside the ETag basis for it. + Returns `{:ok, {{latest_updated_at, count}, entries}}`: every adaptor + with its full version list and icon URLs, rendered once per change + rather than per request, alongside the ETag basis for it. One read, so the stamp always describes the entries it comes with. + + Returns `{:error, term()}` unchanged from `Store.catalogue/1` on a + backing-store failure; callers must handle it. """ - @spec catalogue_with_stamp(atom()) :: - {{DateTime.t() | nil, non_neg_integer()}, [Store.catalogue_entry()]} - def catalogue_with_stamp(sup \\ Config.default_instance()) do - {:ok, catalogue} = Store.catalogue(sup) - catalogue + @spec catalogue(atom()) :: + {:ok, + {{DateTime.t() | nil, non_neg_integer()}, [Store.catalogue_entry()]}} + | {:error, term()} + def catalogue(sup \\ Config.default_instance()) do + Store.catalogue(sup) end @doc """ diff --git a/lib/lightning/adaptors/catalogue.ex b/lib/lightning/adaptors/catalogue.ex index 8a349aed3db..909dc86957f 100644 --- a/lib/lightning/adaptors/catalogue.ex +++ b/lib/lightning/adaptors/catalogue.ex @@ -15,6 +15,8 @@ defmodule Lightning.Adaptors.Catalogue do alias Lightning.Adaptors.Catalogue.AdaptorVersion alias Lightning.Repo + require Logger + @type source :: :npm | :local @type package_meta :: %{ @@ -391,11 +393,11 @@ defmodule Lightning.Adaptors.Catalogue do defp build_version_rows(adaptor_id, records, now) do records + |> Enum.map(&stringify_keys/1) + |> warn_duplicate_versions(adaptor_id) + |> Enum.uniq_by(&Map.get(&1, "version")) |> Enum.reduce_while({:ok, []}, fn record, {:ok, acc} -> - attrs = - record - |> stringify_keys() - |> Map.put("adaptor_id", adaptor_id) + attrs = Map.put(record, "adaptor_id", adaptor_id) changeset = AdaptorVersion.changeset(%AdaptorVersion{}, attrs) @@ -411,6 +413,24 @@ defmodule Lightning.Adaptors.Catalogue do end end + defp warn_duplicate_versions(records, adaptor_id) do + duplicated = + records + |> Enum.frequencies_by(&Map.get(&1, "version")) + |> Enum.filter(fn {_version, count} -> count > 1 end) + |> Enum.map(fn {version, _count} -> version end) + + if duplicated != [] do + Logger.warning( + "Lightning.Adaptors.Catalogue: duplicate version row(s) for adaptor " <> + "#{inspect(adaptor_id)}, first occurrence wins: " <> + Enum.join(duplicated, ", ") + ) + end + + records + end + # `Ecto.Changeset.cast/3` raises on a map mixing atom and string keys, so # every map handed to a changeset here is flattened to string keys first — # that is what a JSON snapshot gives us, and what atom-keyed callers diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 9ec6b8f5b80..fef787b7ef0 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -30,7 +30,10 @@ defmodule Lightning.Adaptors.Scheduler do @doc """ Starts the Scheduler. Required opts: `:name`, `:sup`, `:lock_key`, - `:cache`, `:tasks`, `:source_topic`. + `:cache`, `:tasks`, `:source_topic`. Optional: `:checked_at` (1-arity fn, + default `&Catalogue.max_checked_at/1`) reads the source's last-checked + timestamp; called once at boot to schedule the delay before the scheduler's + initial tick. """ @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts) do @@ -109,36 +112,56 @@ defmodule Lightning.Adaptors.Scheduler do source_topic = Keyword.fetch!(opts, :source_topic) cache = Keyword.fetch!(opts, :cache) tasks = Keyword.fetch!(opts, :tasks) + checked_at = Keyword.get(opts, :checked_at, &Catalogue.max_checked_at/1) source = AdaptorsSupervisor.source(sup) interval_ms = Config.refresh_interval() - if interval_ms > 0 do - delay = - time_until_next_ms(Catalogue.max_checked_at(source), interval_ms) - - Process.send_after(self(), :tick, delay) + state = %{ + sup: sup, + source: source, + interval_ms: interval_ms, + source_topic: source_topic, + cache: cache, + tasks: tasks, + checked_at: checked_at, + refresh: nil, + waiters: [], + package_refreshes: %{}, + icon_refreshes: %{} + } - Logger.info( - "Adaptors[#{source}]: scheduler started interval=#{interval_ms}ms next_tick_in=#{delay}ms" - ) + if interval_ms > 0 do + {:ok, state, {:continue, :schedule_first_tick}} else Logger.info("Adaptors[#{source}]: scheduler started interval=0 (disabled)") + {:ok, state} end + end + + @impl true + def handle_continue(:schedule_first_tick, state) do + delay = first_tick_delay(state) + Process.send_after(self(), :tick, delay) + + Logger.info( + "Adaptors[#{state.source}]: scheduler started interval=#{state.interval_ms}ms " <> + "next_tick_in=#{delay}ms" + ) + + {:noreply, state} + end + + defp first_tick_delay(state) do + time_until_next_ms(state.checked_at.(state.source), state.interval_ms) + rescue + e in DBConnection.ConnectionError -> + Logger.warning( + "Adaptors[#{state.source}]: scheduler could not read max_checked_at, " <> + "ticking immediately: #{Exception.message(e)}" + ) - {:ok, - %{ - sup: sup, - source: source, - interval_ms: interval_ms, - source_topic: source_topic, - cache: cache, - tasks: tasks, - refresh: nil, - waiters: [], - package_refreshes: %{}, - icon_refreshes: %{} - }} + 0 end @impl true diff --git a/lib/lightning_web/controllers/adaptor_controller.ex b/lib/lightning_web/controllers/adaptor_controller.ex index ceb5f15f5af..0f14b907684 100644 --- a/lib/lightning_web/controllers/adaptor_controller.ex +++ b/lib/lightning_web/controllers/adaptor_controller.ex @@ -12,20 +12,34 @@ defmodule LightningWeb.AdaptorController do alias Lightning.Adaptors + require Logger + def index(conn, _params) do - {stamp, entries} = Adaptors.catalogue_with_stamp() - etag = etag_for(stamp) - - conn = - conn - |> put_resp_header("etag", etag) - |> put_resp_header("cache-control", "private, no-cache") - |> put_resp_header("vary", "Cookie") - - if get_req_header(conn, "if-none-match") == [etag] do - send_resp(conn, 304, "") - else - json(conn, %{data: entries}) + case Adaptors.catalogue() do + {:ok, {stamp, entries}} -> + etag = etag_for(stamp) + + conn = + conn + |> put_resp_header("etag", etag) + |> put_resp_header("cache-control", "private, no-cache") + |> put_resp_header("vary", "Cookie") + + if get_req_header(conn, "if-none-match") == [etag] do + send_resp(conn, 304, "") + else + json(conn, %{data: entries}) + end + + {:error, reason} -> + Logger.warning( + "LightningWeb.AdaptorController: adaptor catalogue unavailable: " <> + inspect(reason) + ) + + conn + |> put_status(:service_unavailable) + |> json(%{"error" => "adaptor catalogue unavailable"}) end end diff --git a/lib/lightning_web/controllers/adaptor_icon_controller.ex b/lib/lightning_web/controllers/adaptor_icon_controller.ex index ca89a949a97..55d33f56c0c 100644 --- a/lib/lightning_web/controllers/adaptor_icon_controller.ex +++ b/lib/lightning_web/controllers/adaptor_icon_controller.ex @@ -114,6 +114,7 @@ defmodule LightningWeb.AdaptorIconController do conn |> put_resp_content_type(content_type_for(ext)) |> put_resp_header("cache-control", @immutable_cache) + |> merge_resp_headers(LightningWeb.Utils.sandboxed_asset_headers()) |> send_file(200, path) {:error, _} -> diff --git a/lib/lightning_web/plugs/channel_proxy_plug.ex b/lib/lightning_web/plugs/channel_proxy_plug.ex index 1004991b5e3..b88ba73adf9 100644 --- a/lib/lightning_web/plugs/channel_proxy_plug.ex +++ b/lib/lightning_web/plugs/channel_proxy_plug.ex @@ -38,7 +38,7 @@ defmodule LightningWeb.ChannelProxyPlug do @proxy_security_headers [ {"content-security-policy", - "default-src 'none'; sandbox; frame-ancestors 'none'"}, + LightningWeb.Utils.sandbox_csp() <> "; frame-ancestors 'none'"}, {"x-content-type-options", "nosniff"}, {"x-frame-options", "DENY"}, {"referrer-policy", "no-referrer"} diff --git a/lib/lightning_web/utils.ex b/lib/lightning_web/utils.ex index a0895f3c99c..6b523cf0985 100644 --- a/lib/lightning_web/utils.ex +++ b/lib/lightning_web/utils.ex @@ -165,4 +165,24 @@ defmodule LightningWeb.Utils do if halt?, do: Plug.Conn.halt(conn), else: conn end + + @doc """ + CSP value that blocks script execution and active/plugin content for + untrusted bytes served same-origin. + """ + @spec sandbox_csp() :: String.t() + def sandbox_csp, do: "default-src 'none'; sandbox" + + @doc """ + Response headers for serving untrusted static assets (e.g. adaptor icons) + same-origin without risking script execution if loaded as a top-level + document. + """ + @spec sandboxed_asset_headers() :: [{String.t(), String.t()}] + def sandboxed_asset_headers do + [ + {"content-security-policy", sandbox_csp()}, + {"x-content-type-options", "nosniff"} + ] + end end diff --git a/test/lightning/adaptors/catalogue_test.exs b/test/lightning/adaptors/catalogue_test.exs index 105a51be6d8..3c069106065 100644 --- a/test/lightning/adaptors/catalogue_test.exs +++ b/test/lightning/adaptors/catalogue_test.exs @@ -521,6 +521,23 @@ defmodule Lightning.Adaptors.CatalogueTest do end end + describe "upsert_adaptor/1 — duplicate version rows" do + test "dedupes records sharing the same version, keeping the first" do + record = + adaptor_record( + versions: [ + version_record("1.0.0", size_bytes: 111), + version_record("1.0.0", size_bytes: 222) + ] + ) + + assert {:ok, adaptor} = Catalogue.upsert_adaptor(record) + + assert [%AdaptorVersion{version: "1.0.0", size_bytes: 111}] = + Catalogue.list_versions(adaptor.name, :npm) + end + end + defp adaptor_record(overrides \\ []) do overrides = Map.new(overrides) diff --git a/test/lightning/adaptors/local_test.exs b/test/lightning/adaptors/local_test.exs index 8164a1403ec..3a37d8fa4f6 100644 --- a/test/lightning/adaptors/local_test.exs +++ b/test/lightning/adaptors/local_test.exs @@ -60,6 +60,17 @@ defmodule Lightning.Adaptors.LocalTest do refute log =~ "shadowed" end + test "two same-root directories sharing name and version both surface as candidate rows", + %{root: root} do + write_package!(root, "http-a", "@openfn/language-http", "1.0.0") + write_package!(root, "http-b", "@openfn/language-http", "1.0.0") + + assert {:ok, %{name: "@openfn/language-http", versions: versions}} = + Local.fetch_adaptor("@openfn/language-http") + + assert [%{version: "1.0.0"}, %{version: "1.0.0"}] = versions + end + test "skips a directory with a missing package.json and logs a warning", %{root: root} do write_package!(root, "good", "@openfn/language-good", "1.0.0") diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index 416182a164e..f770c0880ba 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -68,16 +68,17 @@ defmodule Lightning.Adaptors.SchedulerTest do :ok = Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) - pid = - start_supervised!({ - Scheduler, + scheduler_opts = + [ name: global_name, sup: sup, lock_key: AdaptorsSupervisor.lock_key(sup), cache: AdaptorsSupervisor.cache_name(sup), tasks: AdaptorsSupervisor.tasks_name(sup), source_topic: source_topic - }) + ] ++ Keyword.take(opts, [:checked_at]) + + pid = start_supervised!({Scheduler, scheduler_opts}) Application.put_env(:lightning, Lightning.Adaptors, original_env) @@ -195,6 +196,45 @@ defmodule Lightning.Adaptors.SchedulerTest do end end + describe "boot resilience" do + test "a DB error reading max_checked_at does not crash the scheduler, and it ticks immediately", + %{sup: sup} do + test_pid = self() + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :list_adaptors_called) + {:ok, []} + end) + + start_scheduler(sup, + checked_at: fn _source -> + raise DBConnection.ConnectionError, "down" + end + ) + + assert_receive :list_adaptors_called, 2000 + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + {:global, gname} = sched_name + assert is_pid(:global.whereis_name(gname)) + end + + test "a Postgrex.Error reading max_checked_at is not rescued and crashes the scheduler", + %{sup: sup} do + pid = + start_scheduler(sup, + checked_at: fn _source -> + raise Postgrex.Error, message: "undefined_column" + end + ) + + ref = Process.monitor(pid) + + assert_receive {:DOWN, ^ref, :process, ^pid, reason}, 2000 + assert {%Postgrex.Error{}, _stacktrace} = reason + end + end + describe "do_refresh/1 diff logic" do test "unchanged adaptor: touch_checked_at only, no upsert, no broadcast", %{ sup: sup diff --git a/test/lightning_web/controllers/adaptor_controller_test.exs b/test/lightning_web/controllers/adaptor_controller_test.exs index fd9298acc5a..554b03b9b5c 100644 --- a/test/lightning_web/controllers/adaptor_controller_test.exs +++ b/test/lightning_web/controllers/adaptor_controller_test.exs @@ -2,11 +2,15 @@ defmodule LightningWeb.AdaptorControllerTest do use LightningWeb.ConnCase, async: true import Lightning.Factories + import Mimic - alias Lightning.AdaptorTestHelpers + alias Lightning.Adaptors alias Lightning.Adaptors.Catalogue + alias Lightning.AdaptorTestHelpers alias LightningWeb.AdaptorIconURL + setup :verify_on_exit! + describe "GET /adaptors/catalogue" do # The production cache outlives the SQL sandbox, so an entry another # test committed would otherwise be served here. @@ -130,6 +134,18 @@ defmodule LightningWeb.AdaptorControllerTest do assert json_response(conn, 401) == %{"error" => "Unauthorized"} end + test "returns a 503 with a JSON body when the store fails", %{conn: conn} do + conn = log_in_user(conn, insert(:user)) + + stub(Adaptors, :catalogue, fn -> {:error, :unavailable} end) + + conn = get(conn, ~p"/adaptors/catalogue") + + assert json_response(conn, 503) == %{ + "error" => "adaptor catalogue unavailable" + } + end + defp version_record(version) do %{ version: version, diff --git a/test/lightning_web/controllers/adaptor_icon_controller_test.exs b/test/lightning_web/controllers/adaptor_icon_controller_test.exs index bf7712f689c..78af0a5a6d4 100644 --- a/test/lightning_web/controllers/adaptor_icon_controller_test.exs +++ b/test/lightning_web/controllers/adaptor_icon_controller_test.exs @@ -135,6 +135,39 @@ defmodule LightningWeb.AdaptorIconControllerTest do assert ct =~ "image/svg+xml" end + test "sets sandboxed CSP and nosniff headers on svg response", %{ + conn: conn + } do + name = unique_adaptor_name() + bytes = "" + sha256 = :crypto.hash(:sha256, bytes) + sha8 = sha256 |> binary_part(0, 4) |> Base.encode16(case: :lower) + + insert_adaptor(name, %{ + icon_square_ext: "svg", + icon_square_sha256: sha256 + }) + + write_icon(name, :square, "svg", bytes) + + params = %{ + "name" => name, + "shape" => "square", + "sha8" => sha8, + "ext" => "svg" + } + + result = AdaptorIconController.show(conn, params) + + assert result.status == 200 + + assert get_resp_header(result, "content-security-policy") == [ + "default-src 'none'; sandbox" + ] + + assert get_resp_header(result, "x-content-type-options") == ["nosniff"] + end + test "sha8 is case-insensitive on input", %{conn: conn} do name = unique_adaptor_name() bytes = "case test bytes" diff --git a/test/test_helper.exs b/test/test_helper.exs index 942b070c13f..741b6b2ce49 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -16,6 +16,7 @@ Mox.defmock(Lightning.AdaptorService.RepoMock, Mimic.copy(:hackney) Mimic.copy(File) Mimic.copy(IO) +Mimic.copy(Lightning.Adaptors) Mimic.copy(Lightning.Adaptors.Config) Mimic.copy(Lightning.FailureEmail) Mimic.copy(Lightning.Projects.Provisioner) From b132c714778fbf025e02af3724d289601569049b Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Mon, 7 Sep 2026 17:55:03 +0200 Subject: [PATCH 12/37] Fix refresh_now's leaked tick chain, tighten per-adaptor fetch timeouts - refresh_now no longer leaks a recurring tick chain - Icon-only upstream changes persisted even without a version bump - Per-adaptor fetch timeout matched to the strategy's own HTTP budget, proven from config without a 6.5s sleep - Comments reworded - Persisted schema kept when a jsDelivr fetch fails transiently - Icon bytes verified against the catalogue sha before trusting the disk cache --- lib/lightning/adaptors/npm.ex | 40 +++-- lib/lightning/adaptors/npm/schema.ex | 20 ++- lib/lightning/adaptors/scheduler.ex | 49 +++---- lib/lightning/adaptors/store.ex | 35 ++++- lib/lightning/adaptors/strategy.ex | 20 +-- test/lightning/adaptors/npm/schema_test.exs | 16 +- test/lightning/adaptors/npm_test.exs | 15 +- test/lightning/adaptors/scheduler_test.exs | 154 ++++++++++++++++++++ test/lightning/adaptors/store_test.exs | 55 +++++++ 9 files changed, 325 insertions(+), 79 deletions(-) diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index a5e99dd7295..958816bce09 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -68,24 +68,34 @@ defmodule Lightning.Adaptors.NPM do def fetch_adaptor(name) when is_binary(name) do with {:ok, packument} <- Registry.get_packument(name), {:ok, latest_version} <- Registry.latest_version(packument) do - {schema_data, schema_sha} = Schema.schema(name, latest_version) - - {:ok, - %{ - name: Map.get(packument, "name", name), - description: Map.get(packument, "description"), - homepage: Map.get(packument, "homepage"), - repository: Registry.repository_url(Map.get(packument, "repository")), - license: Map.get(packument, "license"), - latest_version: latest_version, - deprecated: Registry.deprecated?(packument, latest_version), - schema_data: encode_schema(schema_data), - schema_sha256: schema_sha, - versions: Registry.build_versions(packument) - }} + base = %{ + name: Map.get(packument, "name", name), + description: Map.get(packument, "description"), + homepage: Map.get(packument, "homepage"), + repository: Registry.repository_url(Map.get(packument, "repository")), + license: Map.get(packument, "license"), + latest_version: latest_version, + deprecated: Registry.deprecated?(packument, latest_version), + versions: Registry.build_versions(packument) + } + + {:ok, put_schema(base, Schema.schema(name, latest_version))} end end + # A transient schema-fetch failure must leave `schema_data`/`schema_sha256` + # absent from the record entirely, not merely `nil` — `Ecto.Changeset.cast/3` + # overwrites a column whenever its key is present in `attrs`, even with a + # nil value, so an absent key is the only way to signal "leave the + # previously-persisted schema untouched." + defp put_schema(record, {nil, :fetch_failed}), do: record + + defp put_schema(record, {schema_data, schema_sha}) do + record + |> Map.put(:schema_data, encode_schema(schema_data)) + |> Map.put(:schema_sha256, schema_sha) + end + # Strategy boundary: re-encode the decoded schema map to a JSON binary # so the row is persisted as text and `Jason.decode!(_, # objects: :ordered_objects)` re-engages downstream. `Schema.schema/2` diff --git a/lib/lightning/adaptors/npm/schema.ex b/lib/lightning/adaptors/npm/schema.ex index 8975888e559..b694db79be3 100644 --- a/lib/lightning/adaptors/npm/schema.ex +++ b/lib/lightning/adaptors/npm/schema.ex @@ -3,9 +3,11 @@ defmodule Lightning.Adaptors.NPM.Schema do jsDelivr CDN client for adaptor configuration schemas. Fetches `/npm/@/configuration-schema.json` from - `cdn.jsdelivr.net`, decodes it, and returns - `{schema_data, schema_sha256}` (or `{nil, nil}` on any failure — - schema fetch is best-effort). + `cdn.jsdelivr.net`, decodes it, and returns `{schema_data, + schema_sha256}`. A genuine 404 (schema removed upstream) returns + `{nil, nil}`; any other failure (timeout, other HTTP status, network + error) returns `{nil, :fetch_failed}` so callers can tell "schema + really doesn't exist" apart from "couldn't check right now." Base URL via `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)[:jsdelivr_url]`, default `https://cdn.jsdelivr.net`. @@ -19,19 +21,21 @@ defmodule Lightning.Adaptors.NPM.Schema do @doc """ Fetch the configuration schema for `name@version` from jsDelivr. - Returns `{schema_data, schema_sha256}` on success, `{nil, nil}` on - any failure (best-effort — schema absence must not fail the adaptor - record assembly). + Returns `{schema_data, schema_sha256}` on success, `{nil, nil}` on a + genuine 404 (schema removed upstream), or `{nil, :fetch_failed}` on + any other failure — a transient failure must not be mistaken for + genuine absence by callers that persist the result. """ @spec schema(String.t(), String.t()) :: - {map() | nil, String.t() | nil} + {map(), String.t()} | {nil, nil} | {nil, :fetch_failed} def schema(name, version) do with {:ok, body} <- fetch_schema_bytes(name, version), {:ok, data} <- Jason.decode(body) do sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) {data, sha} else - _ -> {nil, nil} + {:error, {:http_status, 404}} -> {nil, nil} + _ -> {nil, :fetch_failed} end end diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index fef787b7ef0..3e3d430e83a 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -262,8 +262,7 @@ defmodule Lightning.Adaptors.Scheduler do @impl true def handle_call(:refresh_now, _from, state) do Logger.info("Adaptors[#{state.source}]: refresh_now requested") - send(self(), :tick) - {:reply, :ok, state} + {:reply, :ok, maybe_start_refresh(state)} end def handle_call(:await_refresh, from, state) do @@ -272,8 +271,7 @@ defmodule Lightning.Adaptors.Scheduler do ) state = %{state | waiters: [from | state.waiters]} - state = if state.refresh, do: state, else: start_refresh(state) - {:noreply, state} + {:noreply, maybe_start_refresh(state)} end def handle_call({:refresh_package, name}, from, state) do @@ -326,6 +324,10 @@ defmodule Lightning.Adaptors.Scheduler do end end + defp maybe_start_refresh(state) do + if state.refresh, do: state, else: start_refresh(state) + end + defp start_refresh(state) do task = Task.Supervisor.async_nolink(state.tasks, fn -> do_refresh(state) end) %{state | refresh: task} @@ -357,7 +359,10 @@ defmodule Lightning.Adaptors.Scheduler do &fetch_if_changed(strategy, &1, existing_by_name, state), max_concurrency: @fetch_max_concurrency, ordered: false, - on_timeout: :kill_task + on_timeout: :kill_task, + timeout: + Config.strategy_opts(strategy)[:http_timeout] || + :timer.seconds(30) ) |> Enum.reduce({[], 0, 0}, fn {:ok, {:fetched, record}}, {acc, c, e} -> {[record | acc], c + 1, e} @@ -373,7 +378,16 @@ defmodule Lightning.Adaptors.Scheduler do |> Enum.map(fn record -> persist_with_icons(record, icons, state) end) |> Enum.count(&(&1 == :ok)) - healed = heal_missing_icons(icons, state) + # Rows fetched this tick already have fresh icons from + # persist_with_icons/3. Everything else — touched or errored — is + # reconciled here too, so an icon-only upstream change still lands + # even when the version doesn't bump. + fetched_names = MapSet.new(fetched, & &1.name) + + unfetched_rows = + Enum.reject(existing_rows, &MapSet.member?(fetched_names, &1.name)) + + healed = reapply_icons(unfetched_rows, icons, state).updated not_modified = count_not_modified(icons) listed = length(upstream) @@ -518,24 +532,6 @@ defmodule Lightning.Adaptors.Scheduler do Map.put(record, :"icon_#{shape}_etag", etag) end - # Tops up icons on rows currently missing at least one shape. Runs - # after the main upsert pass on every tick — cheap, scoped to rows - # with gaps, and self-correcting after a strategy outage. - defp heal_missing_icons(icons, _state) when map_size(icons) == 0, do: 0 - - defp heal_missing_icons(icons, state) do - state.source - |> Catalogue.list_missing_icons() - |> Enum.reduce(0, fn row, acc -> - package_icons = Map.get(icons, row.name, %{}) - - case apply_icons_to_existing(row, package_icons, state) do - :updated -> acc + 1 - :unchanged -> acc - end - end) - end - defp reapply_icons(existing_rows, icons, state) do Enum.reduce(existing_rows, %{updated: 0, unchanged: 0}, fn row, acc -> package_icons = Map.get(icons, row.name, %{}) @@ -547,9 +543,8 @@ defmodule Lightning.Adaptors.Scheduler do end) end - # `row` is either an Adaptor struct (from list_adaptors/1) or a lean - # map (from list_missing_icons/1) — both expose :name and the icon - # sha256 fields, which is all we need. + # `row` is an Adaptor struct (from list_adaptors/1), which exposes + # :name and the icon sha256 fields, which is all we need. defp apply_icons_to_existing(_row, package_icons, _state) when map_size(package_icons) == 0, do: :unchanged diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index 0210489e86e..ea3dccb68ea 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -54,7 +54,8 @@ defmodule Lightning.Adaptors.Store do @doc """ Returns the adaptor's credential schema as a JSON binary, not decoded. """ - @spec schema(sup(), String.t()) :: {:ok, String.t()} | {:error, term()} + @spec schema(sup(), String.t()) :: + {:ok, String.t() | nil} | {:error, term()} def schema(sup, name) do cache = AdaptorsSupervisor.cache_name(sup) source = AdaptorsSupervisor.source(sup) @@ -112,14 +113,16 @@ defmodule Lightning.Adaptors.Store do with {:ok, meta} <- icon_meta(sup, name), {:ok, ext} <- ext_for_shape(meta, shape), - {:ok, _sha256} <- sha256_for_shape(meta, shape) do - if IconCache.cached?(source, name, shape, ext) do + {:ok, expected_sha} <- sha256_for_shape(meta, shape) do + if disk_cache_matches?(source, name, shape, ext, expected_sha) do {:ok, IconCache.path(source, name, shape, ext)} else cache |> Cachex.fetch( {:icon_bytes, source, name, shape}, - fn _key -> fetch_icon_bytes(strategy, source, name, shape, ext) end, + fn _key -> + fetch_icon_bytes(strategy, source, name, shape, ext, expected_sha) + end, timeout: Config.cache_timeout_ms() ) |> unwrap() @@ -127,11 +130,29 @@ defmodule Lightning.Adaptors.Store do end end - defp fetch_icon_bytes(strategy, source, name, shape, ext) do + # A cached file existing proves nothing about its content — a node that + # cached an earlier version of this icon keeps that file forever + # otherwise. A sha mismatch, or the file being absent, are both treated + # as a miss so the fetch branch below re-pulls and overwrites it. + defp disk_cache_matches?(source, name, shape, ext, expected_sha) do + case source |> IconCache.path(name, shape, ext) |> File.read() do + {:ok, bytes} -> :crypto.hash(:sha256, bytes) == expected_sha + {:error, _} -> false + end + end + + defp fetch_icon_bytes(strategy, source, name, shape, ext, expected_sha) do case strategy.fetch_icon(name, shape) do {:ok, %{data: bytes, ext: ^ext}} -> - {:ok, _sha} = IconCache.write!(source, name, shape, ext, bytes) - {:ignore, {:ok, IconCache.path(source, name, shape, ext)}} + if :crypto.hash(:sha256, bytes) == expected_sha do + {:ok, _sha} = IconCache.write!(source, name, shape, ext, bytes) + {:ignore, {:ok, IconCache.path(source, name, shape, ext)}} + else + {:ignore, + {:error, + {:icon_sha_mismatch, + expected: expected_sha, got: :crypto.hash(:sha256, bytes)}}} + end {:ok, %{ext: other_ext}} -> {:ignore, {:error, {:ext_mismatch, expected: ext, got: other_ext}}} diff --git a/lib/lightning/adaptors/strategy.ex b/lib/lightning/adaptors/strategy.ex index 735de0a1102..1a7a6832089 100644 --- a/lib/lightning/adaptors/strategy.ex +++ b/lib/lightning/adaptors/strategy.ex @@ -49,16 +49,16 @@ defmodule Lightning.Adaptors.Strategy do `c:fetch_icons/1` — they are not stamped onto this record. """ @type adaptor_record :: %{ - name: String.t(), - description: String.t() | nil, - homepage: String.t() | nil, - repository: String.t() | nil, - license: String.t() | nil, - latest_version: String.t(), - deprecated: boolean(), - schema_data: map() | nil, - schema_sha256: String.t() | nil, - versions: [version_record()] + required(:name) => String.t(), + required(:description) => String.t() | nil, + required(:homepage) => String.t() | nil, + required(:repository) => String.t() | nil, + required(:license) => String.t() | nil, + required(:latest_version) => String.t(), + required(:deprecated) => boolean(), + optional(:schema_data) => map() | nil, + optional(:schema_sha256) => String.t() | nil, + required(:versions) => [version_record()] } @typedoc """ diff --git a/test/lightning/adaptors/npm/schema_test.exs b/test/lightning/adaptors/npm/schema_test.exs index 958fd953a2c..b4baa9ab0cd 100644 --- a/test/lightning/adaptors/npm/schema_test.exs +++ b/test/lightning/adaptors/npm/schema_test.exs @@ -62,25 +62,29 @@ defmodule Lightning.Adaptors.NPM.SchemaTest do assert {nil, nil} = Schema.schema(@package, @version) end - test "returns {nil, nil} on 5xx", %{bypass: bypass} do + test "returns {nil, :fetch_failed} on 5xx", %{bypass: bypass} do Bypass.expect(bypass, "GET", @path, fn conn -> Plug.Conn.resp(conn, 500, "") end) - assert {nil, nil} = Schema.schema(@package, @version) + assert {nil, :fetch_failed} = Schema.schema(@package, @version) end - test "returns {nil, nil} on invalid JSON body", %{bypass: bypass} do + test "returns {nil, :fetch_failed} on invalid JSON body", %{ + bypass: bypass + } do Bypass.expect(bypass, "GET", @path, fn conn -> Plug.Conn.resp(conn, 200, "this is not json {") end) - assert {nil, nil} = Schema.schema(@package, @version) + assert {nil, :fetch_failed} = Schema.schema(@package, @version) end - test "returns {nil, nil} on connection refused", %{bypass: bypass} do + test "returns {nil, :fetch_failed} on connection refused", %{ + bypass: bypass + } do Bypass.down(bypass) - assert {nil, nil} = Schema.schema(@package, @version) + assert {nil, :fetch_failed} = Schema.schema(@package, @version) end end end diff --git a/test/lightning/adaptors/npm_test.exs b/test/lightning/adaptors/npm_test.exs index 53cb4547f86..3729a006434 100644 --- a/test/lightning/adaptors/npm_test.exs +++ b/test/lightning/adaptors/npm_test.exs @@ -108,10 +108,11 @@ defmodule Lightning.Adaptors.NPMTest do assert old.deprecated == true end - test "degrades to nil schema when jsDelivr returns 5xx", %{ - registry: registry, - jsdelivr: jsdelivr - } do + test "omits schema_data/schema_sha256 entirely when jsDelivr returns 5xx", + %{ + registry: registry, + jsdelivr: jsdelivr + } do packument = build_packument() Bypass.expect(registry, "GET", "/" <> @package, fn conn -> @@ -124,8 +125,10 @@ defmodule Lightning.Adaptors.NPMTest do {:ok, record} = NPM.fetch_adaptor(@package) - assert record.schema_data == nil - assert record.schema_sha256 == nil + refute Map.has_key?(record, :schema_data), + "a transient schema-fetch failure must omit the key, not set it to nil, so cast/3 leaves the persisted schema untouched" + + refute Map.has_key?(record, :schema_sha256) assert record.name == @package assert record.latest_version == @latest_version end diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index f770c0880ba..7b4190ccccb 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -85,6 +85,14 @@ defmodule Lightning.Adaptors.SchedulerTest do pid end + defp drain_tick_ran do + receive do + :tick_ran -> drain_tick_ran() + after + 0 -> :ok + end + end + defp adaptor_record(overrides \\ []) do overrides = Map.new(overrides) @@ -386,6 +394,46 @@ defmodule Lightning.Adaptors.SchedulerTest do end end + describe "fetch timeout" do + test "per-adaptor fetch is bounded by the strategy's http_timeout, " <> + "not Task's 5s default", + %{sup: sup} do + original = Application.get_env(:lightning, Lightning.Adaptors.StrategyMock) + + Application.put_env( + :lightning, + Lightning.Adaptors.StrategyMock, + Keyword.put(original, :http_timeout, 100) + ) + + on_exit(fn -> + Application.put_env( + :lightning, + Lightning.Adaptors.StrategyMock, + original + ) + end) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-http", latest_version: "2.0.0"}]} + end) + + # Never returns; only the async_stream timeout can end it. With the + # 5s default the await below would time out, so it passing shows + # the configured budget is what's being applied. + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + receive do + end + end) + + start_scheduler(sup) + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:ok, %{listed: 1, errors: 1}} = + Scheduler.await_refresh(sched_name, 2_000) + end + end + describe "refresh_now/1" do test "triggers an immediate tick on the leader", %{sup: sup} do test_pid = self() @@ -413,6 +461,45 @@ defmodule Lightning.Adaptors.SchedulerTest do assert_receive :tick_ran, 2000 end + + test "repeated calls do not leak extra recurring tick chains", %{sup: sup} do + test_pid = self() + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :tick_ran) + {:ok, []} + end) + + start_scheduler(sup, interval: 200) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + {:global, gname} = sched_name + pid = :global.whereis_name(gname) + + # Init tick, then two manual refresh_now calls — each waited out so it + # starts its own cycle instead of coalescing into the previous one. + assert_receive :tick_ran, 2000 + assert_eventually(:sys.get_state(pid).refresh == nil, 2000) + + assert :ok = Scheduler.refresh_now(sched_name) + assert_receive :tick_ran, 2000 + assert_eventually(:sys.get_state(pid).refresh == nil, 2000) + + assert :ok = Scheduler.refresh_now(sched_name) + assert_receive :tick_ran, 2000 + assert_eventually(:sys.get_state(pid).refresh == nil, 2000) + + # Drain any tick_ran messages belonging to the manual calls themselves + # before counting the chain(s) that fire on their own over one interval. + drain_tick_ran() + + # Only the init-driven chain should still be ticking, arriving ~200ms + # out. A leaked chain per refresh_now call would fire almost + # immediately instead, since they were all armed within milliseconds + # of each other above. + assert_receive :tick_ran, 300 + refute_receive :tick_ran, 100 + end end # Rows are seeded before the Scheduler starts so no init tick fires and @@ -543,6 +630,73 @@ defmodule Lightning.Adaptors.SchedulerTest do assert row.icon_square_sha256 == nil end + test "updates an icon-only change on the periodic tick even when the " <> + "package's version did not bump", + %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + + old_sha = :crypto.hash(:sha256, "OLD") + rect_sha = :crypto.hash(:sha256, "RECT") + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + icon_square_ext: "png", + icon_square_sha256: old_sha, + # Both icon shapes already exist on the row; only the square + # shape's bytes changed upstream. + icon_rectangle_ext: "png", + icon_rectangle_sha256: rect_sha + ) + ) + + new_bytes = "NEW_ICON_BYTES" + new_sha = :crypto.hash(:sha256, new_bytes) + + # Upstream reports the same version, so the diff path marks this + # adaptor :touched instead of re-fetching it — only the icon changed. + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, + %{ + "@openfn/language-http" => %{ + square: %{data: new_bytes, ext: "png", sha256: new_sha} + } + }} + end) + + source_topic = AdaptorsSupervisor.source_topic(sup) + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + :ok = Scheduler.refresh_now(sched_name) + + assert_receive {:changed, "@openfn/language-http", ^source}, 2000 + + row = Catalogue.get_adaptor("@openfn/language-http", source) + assert row.latest_version == "1.0.0" + assert row.icon_square_ext == "png" + assert row.icon_square_sha256 == new_sha + + icon_path = + Lightning.Adaptors.IconCache.path( + source, + "@openfn/language-http", + :square, + "png" + ) + + File.rm(icon_path) + end + test "self-heals iconless rows on the periodic tick", %{sup: sup} do source = AdaptorsSupervisor.source(sup) diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index dad3b61da74..487f2fbfa9b 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -470,6 +470,61 @@ defmodule Lightning.Adaptors.StoreTest do assert File.read!(path) == "PRE_WARMED" end + test "stale disk cache (sha mismatch) self-heals by re-fetching", %{ + sup: sup + } do + source = AdaptorsSupervisor.source(sup) + name = unique_name("stale") + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + name: name, + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "FRESH_BYTES") + ) + ) + + {:ok, _} = + Lightning.Adaptors.IconCache.write!( + source, + name, + :square, + "png", + "STALE_BYTES" + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 1, fn ^name, + :square -> + {:ok, %{data: "FRESH_BYTES", ext: "png"}} + end) + + assert {:ok, path} = Store.icon(sup, name, :square) + assert File.read!(path) == "FRESH_BYTES" + end + + test "Strategy returns bytes that don't match the row's expected sha", %{ + sup: sup + } do + name = unique_name("corrupt") + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + name: name, + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "EXPECTED_BYTES") + ) + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 1, fn ^name, + :square -> + {:ok, %{data: "WRONG_BYTES", ext: "png"}} + end) + + assert {:error, {:icon_sha_mismatch, _}} = Store.icon(sup, name, :square) + end + test "disk miss + Strategy success writes to disk and returns path", %{ sup: sup, cache: cache From 5dfd073fc98aa7af29d7abc43a089c0421ec4454 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Tue, 8 Sep 2026 14:30:20 +0200 Subject: [PATCH 13/37] Give adaptors without a schema an empty schema instead of crashing - Adaptors without a configuration schema hidden from the credential picker - A failed lazy schema fetch no longer cached as an empty schema - Unused Package.Version struct removed - One package_meta shape read from both the cache and the DB fallback - An adaptor without a schema now gets an empty schema, not nil - Store's rendered catalogue type renamed so it stops shadowing Catalogue's - Strategy contract becomes the single place schema_data is encoded - Schema-absence comment trimmed, icon bytes hashed once --- lib/lightning/adaptors.ex | 37 +++------- lib/lightning/adaptors/catalogue.ex | 46 ++++++++----- lib/lightning/adaptors/npm.ex | 15 +--- lib/lightning/adaptors/store.ex | 55 +++++++-------- lib/lightning/adaptors/strategy.ex | 6 +- .../credential_form_component.ex | 9 ++- test/lightning/adaptors/catalogue_test.exs | 64 ++++++++++++++++- .../adaptors/isolated_adaptors_test.exs | 1 + test/lightning/adaptors/store_test.exs | 15 ++++ test/lightning/adaptors_test.exs | 40 ++++++++++- test/lightning/credentials_test.exs | 69 +++++++++++++++++++ .../live/credential_live_test.exs | 8 ++- test/support/adaptor_test_helpers.ex | 3 +- 13 files changed, 274 insertions(+), 94 deletions(-) diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index 3aed31cd9fb..ee738f9f9ef 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -53,28 +53,6 @@ defmodule Lightning.Adaptors do One catalogue adaptor. """ - defmodule Version do - @moduledoc """ - One published version of a catalogue adaptor. - """ - - @type t :: %__MODULE__{ - version: String.t(), - integrity: String.t() | nil, - size_bytes: integer() | nil, - published_at: DateTime.t() | nil, - deprecated: boolean() - } - - defstruct [ - :version, - :integrity, - :size_bytes, - :published_at, - deprecated: false - ] - end - @type t :: %__MODULE__{ name: String.t(), source: :npm | :local, @@ -84,7 +62,8 @@ defmodule Lightning.Adaptors do icon_square_ext: String.t() | nil, icon_rectangle_ext: String.t() | nil, icon_square_sha256: binary() | nil, - icon_rectangle_sha256: binary() | nil + icon_rectangle_sha256: binary() | nil, + has_schema: boolean() } defstruct [ @@ -96,6 +75,7 @@ defmodule Lightning.Adaptors do :icon_rectangle_ext, :icon_square_sha256, :icon_rectangle_sha256, + :has_schema, deprecated: false ] end @@ -113,7 +93,7 @@ defmodule Lightning.Adaptors do @doc """ Returns the credential schema of the adaptor named `pkg`, as a JSON - binary. + binary. An adaptor with no schema yields `"{}"`. """ @spec schema(atom(), String.t()) :: {:ok, String.t()} | {:error, term()} def schema(sup \\ Config.default_instance(), pkg), do: Store.schema(sup, pkg) @@ -163,7 +143,7 @@ defmodule Lightning.Adaptors do """ @spec catalogue(atom()) :: {:ok, - {{DateTime.t() | nil, non_neg_integer()}, [Store.catalogue_entry()]}} + {{DateTime.t() | nil, non_neg_integer()}, [Store.rendered_entry()]}} | {:error, term()} def catalogue(sup \\ Config.default_instance()) do Store.catalogue(sup) @@ -227,15 +207,14 @@ defmodule Lightning.Adaptors do {:error, _} -> nil end - case cached || Catalogue.get_adaptor(name, source) do + case cached || Catalogue.get_package_meta(name, source) do nil -> nil meta -> to_package(meta, source) end end - defp to_package(meta, source) do - struct(Package, meta |> Map.delete(:__struct__) |> Map.put(:source, source)) - end + defp to_package(meta, source), + do: struct!(Package, Map.put(meta, :source, source)) @doc """ Waits until the catalogue has loaded at least once, triggering the diff --git a/lib/lightning/adaptors/catalogue.ex b/lib/lightning/adaptors/catalogue.ex index 909dc86957f..34889b4aafc 100644 --- a/lib/lightning/adaptors/catalogue.ex +++ b/lib/lightning/adaptors/catalogue.ex @@ -24,11 +24,11 @@ defmodule Lightning.Adaptors.Catalogue do latest_version: String.t(), description: String.t() | nil, deprecated: boolean(), - updated_at: DateTime.t(), icon_square_ext: String.t() | nil, icon_rectangle_ext: String.t() | nil, icon_square_sha256: binary() | nil, - icon_rectangle_sha256: binary() | nil + icon_rectangle_sha256: binary() | nil, + has_schema: boolean() } @type catalogue_entry :: %{ @@ -63,20 +63,34 @@ defmodule Lightning.Adaptors.Catalogue do """ @spec list_package_metas(source()) :: [package_meta()] def list_package_metas(source) do - Repo.all( - from a in active_adaptors(source), - select: %{ - name: a.name, - latest_version: a.latest_version, - description: a.description, - deprecated: a.deprecated, - updated_at: a.updated_at, - icon_square_ext: a.icon_square_ext, - icon_rectangle_ext: a.icon_rectangle_ext, - icon_square_sha256: a.icon_square_sha256, - icon_rectangle_sha256: a.icon_rectangle_sha256 - } - ) + source |> active_adaptors() |> select_package_meta() |> Repo.all() + end + + @doc """ + The `t:package_meta/0` projection of one `(name, source)` row, or `nil`. + Unlike `list_package_metas/1` this resolves excluded and deprecated + adaptors too, so jobs already using one keep validating. + """ + @spec get_package_meta(String.t(), source()) :: package_meta() | nil + def get_package_meta(name, source) do + from(a in Adaptor, where: a.name == ^name and a.source == ^source) + |> select_package_meta() + |> Repo.one() + end + + defp select_package_meta(query) do + from a in query, + select: %{ + name: a.name, + latest_version: a.latest_version, + description: a.description, + deprecated: a.deprecated, + icon_square_ext: a.icon_square_ext, + icon_rectangle_ext: a.icon_rectangle_ext, + icon_square_sha256: a.icon_square_sha256, + icon_rectangle_sha256: a.icon_rectangle_sha256, + has_schema: not is_nil(a.schema_data) + } end @doc """ diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index 958816bce09..44beb8483c0 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -83,11 +83,8 @@ defmodule Lightning.Adaptors.NPM do end end - # A transient schema-fetch failure must leave `schema_data`/`schema_sha256` - # absent from the record entirely, not merely `nil` — `Ecto.Changeset.cast/3` - # overwrites a column whenever its key is present in `attrs`, even with a - # nil value, so an absent key is the only way to signal "leave the - # previously-persisted schema untouched." + # Absent, not nil: `Ecto.Changeset.cast/3` overwrites a column for any + # present key, and `Strategy.adaptor_record/0` reserves nil for "no schema". defp put_schema(record, {nil, :fetch_failed}), do: record defp put_schema(record, {schema_data, schema_sha}) do @@ -96,13 +93,7 @@ defmodule Lightning.Adaptors.NPM do |> Map.put(:schema_sha256, schema_sha) end - # Strategy boundary: re-encode the decoded schema map to a JSON binary - # so the row is persisted as text and `Jason.decode!(_, - # objects: :ordered_objects)` re-engages downstream. `Schema.schema/2` - # always decodes via `Jason.decode/1`, so `data` is a map (or nil) here, - # never a raw binary — the Local strategy's own raw-binary schema text - # takes a separate path (`Local.read_schema/1`) and never reaches this - # function. + # Re-encoded so the reader can decode it with ordered objects. defp encode_schema(nil), do: nil defp encode_schema(data) when is_map(data), do: Jason.encode!(data) diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index ea3dccb68ea..a855a79f497 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -37,7 +37,11 @@ defmodule Lightning.Adaptors.Store do @type package_meta :: Catalogue.package_meta() - @type catalogue_entry :: %{ + @typedoc """ + One `t:Lightning.Adaptors.Catalogue.catalogue_entry/0` with its icon + fields rendered to URLs, as the catalogue endpoint serves it. + """ + @type rendered_entry :: %{ name: String.t(), latest_version: String.t(), versions: [String.t()], @@ -49,13 +53,13 @@ defmodule Lightning.Adaptors.Store do } @type catalogue :: - {{DateTime.t() | nil, non_neg_integer()}, [catalogue_entry()]} + {{DateTime.t() | nil, non_neg_integer()}, [rendered_entry()]} @doc """ Returns the adaptor's credential schema as a JSON binary, not decoded. + An adaptor with no schema yields `"{}"`. """ - @spec schema(sup(), String.t()) :: - {:ok, String.t() | nil} | {:error, term()} + @spec schema(sup(), String.t()) :: {:ok, String.t()} | {:error, term()} def schema(sup, name) do cache = AdaptorsSupervisor.cache_name(sup) source = AdaptorsSupervisor.source(sup) @@ -144,14 +148,14 @@ defmodule Lightning.Adaptors.Store do defp fetch_icon_bytes(strategy, source, name, shape, ext, expected_sha) do case strategy.fetch_icon(name, shape) do {:ok, %{data: bytes, ext: ^ext}} -> - if :crypto.hash(:sha256, bytes) == expected_sha do - {:ok, _sha} = IconCache.write!(source, name, shape, ext, bytes) - {:ignore, {:ok, IconCache.path(source, name, shape, ext)}} - else - {:ignore, - {:error, - {:icon_sha_mismatch, - expected: expected_sha, got: :crypto.hash(:sha256, bytes)}}} + case :crypto.hash(:sha256, bytes) do + ^expected_sha -> + {:ok, _sha} = IconCache.write!(source, name, shape, ext, bytes) + {:ignore, {:ok, IconCache.path(source, name, shape, ext)}} + + got -> + {:ignore, + {:error, {:icon_sha_mismatch, expected: expected_sha, got: got}}} end {:ok, %{ext: other_ext}} -> @@ -278,7 +282,7 @@ defmodule Lightning.Adaptors.Store do end @spec render_entry(Catalogue.catalogue_entry(), Catalogue.source()) :: - catalogue_entry() + rendered_entry() defp render_entry(entry, source) do %{ name: entry.name, @@ -308,13 +312,15 @@ defmodule Lightning.Adaptors.Store do defp fetch_and_persist_known(sup, name, source, field) do case AdaptorsSupervisor.strategy(sup).fetch_adaptor(name) do {:ok, %{name: ^name} = record} -> - record = - record - |> Map.put(:source, source) - |> normalize_schema_data() - + record = Map.put(record, :source, source) {:ok, _} = Catalogue.upsert_adaptor(record) - {:commit, {:ok, record |> Map.get(field) |> project_field(field)}} + + # The strategy leaves a field off the record when its fetch failed + # transiently. Don't cache that as "no value"; let the next call retry. + case Map.fetch(record, field) do + {:ok, value} -> {:commit, {:ok, project_field(value, field)}} + :error -> {:ignore, {:ok, project_field(nil, field)}} + end {:ok, %{name: other}} -> {:ignore, {:error, {:name_mismatch, other}}} @@ -328,18 +334,9 @@ defmodule Lightning.Adaptors.Store do defp project_field(rows, :versions) when is_list(rows), do: project_versions(rows) + defp project_field(nil, :schema_data), do: "{}" defp project_field(value, _field), do: value - # The real strategies already encode schema_data to a JSON binary, but - # a strategy is still free to hand back a map, so normalize here to - # keep the cached value consistent with what a DB-backed read returns. - defp normalize_schema_data(%{schema_data: data} = record) - when is_map(data) and not is_struct(data) do - %{record | schema_data: Jason.encode!(data)} - end - - defp normalize_schema_data(record), do: record - @spec project_icon_meta(map()) :: icon_meta() defp project_icon_meta(adaptor) do Map.take(adaptor, [ diff --git a/lib/lightning/adaptors/strategy.ex b/lib/lightning/adaptors/strategy.ex index 1a7a6832089..6d8a444abd4 100644 --- a/lib/lightning/adaptors/strategy.ex +++ b/lib/lightning/adaptors/strategy.ex @@ -47,6 +47,10 @@ defmodule Lightning.Adaptors.Strategy do The structured adaptor record returned by `c:fetch_adaptor/1`. Icon fields are persisted separately by the Scheduler after joining `c:fetch_icons/1` — they are not stamped onto this record. + + `schema_data` is the credential schema as a JSON binary. `nil` means + the adaptor has no schema; leaving the key off means the schema fetch + failed transiently and the caller should keep what it already has. """ @type adaptor_record :: %{ required(:name) => String.t(), @@ -56,7 +60,7 @@ defmodule Lightning.Adaptors.Strategy do required(:license) => String.t() | nil, required(:latest_version) => String.t(), required(:deprecated) => boolean(), - optional(:schema_data) => map() | nil, + optional(:schema_data) => String.t() | nil, optional(:schema_sha256) => String.t() | nil, required(:versions) => [version_record()] } diff --git a/lib/lightning_web/live/credential_live/credential_form_component.ex b/lib/lightning_web/live/credential_live/credential_form_component.ex index cda03aa3af5..732860d0d5f 100644 --- a/lib/lightning_web/live/credential_live/credential_form_component.ex +++ b/lib/lightning_web/live/credential_live/credential_form_component.ex @@ -1176,8 +1176,13 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do defp get_type_options do adaptor_options = case Adaptors.packages() do - {:ok, packages} -> Enum.map(packages, &adaptor_type_option/1) - {:error, _} -> [] + {:ok, packages} -> + packages + |> Enum.filter(& &1.has_schema) + |> Enum.map(&adaptor_type_option/1) + + {:error, _} -> + [] end adaptor_options diff --git a/test/lightning/adaptors/catalogue_test.exs b/test/lightning/adaptors/catalogue_test.exs index 3c069106065..85d55fe6d65 100644 --- a/test/lightning/adaptors/catalogue_test.exs +++ b/test/lightning/adaptors/catalogue_test.exs @@ -85,6 +85,47 @@ defmodule Lightning.Adaptors.CatalogueTest do end end + describe "upsert_adaptor/1 — schema preservation on transient fetch failure" do + test "transient failure keeps the old schema" do + {:ok, first} = + Catalogue.upsert_adaptor( + adaptor_record( + schema_data: %{"type" => "object"}, + schema_sha256: "abc" + ) + ) + + {:ok, second} = + Catalogue.upsert_adaptor( + adaptor_record() + |> Map.drop([:schema_data, :schema_sha256]) + ) + + assert second.id == first.id + assert second.schema_data == first.schema_data + assert second.schema_sha256 == first.schema_sha256 + end + + test "genuinely removed schema does clear" do + {:ok, first} = + Catalogue.upsert_adaptor( + adaptor_record( + schema_data: %{"type" => "object"}, + schema_sha256: "abc" + ) + ) + + {:ok, second} = + Catalogue.upsert_adaptor( + adaptor_record(schema_data: nil, schema_sha256: nil) + ) + + assert second.id == first.id + assert second.schema_data == nil + assert second.schema_sha256 == nil + end + end + describe "upsert_adaptor/1 — diff-aware :updated_at" do test "changing :latest_version bumps :updated_at" do {:ok, first} = Catalogue.upsert_adaptor(adaptor_record()) @@ -286,7 +327,7 @@ defmodule Lightning.Adaptors.CatalogueTest do assert meta.latest_version == "1.0.0" assert meta.description == "yep" assert meta.deprecated == false - assert %DateTime{} = meta.updated_at + assert meta.has_schema refute Map.has_key?(meta, :schema_data) refute Map.has_key?(meta, :homepage) @@ -303,6 +344,27 @@ defmodule Lightning.Adaptors.CatalogueTest do Catalogue.list_package_metas(:local) end + test "has_schema reflects whether schema_data is set" do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + name: "@openfn/language-with-schema", + schema_data: %{"type" => "object"} + ) + ) + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record(name: "@openfn/language-no-schema", schema_data: nil) + ) + + metas = Catalogue.list_package_metas(:npm) + + assert Enum.find(metas, &(&1.name == "@openfn/language-with-schema")).has_schema + + refute Enum.find(metas, &(&1.name == "@openfn/language-no-schema")).has_schema + end + test "omits the excluded adaptors" do for name <- [ "@openfn/language-devtools", diff --git a/test/lightning/adaptors/isolated_adaptors_test.exs b/test/lightning/adaptors/isolated_adaptors_test.exs index e334ab45880..d5001ff510d 100644 --- a/test/lightning/adaptors/isolated_adaptors_test.exs +++ b/test/lightning/adaptors/isolated_adaptors_test.exs @@ -30,6 +30,7 @@ defmodule Lightning.Adaptors.IsolatedAdaptorsTest do latest_version: "1.2.3", description: nil, deprecated: false, + has_schema: false, icon_square_ext: nil, icon_rectangle_ext: nil, icon_square_sha256: nil, diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index 487f2fbfa9b..10164c38c58 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -93,6 +93,21 @@ defmodule Lightning.Adaptors.StoreTest do Cachex.get(cache, {:schema, "@openfn/language-http", source}) end + test "a failed schema fetch returns an empty schema without caching it", + %{sup: sup, cache: cache} do + source = AdaptorsSupervisor.source(sup) + name = "@openfn/language-http" + + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> + {:ok, adaptor_record() |> Map.drop([:schema_data, :schema_sha256])} + end) + + assert {:ok, "{}"} = Store.schema(sup, name) + assert {:ok, nil} = Cachex.get(cache, {:schema, name, source}) + end + test "unknown adaptor returns {:error, :not_found} without calling Strategy or minting a row", %{sup: sup, cache: cache} do expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index a76a4ef15f8..a1d98e308ef 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -100,6 +100,17 @@ defmodule Lightning.AdaptorsTest do test "returns {:ok, []} when DB is empty", %{sup: sup} do assert {:ok, []} = Adaptors.packages(sup) end + + test "has_schema is false for a package with no schema_data", %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> + {:error, :unreachable} + end) + + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) + + assert {:ok, [%Adaptors.Package{has_schema: false}]} = + Adaptors.packages(sup) + end end describe "default instance resolution" do @@ -116,7 +127,8 @@ defmodule Lightning.AdaptorsTest do icon_square_ext: nil, icon_rectangle_ext: nil, icon_square_sha256: nil, - icon_rectangle_sha256: nil + icon_rectangle_sha256: nil, + has_schema: false } Cachex.put( @@ -204,6 +216,32 @@ defmodule Lightning.AdaptorsTest do assert Adaptors.get_adaptor("@openfn/language-http") == nil end + + test "computes has_schema on the DB-fallback path (excluded from the lean listing)" do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + name: "@openfn/language-collections", + schema_data: ~s({"type":"object"}) + ) + ) + + assert %Adaptors.Package{has_schema: true} = + Adaptors.get_adaptor("@openfn/language-collections") + end + + test "has_schema is false on the DB-fallback path when schema_data is nil" do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + name: "@openfn/language-collections", + schema_data: nil + ) + ) + + assert %Adaptors.Package{has_schema: false} = + Adaptors.get_adaptor("@openfn/language-collections") + end end describe "resolve_name/2" do diff --git a/test/lightning/credentials_test.exs b/test/lightning/credentials_test.exs index 1564874510c..fca0e5f1f2e 100644 --- a/test/lightning/credentials_test.exs +++ b/test/lightning/credentials_test.exs @@ -1,6 +1,8 @@ defmodule Lightning.CredentialsTest do use Lightning.DataCase, async: true + @endpoint LightningWeb.Endpoint + alias Lightning.Auditing alias Lightning.Credentials alias Lightning.Credentials.Audit @@ -11,6 +13,9 @@ defmodule Lightning.CredentialsTest do import Lightning.Factories import Ecto.Query import Mox + import Phoenix.ConnTest + import Phoenix.LiveViewTest + import LightningWeb.CredentialLiveHelpers import Lightning.JobsFixtures @@ -2874,4 +2879,68 @@ defmodule Lightning.CredentialsTest do assert Repo.get!(Credential, custom.id).schema == "totally-custom" end end + + describe "get_schema/1" do + setup :isolated_adaptors + + test "returns a permissive schema for an adaptor with no schema, instead of crashing", + %{sup: sup} do + name = "@openfn/language-no-schema" + insert(:adaptor, name: name, schema_data: nil) + + cache = Lightning.Adaptors.Supervisor.cache_name(sup) + source = Lightning.Adaptors.Supervisor.source(sup) + Cachex.put(cache, {:schema, name, source}, {:ok, "{}"}) + + assert %Credentials.Schema{name: ^name, fields: []} = + Credentials.get_schema(name) + end + + test "returns the adaptor's schema when one is present" do + seed_credential_schema("http") + + assert %Credentials.Schema{fields: fields} = + Credentials.get_schema("@openfn/language-http") + + assert fields != [] + end + end + + describe "credential type picker (LiveView)" do + setup :isolated_adaptors + + setup do + Lightning.AccountsFixtures.superuser_fixture() + + conn = + LightningWeb.ConnCase.log_in_user(build_conn(), insert(:user)) + + %{conn: conn} + end + + test "omits a schema-less adaptor, closing the FunctionClauseError crash it used to reach", + %{conn: conn} do + insert(:adaptor, + name: "@openfn/language-with-schema", + schema_data: ~s({"type":"object"}) + ) + + insert(:adaptor, name: "@openfn/language-no-schema", schema_data: nil) + + {:ok, view, _html} = live(conn, "/credentials") + + html = open_create_credential_modal(view) + html_tree = Floki.parse_document!(html) + + assert Floki.find( + html_tree, + "label[for='credential-schema-picker_selected_@openfn/language-with-schema']" + ) != [] + + assert Floki.find( + html_tree, + "label[for='credential-schema-picker_selected_@openfn/language-no-schema']" + ) == [] + end + end end diff --git a/test/lightning_web/live/credential_live_test.exs b/test/lightning_web/live/credential_live_test.exs index ebdd9491352..ce1569308e4 100644 --- a/test/lightning_web/live/credential_live_test.exs +++ b/test/lightning_web/live/credential_live_test.exs @@ -2781,7 +2781,11 @@ defmodule LightningWeb.CredentialLiveTest do end test "omits a deprecated adaptor from the type options", %{conn: conn} do - insert(:adaptor, name: "deprecated-adaptor", deprecated: true) + insert(:adaptor, + name: "deprecated-adaptor", + deprecated: true, + schema_data: ~s({"type":"object"}) + ) # `seed_all_credential_schemas/0` primes the packages cache by hand # (bypassing `Catalogue.list_package_metas/1`), so drop it here to @@ -2797,7 +2801,7 @@ defmodule LightningWeb.CredentialLiveTest do assert Floki.find( html_tree, - "label[for='credential-schema-picker_selected_http']" + "label[for='credential-schema-picker_selected_@openfn/language-http']" ) != [] assert Floki.find( diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex index 17d89b7ec83..44f36eceda9 100644 --- a/test/support/adaptor_test_helpers.ex +++ b/test/support/adaptor_test_helpers.ex @@ -191,7 +191,8 @@ defmodule Lightning.AdaptorTestHelpers do icon_rectangle_ext: "png", icon_square_sha256: :crypto.hash(:sha256, short_name <> "-square"), icon_rectangle_sha256: - :crypto.hash(:sha256, short_name <> "-rectangle") + :crypto.hash(:sha256, short_name <> "-rectangle"), + has_schema: true } end) From 7e93a929fc673d804f232a3f3901a52d5eeed790 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Tue, 8 Sep 2026 15:45:22 +0200 Subject: [PATCH 14/37] Broadcast lazy schema/version fills, move icon sha checks into IconCache - Lazy schema or version fill broadcast like any other catalogue write - Missing schema told apart from a schema fetch that failed - Schema bytes jsDelivr served persisted instead of a re-encoded map - Local leaves schema_data off the record when the file is unreadable - Icon sha check moved into IconCache, existence-only cached?/4 dropped - Picker test moved beside its sibling, a test that primed its own answer dropped - Schema-less adaptor crash fix attributed to the empty-schema read --- lib/lightning/adaptors/icon_cache.ex | 23 ++++---- lib/lightning/adaptors/local.ex | 37 ++++++------ lib/lightning/adaptors/npm.ex | 6 +- lib/lightning/adaptors/npm/schema.ex | 10 ++-- lib/lightning/adaptors/store.ex | 45 ++++++++------- test/lightning/adaptors/icon_cache_test.exs | 23 +++++--- test/lightning/adaptors/npm/schema_test.exs | 4 +- test/lightning/adaptors/store_test.exs | 45 +++++++++++---- test/lightning/credentials_test.exs | 56 ------------------- .../live/credential_live_test.exs | 23 ++++++++ test/test_helper.exs | 2 +- 11 files changed, 136 insertions(+), 138 deletions(-) diff --git a/lib/lightning/adaptors/icon_cache.ex b/lib/lightning/adaptors/icon_cache.ex index 852e284f2a3..3dc28209d10 100644 --- a/lib/lightning/adaptors/icon_cache.ex +++ b/lib/lightning/adaptors/icon_cache.ex @@ -13,9 +13,9 @@ defmodule Lightning.Adaptors.IconCache do Source partitioning means flipping `ADAPTORS_STRATEGY` between restarts cannot accidentally serve `:npm` bytes from a row that's now resolved via `:local` (or vice versa). Latest-only means a subsequent - `write!/5` for the same key overwrites — content-addressable URLs - carry the sha8 prefix, so cache invalidation is intrinsic and we - don't need to keep old versions on disk. + `write!/5` for the same key overwrites, and `cached?/5` only trusts a + file whose bytes still hash to the sha on the adaptor row, so a node + that cached an earlier icon refetches instead of serving it forever. Concurrent first-request fetchers are coalesced upstream by Cachex's courier on `{:icon_bytes, source, name, shape}` inside @@ -46,11 +46,15 @@ defmodule Lightning.Adaptors.IconCache do end @doc """ - Whether the icon at `path(source, name, shape, ext)` exists on disk. + Whether the icon at `path(source, name, shape, ext)` is on disk with + bytes hashing to `sha256`. """ - @spec cached?(source(), name(), shape(), ext()) :: boolean() - def cached?(source, name, shape, ext) do - File.exists?(path(source, name, shape, ext)) + @spec cached?(source(), name(), shape(), ext(), binary()) :: boolean() + def cached?(source, name, shape, ext, sha256) do + case File.read(path(source, name, shape, ext)) do + {:ok, bytes} -> :crypto.hash(:sha256, bytes) == sha256 + {:error, _} -> false + end end @doc """ @@ -59,11 +63,6 @@ defmodule Lightning.Adaptors.IconCache do The write is staged in a sibling temp file and then renamed into place, so concurrent readers never observe a half-written file. - - The caller (Strategy / Scheduler) persists the returned sha on the - adaptor row and is responsible for verifying it matches the expected - sha from the upstream `adaptor_record` — defence against tarball or - filesystem corruption. """ @spec write!(source(), name(), shape(), ext(), binary()) :: {:ok, binary()} diff --git a/lib/lightning/adaptors/local.ex b/lib/lightning/adaptors/local.ex index 558401e2a1c..d75abcad923 100644 --- a/lib/lightning/adaptors/local.ex +++ b/lib/lightning/adaptors/local.ex @@ -210,9 +210,8 @@ defmodule Lightning.Adaptors.Local do defp build_adaptor_record(record) do pkg = record.latest_package_json - {schema_data, schema_sha256} = read_schema(record.latest_path) - %{ + base = %{ name: record.name, description: pkg["description"], homepage: pkg["homepage"], @@ -220,10 +219,16 @@ defmodule Lightning.Adaptors.Local do license: pkg["license"], latest_version: record.latest_version, deprecated: false, - schema_data: schema_data, - schema_sha256: schema_sha256, versions: Enum.map(record.versions, &build_version_record/1) } + + case read_schema(record.latest_path) do + {:ok, schema_data, schema_sha256} -> + Map.merge(base, %{schema_data: schema_data, schema_sha256: schema_sha256}) + + :unreadable -> + base + end end defp build_version_record(%{version: v, package_json: pkg}) do @@ -239,22 +244,16 @@ defmodule Lightning.Adaptors.Local do } end + # A missing file is "no schema"; anything else (a permissions error, a + # half-written file) is left off the record per `Strategy.adaptor_record/0`. defp read_schema(dir) do - case File.read(Path.join(dir, @schema_filename)) do - {:ok, body} -> - # Validate JSON, but keep the raw binary so credential-form - # rendering can re-engage ordered_objects decoding downstream. - case Jason.decode(body) do - {:ok, _data} -> - sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) - {body, sha} - - {:error, _} -> - {nil, nil} - end - - {:error, _} -> - {nil, nil} + with {:ok, body} <- File.read(Path.join(dir, @schema_filename)), + {:ok, _} <- Jason.decode(body) do + sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) + {:ok, body, sha} + else + {:error, :enoent} -> {:ok, nil, nil} + _ -> :unreadable end end diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index 44beb8483c0..6b66ae2f0ad 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -89,14 +89,10 @@ defmodule Lightning.Adaptors.NPM do defp put_schema(record, {schema_data, schema_sha}) do record - |> Map.put(:schema_data, encode_schema(schema_data)) + |> Map.put(:schema_data, schema_data) |> Map.put(:schema_sha256, schema_sha) end - # Re-encoded so the reader can decode it with ordered objects. - defp encode_schema(nil), do: nil - defp encode_schema(data) when is_map(data), do: Jason.encode!(data) - @impl Lightning.Adaptors.Strategy def fetch_icon(name, shape) when is_binary(name) and shape in [:square, :rectangle] do diff --git a/lib/lightning/adaptors/npm/schema.ex b/lib/lightning/adaptors/npm/schema.ex index b694db79be3..6140d024f17 100644 --- a/lib/lightning/adaptors/npm/schema.ex +++ b/lib/lightning/adaptors/npm/schema.ex @@ -3,8 +3,8 @@ defmodule Lightning.Adaptors.NPM.Schema do jsDelivr CDN client for adaptor configuration schemas. Fetches `/npm/@/configuration-schema.json` from - `cdn.jsdelivr.net`, decodes it, and returns `{schema_data, - schema_sha256}`. A genuine 404 (schema removed upstream) returns + `cdn.jsdelivr.net`, checks it decodes, and returns `{schema_data, + schema_sha256}` with the body kept as the bytes served. A genuine 404 (schema removed upstream) returns `{nil, nil}`; any other failure (timeout, other HTTP status, network error) returns `{nil, :fetch_failed}` so callers can tell "schema really doesn't exist" apart from "couldn't check right now." @@ -27,12 +27,12 @@ defmodule Lightning.Adaptors.NPM.Schema do genuine absence by callers that persist the result. """ @spec schema(String.t(), String.t()) :: - {map(), String.t()} | {nil, nil} | {nil, :fetch_failed} + {String.t(), String.t()} | {nil, nil} | {nil, :fetch_failed} def schema(name, version) do with {:ok, body} <- fetch_schema_bytes(name, version), - {:ok, data} <- Jason.decode(body) do + {:ok, _} <- Jason.decode(body) do sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) - {data, sha} + {body, sha} else {:error, {:http_status, 404}} -> {nil, nil} _ -> {nil, :fetch_failed} diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index a855a79f497..b726ff7fffd 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -6,7 +6,9 @@ defmodule Lightning.Adaptors.Store do catalogue table. `schema/2` and `versions/2` also fetch from the strategy when the row has no data yet and persist what they get; this only fills gaps on adaptors already in the catalogue, and an unknown - name returns `{:error, :not_found}`. `icon/3` returns a path on disk, + name returns `{:error, :not_found}`. A lazy fill that lands a value + broadcasts the change like a scheduler write; one whose fetch failed + returns `{:error, :unavailable}`. `icon/3` returns a path on disk, fetching the bytes from the strategy on the first miss. `catalogue/1` caches the picker payload already rendered, together with the ETag stamp that describes it. @@ -118,7 +120,7 @@ defmodule Lightning.Adaptors.Store do with {:ok, meta} <- icon_meta(sup, name), {:ok, ext} <- ext_for_shape(meta, shape), {:ok, expected_sha} <- sha256_for_shape(meta, shape) do - if disk_cache_matches?(source, name, shape, ext, expected_sha) do + if IconCache.cached?(source, name, shape, ext, expected_sha) do {:ok, IconCache.path(source, name, shape, ext)} else cache @@ -134,17 +136,6 @@ defmodule Lightning.Adaptors.Store do end end - # A cached file existing proves nothing about its content — a node that - # cached an earlier version of this icon keeps that file forever - # otherwise. A sha mismatch, or the file being absent, are both treated - # as a miss so the fetch branch below re-pulls and overwrites it. - defp disk_cache_matches?(source, name, shape, ext, expected_sha) do - case source |> IconCache.path(name, shape, ext) |> File.read() do - {:ok, bytes} -> :crypto.hash(:sha256, bytes) == expected_sha - {:error, _} -> false - end - end - defp fetch_icon_bytes(strategy, source, name, shape, ext, expected_sha) do case strategy.fetch_icon(name, shape) do {:ok, %{data: bytes, ext: ^ext}} -> @@ -300,7 +291,7 @@ defmodule Lightning.Adaptors.Store do # Lazy fetches only fill gaps on adaptors already in the catalogue; # they never add one. @spec fetch_and_persist(atom(), String.t(), :npm | :local, atom()) :: - {:commit, {:ok, term()}} | {:ignore, {:error, term()}} + {:commit, {:ok, term()}} | {:ignore, {:ok, term()} | {:error, term()}} defp fetch_and_persist(sup, name, source, field) do if Catalogue.get_adaptor(name, source) do fetch_and_persist_known(sup, name, source, field) @@ -315,11 +306,27 @@ defmodule Lightning.Adaptors.Store do record = Map.put(record, :source, source) {:ok, _} = Catalogue.upsert_adaptor(record) - # The strategy leaves a field off the record when its fetch failed - # transiently. Don't cache that as "no value"; let the next call retry. case Map.fetch(record, field) do - {:ok, value} -> {:commit, {:ok, project_field(value, field)}} - :error -> {:ignore, {:ok, project_field(nil, field)}} + # The source has nothing; cache that so the next read stays local. + {:ok, nil} -> + {:commit, {:ok, project_field(nil, field)}} + + # Landed a value: announce it like a scheduler write. The + # Invalidator drops the stale keys and the next read refills them + # from the row, so there is nothing to commit here. + {:ok, value} -> + Phoenix.PubSub.broadcast( + Lightning.PubSub, + AdaptorsSupervisor.source_topic(sup), + {:changed, name, source} + ) + + {:ignore, {:ok, project_field(value, field)}} + + # Left off the record: the fetch failed transiently. Say so rather + # than hand back an empty schema that would validate anything. + :error -> + {:ignore, {:error, :unavailable}} end {:ok, %{name: other}} -> @@ -330,7 +337,7 @@ defmodule Lightning.Adaptors.Store do end end - # Both cache paths must store the same projected shape. + # The value handed back here must match what a DB-backed read caches. defp project_field(rows, :versions) when is_list(rows), do: project_versions(rows) diff --git a/test/lightning/adaptors/icon_cache_test.exs b/test/lightning/adaptors/icon_cache_test.exs index 849ee02d599..61631cbc601 100644 --- a/test/lightning/adaptors/icon_cache_test.exs +++ b/test/lightning/adaptors/icon_cache_test.exs @@ -68,22 +68,31 @@ defmodule Lightning.Adaptors.IconCacheTest do end end - describe "cached?/4" do + describe "cached?/5" do + @sha :crypto.hash(:sha256, "x") + test "returns false when the file does not exist" do - refute IconCache.cached?(:npm, "definitely-missing", :square, "png") + refute IconCache.cached?(:npm, "definitely-missing", :square, "png", @sha) + end + + test "returns true after write!/5 places bytes with that sha" do + {:ok, sha} = IconCache.write!(:npm, "cached-pkg", :square, "png", "x") + assert sha == @sha + + assert IconCache.cached?(:npm, "cached-pkg", :square, "png", @sha) end - test "returns true after write!/5 places the file" do - {:ok, _sha} = IconCache.write!(:npm, "cached-pkg", :square, "png", "x") + test "returns false when the file on disk has other bytes" do + {:ok, _} = IconCache.write!(:npm, "stale-pkg", :square, "png", "old") - assert IconCache.cached?(:npm, "cached-pkg", :square, "png") + refute IconCache.cached?(:npm, "stale-pkg", :square, "png", @sha) end test "stays source-partitioned: a write to :npm doesn't satisfy :local" do {:ok, _} = IconCache.write!(:npm, "split-pkg", :square, "png", "x") - assert IconCache.cached?(:npm, "split-pkg", :square, "png") - refute IconCache.cached?(:local, "split-pkg", :square, "png") + assert IconCache.cached?(:npm, "split-pkg", :square, "png", @sha) + refute IconCache.cached?(:local, "split-pkg", :square, "png", @sha) end end diff --git a/test/lightning/adaptors/npm/schema_test.exs b/test/lightning/adaptors/npm/schema_test.exs index b4baa9ab0cd..a6c99c77090 100644 --- a/test/lightning/adaptors/npm/schema_test.exs +++ b/test/lightning/adaptors/npm/schema_test.exs @@ -37,7 +37,7 @@ defmodule Lightning.Adaptors.NPM.SchemaTest do end describe "schema/2" do - test "returns the decoded schema and a hex sha256 on 200", %{bypass: bypass} do + test "returns the raw body and a hex sha256 on 200", %{bypass: bypass} do schema = %{"type" => "object", "properties" => %{"baseUrl" => %{}}} body = Jason.encode!(schema) @@ -51,7 +51,7 @@ defmodule Lightning.Adaptors.NPM.SchemaTest do Plug.Conn.resp(conn, 200, body) end) - assert {^schema, ^expected_sha} = Schema.schema(@package, @version) + assert {^body, ^expected_sha} = Schema.schema(@package, @version) end test "returns {nil, nil} on 404", %{bypass: bypass} do diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index 10164c38c58..7e19f62816e 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -65,13 +65,18 @@ defmodule Lightning.Adaptors.StoreTest do Store.schema(sup, "@openfn/language-http") end - test "known adaptor with missing schema calls Strategy once, upserts to DB, caches result", + test "known adaptor with missing schema calls Strategy once, upserts to DB, broadcasts the change", %{ sup: sup, cache: cache } do source = AdaptorsSupervisor.source(sup) + Phoenix.PubSub.subscribe( + Lightning.PubSub, + AdaptorsSupervisor.source_topic(sup) + ) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) expect( @@ -86,14 +91,16 @@ defmodule Lightning.Adaptors.StoreTest do assert {:ok, ~s({"type":"object"})} = Store.schema(sup, "@openfn/language-http") + assert_receive {:changed, "@openfn/language-http", ^source} + assert %{schema_data: ~s({"type":"object"})} = Catalogue.get_adaptor("@openfn/language-http", source) - assert {:ok, {:ok, ~s({"type":"object"})}} = + assert {:ok, nil} = Cachex.get(cache, {:schema, "@openfn/language-http", source}) end - test "a failed schema fetch returns an empty schema without caching it", + test "a failed schema fetch returns an error without caching it", %{sup: sup, cache: cache} do source = AdaptorsSupervisor.source(sup) name = "@openfn/language-http" @@ -104,10 +111,26 @@ defmodule Lightning.Adaptors.StoreTest do {:ok, adaptor_record() |> Map.drop([:schema_data, :schema_sha256])} end) - assert {:ok, "{}"} = Store.schema(sup, name) + assert {:error, :unavailable} = Store.schema(sup, name) assert {:ok, nil} = Cachex.get(cache, {:schema, name, source}) end + test "an adaptor the source confirms has no schema caches an empty one", + %{sup: sup, cache: cache} do + source = AdaptorsSupervisor.source(sup) + name = "@openfn/language-http" + + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> + {:ok, adaptor_record(schema_data: nil)} + end) + + assert {:ok, "{}"} = Store.schema(sup, name) + assert {:ok, "{}"} = Store.schema(sup, name) + assert {:ok, {:ok, "{}"}} = Cachex.get(cache, {:schema, name, source}) + end + test "unknown adaptor returns {:error, :not_found} without calling Strategy or minting a row", %{sup: sup, cache: cache} do expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> @@ -223,7 +246,7 @@ defmodule Lightning.Adaptors.StoreTest do assert Enum.all?(versions, &Map.has_key?(&1, :deprecated)) end - test "known adaptor with no version rows calls Strategy and caches projected versions", + test "known adaptor with no version rows calls Strategy and returns projected versions", %{ sup: sup, cache: cache @@ -247,15 +270,13 @@ defmodule Lightning.Adaptors.StoreTest do assert {:ok, versions} = Store.versions(sup, "@openfn/language-http") assert length(versions) == 2 - assert {:ok, {:ok, cached_versions}} = - Cachex.get(cache, {:versions, "@openfn/language-http", source}) - - assert length(cached_versions) == 2 - - for cached <- cached_versions do - assert Map.keys(cached) |> Enum.sort() == + for v <- versions do + assert Map.keys(v) |> Enum.sort() == [:deprecated, :integrity, :published_at, :size_bytes, :version] end + + assert {:ok, nil} = + Cachex.get(cache, {:versions, "@openfn/language-http", source}) end test "a fetched record whose name differs from the requested name is refused", diff --git a/test/lightning/credentials_test.exs b/test/lightning/credentials_test.exs index fca0e5f1f2e..543e5731bed 100644 --- a/test/lightning/credentials_test.exs +++ b/test/lightning/credentials_test.exs @@ -1,8 +1,6 @@ defmodule Lightning.CredentialsTest do use Lightning.DataCase, async: true - @endpoint LightningWeb.Endpoint - alias Lightning.Auditing alias Lightning.Credentials alias Lightning.Credentials.Audit @@ -13,9 +11,6 @@ defmodule Lightning.CredentialsTest do import Lightning.Factories import Ecto.Query import Mox - import Phoenix.ConnTest - import Phoenix.LiveViewTest - import LightningWeb.CredentialLiveHelpers import Lightning.JobsFixtures @@ -2883,19 +2878,6 @@ defmodule Lightning.CredentialsTest do describe "get_schema/1" do setup :isolated_adaptors - test "returns a permissive schema for an adaptor with no schema, instead of crashing", - %{sup: sup} do - name = "@openfn/language-no-schema" - insert(:adaptor, name: name, schema_data: nil) - - cache = Lightning.Adaptors.Supervisor.cache_name(sup) - source = Lightning.Adaptors.Supervisor.source(sup) - Cachex.put(cache, {:schema, name, source}, {:ok, "{}"}) - - assert %Credentials.Schema{name: ^name, fields: []} = - Credentials.get_schema(name) - end - test "returns the adaptor's schema when one is present" do seed_credential_schema("http") @@ -2905,42 +2887,4 @@ defmodule Lightning.CredentialsTest do assert fields != [] end end - - describe "credential type picker (LiveView)" do - setup :isolated_adaptors - - setup do - Lightning.AccountsFixtures.superuser_fixture() - - conn = - LightningWeb.ConnCase.log_in_user(build_conn(), insert(:user)) - - %{conn: conn} - end - - test "omits a schema-less adaptor, closing the FunctionClauseError crash it used to reach", - %{conn: conn} do - insert(:adaptor, - name: "@openfn/language-with-schema", - schema_data: ~s({"type":"object"}) - ) - - insert(:adaptor, name: "@openfn/language-no-schema", schema_data: nil) - - {:ok, view, _html} = live(conn, "/credentials") - - html = open_create_credential_modal(view) - html_tree = Floki.parse_document!(html) - - assert Floki.find( - html_tree, - "label[for='credential-schema-picker_selected_@openfn/language-with-schema']" - ) != [] - - assert Floki.find( - html_tree, - "label[for='credential-schema-picker_selected_@openfn/language-no-schema']" - ) == [] - end - end end diff --git a/test/lightning_web/live/credential_live_test.exs b/test/lightning_web/live/credential_live_test.exs index ce1569308e4..84a6eaf7923 100644 --- a/test/lightning_web/live/credential_live_test.exs +++ b/test/lightning_web/live/credential_live_test.exs @@ -2809,6 +2809,29 @@ defmodule LightningWeb.CredentialLiveTest do "label[for='credential-schema-picker_selected_deprecated-adaptor']" ) == [] end + + test "omits an adaptor with no configuration schema", %{conn: conn} do + insert(:adaptor, name: "@openfn/language-no-schema", schema_data: nil) + + cache = AdaptorsSupervisor.cache_name(Config.default_instance()) + source = AdaptorsSupervisor.source(Config.default_instance()) + Cachex.del(cache, {:packages, source}) + + {:ok, view, _html} = live(conn, ~p"/credentials") + + html = open_create_credential_modal(view) + html_tree = Floki.parse_document!(html) + + assert Floki.find( + html_tree, + "label[for='credential-schema-picker_selected_@openfn/language-http']" + ) != [] + + assert Floki.find( + html_tree, + "label[for='credential-schema-picker_selected_@openfn/language-no-schema']" + ) == [] + end end describe "generic oauth credential" do diff --git a/test/test_helper.exs b/test/test_helper.exs index 741b6b2ce49..ce4e9365f65 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -80,7 +80,7 @@ Application.put_env(:lightning, Lightning.Extensions, # Pin the `Lightning.Adaptors.IconCache` on-disk path to a per-OS-PID # directory and wipe it at startup. Without the wipe, leftover files # from a prior run can mask a Mox expectation by short-circuiting -# `IconCache.cached?/4`, since `System.unique_integer/1` resets per-VM +# `IconCache.cached?/5`, since `System.unique_integer/1` resets per-VM # and recycles. Keying by OS PID also keeps concurrent `mix test` runs # (parallel CI shards, separate tmux panes) from colliding. icon_dir = From 14165b7381fe439032b6f7e88efc4755ef8ba92f Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Tue, 8 Sep 2026 16:02:42 +0200 Subject: [PATCH 15/37] Delete Store.versions/2 and Catalogue.list_missing_icons/1 - A failed lazy fetch named :fetch_failed, caching an empty version list - fetch_adaptor fails outright when the schema fetch fails - A stored schema is kept when a bumped version reports none - An icon that disagrees with its row is cached instead of refetched - Store.versions/2 and its projection deleted - Catalogue.list_missing_icons/1 deleted --- lib/lightning/adaptors.ex | 4 +- lib/lightning/adaptors/catalogue.ex | 26 ---- lib/lightning/adaptors/icon_cache.ex | 9 +- lib/lightning/adaptors/invalidator.ex | 14 +- lib/lightning/adaptors/local.ex | 18 +-- lib/lightning/adaptors/npm.ex | 50 +++---- lib/lightning/adaptors/npm/schema.ex | 27 ++-- lib/lightning/adaptors/scheduler.ex | 23 ++- lib/lightning/adaptors/store.ex | 111 ++++---------- lib/lightning/adaptors/strategy.ex | 8 +- test/lightning/adaptors/catalogue_test.exs | 65 +-------- test/lightning/adaptors/invalidator_test.exs | 19 ++- test/lightning/adaptors/local_test.exs | 9 ++ test/lightning/adaptors/npm/schema_test.exs | 16 +- test/lightning/adaptors/npm_test.exs | 12 +- test/lightning/adaptors/scheduler_test.exs | 116 +++++++++++++++ test/lightning/adaptors/store_test.exs | 146 ++++++------------- 17 files changed, 305 insertions(+), 368 deletions(-) diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index ee738f9f9ef..1e2422d7846 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -93,7 +93,9 @@ defmodule Lightning.Adaptors do @doc """ Returns the credential schema of the adaptor named `pkg`, as a JSON - binary. An adaptor with no schema yields `"{}"`. + binary. An adaptor with no schema yields `"{}"`; an unknown name is + `{:error, :not_found}` and a schema the source could not be reached + for is the strategy's own `{:error, reason}`. """ @spec schema(atom(), String.t()) :: {:ok, String.t()} | {:error, term()} def schema(sup \\ Config.default_instance(), pkg), do: Store.schema(sup, pkg) diff --git a/lib/lightning/adaptors/catalogue.ex b/lib/lightning/adaptors/catalogue.ex index 34889b4aafc..61adffc97e8 100644 --- a/lib/lightning/adaptors/catalogue.ex +++ b/lib/lightning/adaptors/catalogue.ex @@ -217,32 +217,6 @@ defmodule Lightning.Adaptors.Catalogue do :ok end - @doc """ - Lean list of source-scoped adaptors that are missing at least one icon - shape. Returns only the fields the Scheduler needs to decide whether to - re-apply the bulk icon fetch result. - """ - @spec list_missing_icons(source()) :: [ - %{ - name: String.t(), - icon_square_sha256: binary() | nil, - icon_rectangle_sha256: binary() | nil - } - ] - def list_missing_icons(source) do - Repo.all( - from a in Adaptor, - where: - a.source == ^source and - (is_nil(a.icon_square_sha256) or is_nil(a.icon_rectangle_sha256)), - select: %{ - name: a.name, - icon_square_sha256: a.icon_square_sha256, - icon_rectangle_sha256: a.icon_rectangle_sha256 - } - ) - end - @doc """ Update only the icon columns for a single `(name, source)` row. diff --git a/lib/lightning/adaptors/icon_cache.ex b/lib/lightning/adaptors/icon_cache.ex index 3dc28209d10..2e640d68131 100644 --- a/lib/lightning/adaptors/icon_cache.ex +++ b/lib/lightning/adaptors/icon_cache.ex @@ -19,9 +19,12 @@ defmodule Lightning.Adaptors.IconCache do Concurrent first-request fetchers are coalesced upstream by Cachex's courier on `{:icon_bytes, source, name, shape}` inside - `Lightning.Adaptors.Store.icon/3` — the courier returns `{:ignore, _}` - so no entry is committed, but all in-flight peers receive the courier's - result for free. The temp-then-rename in `write!/5` is the belt-and- + `Lightning.Adaptors.Store.icon/3`, and all in-flight peers receive the + courier's result for free. Bytes that verify are left uncommitted — + this directory is their cache — but bytes that disagree with the row's + sha or extension are committed as an error, so the disagreement is not + re-fetched from the source on every request until the row moves. + The temp-then-rename in `write!/5` is the belt-and- braces guarantee for the file-write step itself: readers never observe a half-written file. """ diff --git a/lib/lightning/adaptors/invalidator.ex b/lib/lightning/adaptors/invalidator.ex index 0b3b1f5eba0..5342c82f868 100644 --- a/lib/lightning/adaptors/invalidator.ex +++ b/lib/lightning/adaptors/invalidator.ex @@ -4,10 +4,13 @@ defmodule Lightning.Adaptors.Invalidator do local Cachex entries, keeping each node coherent with Postgres. Subscribes to `opts[:source_topic]` on `Lightning.PubSub` at init. - On `{:changed, name, source}`, deletes the five cache keys written by - `Lightning.Adaptors.Store`: the three keyed by name (`:schema`, - `:versions`, `:icon_meta`) plus the two source-wide ones (`:packages`, - `:catalogue`), which any change invalidates. No source filtering on the + On `{:changed, name, source}`, deletes the six cache keys written by + `Lightning.Adaptors.Store`: the four keyed by name (`:schema`, + `:icon_meta` and the two `:icon_bytes` shapes) plus the two + source-wide ones (`:packages`, `:catalogue`), which any change + invalidates. Dropping `:icon_bytes` is what lets a committed icon error + clear: the row moving is the only thing that can resolve it, and the row + moving always broadcasts. No source filtering on the hot path — a broadcast for a source that isn't active on this node is a no-op because those keys simply don't exist in Cachex. """ @@ -39,8 +42,9 @@ defmodule Lightning.Adaptors.Invalidator do @impl true def handle_info({:changed, name, source}, state) do Cachex.del(state.cache, {:schema, name, source}) - Cachex.del(state.cache, {:versions, name, source}) Cachex.del(state.cache, {:icon_meta, name, source}) + Cachex.del(state.cache, {:icon_bytes, source, name, :square}) + Cachex.del(state.cache, {:icon_bytes, source, name, :rectangle}) Cachex.del(state.cache, {:packages, source}) Cachex.del(state.cache, {:catalogue, source}) {:noreply, state} diff --git a/lib/lightning/adaptors/local.ex b/lib/lightning/adaptors/local.ex index d75abcad923..9440a808ad6 100644 --- a/lib/lightning/adaptors/local.ex +++ b/lib/lightning/adaptors/local.ex @@ -56,7 +56,7 @@ defmodule Lightning.Adaptors.Local do with {:ok, records} <- discover() do case Enum.find(records, &(&1.name == name)) do nil -> {:error, :not_found} - record -> {:ok, build_adaptor_record(record)} + record -> build_adaptor_record(record) end end end @@ -222,12 +222,12 @@ defmodule Lightning.Adaptors.Local do versions: Enum.map(record.versions, &build_version_record/1) } - case read_schema(record.latest_path) do - {:ok, schema_data, schema_sha256} -> - Map.merge(base, %{schema_data: schema_data, schema_sha256: schema_sha256}) - - :unreadable -> - base + with {:ok, schema_data, schema_sha256} <- read_schema(record.latest_path) do + {:ok, + Map.merge(base, %{ + schema_data: schema_data, + schema_sha256: schema_sha256 + })} end end @@ -244,8 +244,6 @@ defmodule Lightning.Adaptors.Local do } end - # A missing file is "no schema"; anything else (a permissions error, a - # half-written file) is left off the record per `Strategy.adaptor_record/0`. defp read_schema(dir) do with {:ok, body} <- File.read(Path.join(dir, @schema_filename)), {:ok, _} <- Jason.decode(body) do @@ -253,7 +251,7 @@ defmodule Lightning.Adaptors.Local do {:ok, body, sha} else {:error, :enoent} -> {:ok, nil, nil} - _ -> :unreadable + {:error, reason} -> {:error, {:schema_fetch_failed, reason}} end end diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index 6b66ae2f0ad..ede808b1a27 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -40,10 +40,10 @@ defmodule Lightning.Adaptors.NPM do transient failures (5xx, timeout, nxdomain) of the *primary* request (`packument` for `fetch_adaptor/1`, the org package listing for `list_adaptors/0` and `fetch_icons/1`) surface as `{:error, term()}` - unchanged. The schema fetch inside `fetch_adaptor/1` and each icon - fetch inside `fetch_icons/1` are best-effort instead: a miss there - degrades to a nil schema or an absent icon shape, rather than failing - the whole record or batch. + unchanged, as does a failed schema fetch inside `fetch_adaptor/1` + (`{:error, {:schema_fetch_failed, reason}}`). Each icon fetch inside + `fetch_icons/1` is best-effort instead: a miss there degrades to an + absent icon shape rather than failing the batch. ## Configuration @@ -67,30 +67,30 @@ defmodule Lightning.Adaptors.NPM do @impl Lightning.Adaptors.Strategy def fetch_adaptor(name) when is_binary(name) do with {:ok, packument} <- Registry.get_packument(name), - {:ok, latest_version} <- Registry.latest_version(packument) do - base = %{ - name: Map.get(packument, "name", name), - description: Map.get(packument, "description"), - homepage: Map.get(packument, "homepage"), - repository: Registry.repository_url(Map.get(packument, "repository")), - license: Map.get(packument, "license"), - latest_version: latest_version, - deprecated: Registry.deprecated?(packument, latest_version), - versions: Registry.build_versions(packument) - } - - {:ok, put_schema(base, Schema.schema(name, latest_version))} + {:ok, latest_version} <- Registry.latest_version(packument), + {:ok, {schema_data, schema_sha256}} <- + schema(name, latest_version) do + {:ok, + %{ + name: Map.get(packument, "name", name), + description: Map.get(packument, "description"), + homepage: Map.get(packument, "homepage"), + repository: Registry.repository_url(Map.get(packument, "repository")), + license: Map.get(packument, "license"), + latest_version: latest_version, + deprecated: Registry.deprecated?(packument, latest_version), + versions: Registry.build_versions(packument), + schema_data: schema_data, + schema_sha256: schema_sha256 + }} end end - # Absent, not nil: `Ecto.Changeset.cast/3` overwrites a column for any - # present key, and `Strategy.adaptor_record/0` reserves nil for "no schema". - defp put_schema(record, {nil, :fetch_failed}), do: record - - defp put_schema(record, {schema_data, schema_sha}) do - record - |> Map.put(:schema_data, schema_data) - |> Map.put(:schema_sha256, schema_sha) + defp schema(name, version) do + case Schema.schema(name, version) do + {:ok, pair} -> {:ok, pair} + {:error, reason} -> {:error, {:schema_fetch_failed, reason}} + end end @impl Lightning.Adaptors.Strategy diff --git a/lib/lightning/adaptors/npm/schema.ex b/lib/lightning/adaptors/npm/schema.ex index 6140d024f17..801479ce6f3 100644 --- a/lib/lightning/adaptors/npm/schema.ex +++ b/lib/lightning/adaptors/npm/schema.ex @@ -3,11 +3,11 @@ defmodule Lightning.Adaptors.NPM.Schema do jsDelivr CDN client for adaptor configuration schemas. Fetches `/npm/@/configuration-schema.json` from - `cdn.jsdelivr.net`, checks it decodes, and returns `{schema_data, - schema_sha256}` with the body kept as the bytes served. A genuine 404 (schema removed upstream) returns - `{nil, nil}`; any other failure (timeout, other HTTP status, network - error) returns `{nil, :fetch_failed}` so callers can tell "schema - really doesn't exist" apart from "couldn't check right now." + `cdn.jsdelivr.net`, checks it decodes, and returns `{:ok, + {schema_data, schema_sha256}}` with the body kept as the bytes + served. A genuine 404 (schema removed upstream) is `{:ok, {nil, + nil}}`; any other failure (timeout, other HTTP status, network error, + undecodable body) is `{:error, reason}`. Base URL via `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)[:jsdelivr_url]`, default `https://cdn.jsdelivr.net`. @@ -21,21 +21,22 @@ defmodule Lightning.Adaptors.NPM.Schema do @doc """ Fetch the configuration schema for `name@version` from jsDelivr. - Returns `{schema_data, schema_sha256}` on success, `{nil, nil}` on a - genuine 404 (schema removed upstream), or `{nil, :fetch_failed}` on - any other failure — a transient failure must not be mistaken for - genuine absence by callers that persist the result. + Returns `{:ok, {schema_data, schema_sha256}}` on success, `{:ok, + {nil, nil}}` on a genuine 404 (schema removed upstream), and + `{:error, reason}` on any other failure — a transient failure must + not be mistaken for genuine absence by callers that persist the + result. """ @spec schema(String.t(), String.t()) :: - {String.t(), String.t()} | {nil, nil} | {nil, :fetch_failed} + {:ok, {String.t(), String.t()}} | {:ok, {nil, nil}} | {:error, term()} def schema(name, version) do with {:ok, body} <- fetch_schema_bytes(name, version), {:ok, _} <- Jason.decode(body) do sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) - {body, sha} + {:ok, {body, sha}} else - {:error, {:http_status, 404}} -> {nil, nil} - _ -> {nil, :fetch_failed} + {:error, {:http_status, 404}} -> {:ok, {nil, nil}} + {:error, reason} -> {:error, reason} end end diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 3e3d430e83a..7eca2852567 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -342,8 +342,7 @@ defmodule Lightning.Adaptors.Scheduler do existing_rows = Catalogue.list_adaptors(state.source) prior_etags = prior_etags_from_rows(existing_rows) - existing_by_name = - Map.new(existing_rows, fn a -> {a.name, a.latest_version} end) + existing_by_name = Map.new(existing_rows, fn a -> {a.name, a} end) icons_task = Task.Supervisor.async_nolink(state.tasks, fn -> @@ -425,7 +424,9 @@ defmodule Lightning.Adaptors.Scheduler do existing_by_name, state ) do - if Map.get(existing_by_name, name) == version do + existing = Map.get(existing_by_name, name) + + if existing && existing.latest_version == version do Catalogue.touch_checked_at(name, state.source) :touched else @@ -433,7 +434,7 @@ defmodule Lightning.Adaptors.Scheduler do {:ok, %{latest_version: version} = record} -> Logger.debug("Adaptors[#{state.source}]: fetched #{name}@#{version}") - {:fetched, record} + {:fetched, keep_stored_schema(record, existing)} {:error, reason} -> Logger.warning( @@ -445,6 +446,20 @@ defmodule Lightning.Adaptors.Scheduler do end end + # jsDelivr 404s for a version it has not mirrored yet, which is + # indistinguishable from a schema the source really dropped. On the + # periodic path we keep what we have; an operator refresh takes upstream + # as-is and is where a real removal lands. + defp keep_stored_schema( + %{schema_data: nil} = record, + %{schema_data: stored} = row + ) + when not is_nil(stored) do + %{record | schema_data: stored, schema_sha256: row.schema_sha256} + end + + defp keep_stored_schema(record, _existing), do: record + defp await_icons(task) do case Task.yield(task, @icons_task_timeout) || Task.shutdown(task) do {:ok, {:ok, map}} when is_map(map) -> diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index b726ff7fffd..8955342ae3c 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -3,12 +3,12 @@ defmodule Lightning.Adaptors.Store do Cached reads over `Lightning.Adaptors.Catalogue`. Every read checks the instance's Cachex first and falls back to the - catalogue table. `schema/2` and `versions/2` also fetch from the - strategy when the row has no data yet and persist what they get; this - only fills gaps on adaptors already in the catalogue, and an unknown - name returns `{:error, :not_found}`. A lazy fill that lands a value + catalogue table. `schema/2` also fetches from the strategy when the + row has no data yet and persists what it gets; this only fills gaps on + adaptors already in the catalogue, and an unknown name returns + `{:error, :not_found}`. A lazy fill that lands a value broadcasts the change like a scheduler write; one whose fetch failed - returns `{:error, :unavailable}`. `icon/3` returns a path on disk, + passes the strategy's error through. `icon/3` returns a path on disk, fetching the bytes from the strategy on the first miss. `catalogue/1` caches the picker payload already rendered, together with the ETag stamp that describes it. @@ -22,14 +22,6 @@ defmodule Lightning.Adaptors.Store do @type sup :: atom() - @type version_meta :: %{ - version: String.t(), - integrity: String.t() | nil, - size_bytes: integer() | nil, - published_at: DateTime.t() | nil, - deprecated: boolean() - } - @type icon_meta :: %{ icon_square_ext: String.t() | nil, icon_rectangle_ext: String.t() | nil, @@ -71,34 +63,14 @@ defmodule Lightning.Adaptors.Store do {:schema, name, source}, fn _key -> case Catalogue.get_adaptor(name, source) do + nil -> + {:ignore, {:error, :not_found}} + %{schema_data: data} when not is_nil(data) -> {:commit, {:ok, data}} _ -> - fetch_and_persist(sup, name, source, :schema_data) - end - end, - timeout: Config.cache_timeout_ms() - ) - |> unwrap() - end - - @doc """ - Returns the adaptor's version history as `t:version_meta/0` maps. - """ - @spec versions(sup(), String.t()) :: - {:ok, [version_meta()]} | {:error, term()} - def versions(sup, name) do - cache = AdaptorsSupervisor.cache_name(sup) - source = AdaptorsSupervisor.source(sup) - - cache - |> Cachex.fetch( - {:versions, name, source}, - fn _key -> - case Catalogue.list_versions(name, source) do - [] -> fetch_and_persist(sup, name, source, :versions) - rows -> {:commit, {:ok, project_versions(rows)}} + fetch_and_persist(sup, name, source) end end, timeout: Config.cache_timeout_ms() @@ -145,13 +117,15 @@ defmodule Lightning.Adaptors.Store do {:ignore, {:ok, IconCache.path(source, name, shape, ext)}} got -> - {:ignore, + {:commit, {:error, {:icon_sha_mismatch, expected: expected_sha, got: got}}} end {:ok, %{ext: other_ext}} -> - {:ignore, {:error, {:ext_mismatch, expected: ext, got: other_ext}}} + {:commit, {:error, {:ext_mismatch, expected: ext, got: other_ext}}} + # A transport failure says nothing about the icon, so it is never + # cached; only a disagreement between the row and the bytes is. {:error, _} = err -> {:ignore, err} end @@ -288,45 +262,30 @@ defmodule Lightning.Adaptors.Store do } end - # Lazy fetches only fill gaps on adaptors already in the catalogue; - # they never add one. - @spec fetch_and_persist(atom(), String.t(), :npm | :local, atom()) :: + @spec fetch_and_persist(atom(), String.t(), :npm | :local) :: {:commit, {:ok, term()}} | {:ignore, {:ok, term()} | {:error, term()}} - defp fetch_and_persist(sup, name, source, field) do - if Catalogue.get_adaptor(name, source) do - fetch_and_persist_known(sup, name, source, field) - else - {:ignore, {:error, :not_found}} - end - end - - defp fetch_and_persist_known(sup, name, source, field) do + defp fetch_and_persist(sup, name, source) do case AdaptorsSupervisor.strategy(sup).fetch_adaptor(name) do {:ok, %{name: ^name} = record} -> record = Map.put(record, :source, source) {:ok, _} = Catalogue.upsert_adaptor(record) - case Map.fetch(record, field) do + case record.schema_data do # The source has nothing; cache that so the next read stays local. - {:ok, nil} -> - {:commit, {:ok, project_field(nil, field)}} + nil -> + {:commit, {:ok, "{}"}} # Landed a value: announce it like a scheduler write. The # Invalidator drops the stale keys and the next read refills them # from the row, so there is nothing to commit here. - {:ok, value} -> + value -> Phoenix.PubSub.broadcast( Lightning.PubSub, AdaptorsSupervisor.source_topic(sup), {:changed, name, source} ) - {:ignore, {:ok, project_field(value, field)}} - - # Left off the record: the fetch failed transiently. Say so rather - # than hand back an empty schema that would validate anything. - :error -> - {:ignore, {:error, :unavailable}} + {:ignore, {:ok, value}} end {:ok, %{name: other}} -> @@ -337,13 +296,6 @@ defmodule Lightning.Adaptors.Store do end end - # The value handed back here must match what a DB-backed read caches. - defp project_field(rows, :versions) when is_list(rows), - do: project_versions(rows) - - defp project_field(nil, :schema_data), do: "{}" - defp project_field(value, _field), do: value - @spec project_icon_meta(map()) :: icon_meta() defp project_icon_meta(adaptor) do Map.take(adaptor, [ @@ -354,20 +306,6 @@ defmodule Lightning.Adaptors.Store do ]) end - @spec project_versions([map()]) :: [version_meta()] - defp project_versions(rows) do - Enum.map( - rows, - &Map.take(&1, [ - :version, - :integrity, - :size_bytes, - :published_at, - :deprecated - ]) - ) - end - @spec ext_for_shape(icon_meta(), :square | :rectangle) :: {:ok, String.t()} | {:error, :not_found} defp ext_for_shape(meta, shape) do @@ -392,10 +330,11 @@ defmodule Lightning.Adaptors.Store do # * `{:ignore, value}` — fallback ran and chose not to cache # * `{:error, term}` — Cachex-side failure (fallback raised, etc.) # - # Our fallbacks return `{:commit, {:ok, _}}` / `{:ignore, {:error, _}}`, - # so the wrapper tuple's second element is itself the public - # `{:ok, _} | {:error, _}` we want to return. Cachex-side `{:error, _}` - # passes through unchanged. + # Every fallback returns an inner `{:ok, _} | {:error, _}`, whichever + # wrapper it chooses, so the wrapper tuple's second element is itself + # the public value we want to return — including a committed + # `{:error, _}`, which comes back as `{:ok, {:error, _}}` on a later + # hit. Cachex-side `{:error, _}` passes through unchanged. @spec unwrap(tuple()) :: {:ok, term()} | {:error, term()} defp unwrap({:ok, inner}), do: inner defp unwrap({:commit, inner}), do: inner diff --git a/lib/lightning/adaptors/strategy.ex b/lib/lightning/adaptors/strategy.ex index 6d8a444abd4..e6c273874bb 100644 --- a/lib/lightning/adaptors/strategy.ex +++ b/lib/lightning/adaptors/strategy.ex @@ -49,8 +49,8 @@ defmodule Lightning.Adaptors.Strategy do `c:fetch_icons/1` — they are not stamped onto this record. `schema_data` is the credential schema as a JSON binary. `nil` means - the adaptor has no schema; leaving the key off means the schema fetch - failed transiently and the caller should keep what it already has. + the source sees no schema for this version; the Scheduler decides + whether that replaces a stored one. """ @type adaptor_record :: %{ required(:name) => String.t(), @@ -60,8 +60,8 @@ defmodule Lightning.Adaptors.Strategy do required(:license) => String.t() | nil, required(:latest_version) => String.t(), required(:deprecated) => boolean(), - optional(:schema_data) => String.t() | nil, - optional(:schema_sha256) => String.t() | nil, + required(:schema_data) => String.t() | nil, + required(:schema_sha256) => String.t() | nil, required(:versions) => [version_record()] } diff --git a/test/lightning/adaptors/catalogue_test.exs b/test/lightning/adaptors/catalogue_test.exs index 85d55fe6d65..1235de20cc7 100644 --- a/test/lightning/adaptors/catalogue_test.exs +++ b/test/lightning/adaptors/catalogue_test.exs @@ -85,27 +85,7 @@ defmodule Lightning.Adaptors.CatalogueTest do end end - describe "upsert_adaptor/1 — schema preservation on transient fetch failure" do - test "transient failure keeps the old schema" do - {:ok, first} = - Catalogue.upsert_adaptor( - adaptor_record( - schema_data: %{"type" => "object"}, - schema_sha256: "abc" - ) - ) - - {:ok, second} = - Catalogue.upsert_adaptor( - adaptor_record() - |> Map.drop([:schema_data, :schema_sha256]) - ) - - assert second.id == first.id - assert second.schema_data == first.schema_data - assert second.schema_sha256 == first.schema_sha256 - end - + describe "upsert_adaptor/1 — schema clearing" do test "genuinely removed schema does clear" do {:ok, first} = Catalogue.upsert_adaptor( @@ -467,49 +447,6 @@ defmodule Lightning.Adaptors.CatalogueTest do end end - describe "list_missing_icons/1" do - test "returns rows where either icon shape sha256 is nil" do - {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/a")) - - {:ok, _} = - Catalogue.upsert_adaptor( - adaptor_record( - name: "@openfn/b", - icon_square_ext: "png", - icon_square_sha256: :crypto.hash(:sha256, "x") - ) - ) - - {:ok, _} = - Catalogue.upsert_adaptor( - adaptor_record( - name: "@openfn/c", - icon_square_ext: "png", - icon_square_sha256: :crypto.hash(:sha256, "y"), - icon_rectangle_ext: "png", - icon_rectangle_sha256: :crypto.hash(:sha256, "z") - ) - ) - - names = - Catalogue.list_missing_icons(:npm) - |> Enum.map(& &1.name) - |> Enum.sort() - - assert names == ["@openfn/a", "@openfn/b"] - end - - test "is source-scoped" do - {:ok, _} = - Catalogue.upsert_adaptor( - adaptor_record(name: "@openfn/x", source: :local) - ) - - assert Catalogue.list_missing_icons(:npm) == [] - assert [%{name: "@openfn/x"}] = Catalogue.list_missing_icons(:local) - end - end - describe "update_icons/3" do test "writes only icon columns and bumps :updated_at" do {:ok, before} = Catalogue.upsert_adaptor(adaptor_record()) diff --git a/test/lightning/adaptors/invalidator_test.exs b/test/lightning/adaptors/invalidator_test.exs index 6417e1f28d9..1b40430e7ae 100644 --- a/test/lightning/adaptors/invalidator_test.exs +++ b/test/lightning/adaptors/invalidator_test.exs @@ -25,7 +25,7 @@ defmodule Lightning.Adaptors.InvalidatorTest do end describe "handle_info/2 - {:changed, name, source}" do - test "evicts all five matching cache keys on broadcast", %{ + test "evicts all six matching cache keys on broadcast", %{ sup: sup, cache: cache, inv_name: inv_name @@ -35,7 +35,6 @@ defmodule Lightning.Adaptors.InvalidatorTest do name = "@openfn/language-http" Cachex.put!(cache, {:schema, name, source}, {:ok, %{"type" => "object"}}) - Cachex.put!(cache, {:versions, name, source}, {:ok, [%{version: "1.0.0"}]}) Cachex.put!( cache, @@ -43,6 +42,14 @@ defmodule Lightning.Adaptors.InvalidatorTest do {:ok, %{icon_square_ext: "svg"}} ) + Cachex.put!(cache, {:icon_bytes, source, name, :square}, {:ok, "/a.png"}) + + Cachex.put!( + cache, + {:icon_bytes, source, name, :rectangle}, + {:error, {:ext_mismatch, expected: "png", got: "svg"}} + ) + Cachex.put!(cache, {:packages, source}, {:ok, [%{name: name}]}) Cachex.put!(cache, {:catalogue, source}, {:ok, {{nil, 0}, []}}) @@ -55,8 +62,12 @@ defmodule Lightning.Adaptors.InvalidatorTest do :sys.get_state(inv_name) assert {:ok, nil} = Cachex.get(cache, {:schema, name, source}) - assert {:ok, nil} = Cachex.get(cache, {:versions, name, source}) assert {:ok, nil} = Cachex.get(cache, {:icon_meta, name, source}) + assert {:ok, nil} = Cachex.get(cache, {:icon_bytes, source, name, :square}) + + assert {:ok, nil} = + Cachex.get(cache, {:icon_bytes, source, name, :rectangle}) + assert {:ok, nil} = Cachex.get(cache, {:packages, source}) assert {:ok, nil} = Cachex.get(cache, {:catalogue, source}) end @@ -78,7 +89,6 @@ defmodule Lightning.Adaptors.InvalidatorTest do {:ok, %{"type" => "object"}} ) - Cachex.put!(cache, {:versions, bystander, source}, {:ok, []}) Cachex.put!(cache, {:icon_meta, bystander, source}, {:ok, %{}}) Phoenix.PubSub.broadcast!( @@ -90,7 +100,6 @@ defmodule Lightning.Adaptors.InvalidatorTest do :sys.get_state(inv_name) assert {:ok, {:ok, _}} = Cachex.get(cache, {:schema, bystander, source}) - assert {:ok, {:ok, _}} = Cachex.get(cache, {:versions, bystander, source}) assert {:ok, {:ok, _}} = Cachex.get(cache, {:icon_meta, bystander, source}) end diff --git a/test/lightning/adaptors/local_test.exs b/test/lightning/adaptors/local_test.exs index 3a37d8fa4f6..6234ba52245 100644 --- a/test/lightning/adaptors/local_test.exs +++ b/test/lightning/adaptors/local_test.exs @@ -328,6 +328,15 @@ defmodule Lightning.Adaptors.LocalTest do assert record.schema_sha256 == nil end + test "fails the whole record when the schema file cannot be read", + %{root: root} do + dir = write_package!(root, "locked", "@openfn/language-locked", "1.0.0") + File.mkdir!(Path.join(dir, "configuration-schema.json")) + + assert {:error, {:schema_fetch_failed, _reason}} = + Local.fetch_adaptor("@openfn/language-locked") + end + test "handles a plain-string repository field", %{root: root} do write_package_raw!(root, "p", %{ "name" => "@openfn/language-p", diff --git a/test/lightning/adaptors/npm/schema_test.exs b/test/lightning/adaptors/npm/schema_test.exs index a6c99c77090..0c61e5ca031 100644 --- a/test/lightning/adaptors/npm/schema_test.exs +++ b/test/lightning/adaptors/npm/schema_test.exs @@ -51,7 +51,7 @@ defmodule Lightning.Adaptors.NPM.SchemaTest do Plug.Conn.resp(conn, 200, body) end) - assert {^body, ^expected_sha} = Schema.schema(@package, @version) + assert {:ok, {^body, ^expected_sha}} = Schema.schema(@package, @version) end test "returns {nil, nil} on 404", %{bypass: bypass} do @@ -59,32 +59,32 @@ defmodule Lightning.Adaptors.NPM.SchemaTest do Plug.Conn.resp(conn, 404, "") end) - assert {nil, nil} = Schema.schema(@package, @version) + assert {:ok, {nil, nil}} = Schema.schema(@package, @version) end - test "returns {nil, :fetch_failed} on 5xx", %{bypass: bypass} do + test "returns an error on 5xx", %{bypass: bypass} do Bypass.expect(bypass, "GET", @path, fn conn -> Plug.Conn.resp(conn, 500, "") end) - assert {nil, :fetch_failed} = Schema.schema(@package, @version) + assert {:error, {:http_status, 500}} = Schema.schema(@package, @version) end - test "returns {nil, :fetch_failed} on invalid JSON body", %{ + test "returns an error on invalid JSON body", %{ bypass: bypass } do Bypass.expect(bypass, "GET", @path, fn conn -> Plug.Conn.resp(conn, 200, "this is not json {") end) - assert {nil, :fetch_failed} = Schema.schema(@package, @version) + assert {:error, %Jason.DecodeError{}} = Schema.schema(@package, @version) end - test "returns {nil, :fetch_failed} on connection refused", %{ + test "returns an error on connection refused", %{ bypass: bypass } do Bypass.down(bypass) - assert {nil, :fetch_failed} = Schema.schema(@package, @version) + assert {:error, _reason} = Schema.schema(@package, @version) end end end diff --git a/test/lightning/adaptors/npm_test.exs b/test/lightning/adaptors/npm_test.exs index 3729a006434..9c2944ab724 100644 --- a/test/lightning/adaptors/npm_test.exs +++ b/test/lightning/adaptors/npm_test.exs @@ -108,7 +108,7 @@ defmodule Lightning.Adaptors.NPMTest do assert old.deprecated == true end - test "omits schema_data/schema_sha256 entirely when jsDelivr returns 5xx", + test "fails the whole record when jsDelivr returns 5xx", %{ registry: registry, jsdelivr: jsdelivr @@ -123,14 +123,8 @@ defmodule Lightning.Adaptors.NPMTest do Plug.Conn.resp(conn, 500, "") end) - {:ok, record} = NPM.fetch_adaptor(@package) - - refute Map.has_key?(record, :schema_data), - "a transient schema-fetch failure must omit the key, not set it to nil, so cast/3 leaves the persisted schema untouched" - - refute Map.has_key?(record, :schema_sha256) - assert record.name == @package - assert record.latest_version == @latest_version + assert {:error, {:schema_fetch_failed, _reason}} = + NPM.fetch_adaptor(@package) end end diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index 7b4190ccccb..18f984c2065 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -316,6 +316,52 @@ defmodule Lightning.Adaptors.SchedulerTest do assert row.latest_version == "2.0.0" end + test "bumped version reporting no schema keeps the stored schema", %{ + sup: sup + } do + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + schema_data: ~s({"type":"object"}), + schema_sha256: "sha-1" + ) + ) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-http", latest_version: "2.0.0"}]} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> + {:ok, + adaptor_record( + latest_version: "2.0.0", + schema_data: nil, + schema_sha256: nil + )} + end + ) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + Scheduler.refresh_now(sched_name) + + assert_receive {:changed, "@openfn/language-http", ^source}, 2000 + + row = Catalogue.get_adaptor("@openfn/language-http", source) + assert row.latest_version == "2.0.0" + assert row.schema_data == ~s({"type":"object"}) + assert row.schema_sha256 == "sha-1" + end + test "new adaptor (not in DB): upsert and broadcast", %{sup: sup} do source = AdaptorsSupervisor.source(sup) source_topic = AdaptorsSupervisor.source_topic(sup) @@ -340,6 +386,40 @@ defmodule Lightning.Adaptors.SchedulerTest do assert Catalogue.get_adaptor("@openfn/language-new", source) != nil end + test "a failed fetch persists nothing, and the next tick retries", %{ + sup: sup + } do + test_pid = self() + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + name = "@openfn/language-new" + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 2, fn -> + {:ok, [%{name: name, latest_version: "1.0.0"}]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> + send(test_pid, :first_fetch) + {:error, {:schema_fetch_failed, :timeout}} + end) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + assert_receive :first_fetch, 2000 + refute_receive {:changed, _, _}, 200 + assert Catalogue.get_adaptor(name, source) == nil + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> + {:ok, adaptor_record(name: name)} + end) + + Scheduler.refresh_now(AdaptorsSupervisor.global_scheduler_name(sup)) + + assert_receive {:changed, ^name, ^source}, 2000 + assert Catalogue.get_adaptor(name, source) != nil + end + test "list_adaptors error: no DB writes, no broadcasts", %{sup: sup} do test_pid = self() source_topic = AdaptorsSupervisor.source_topic(sup) @@ -787,6 +867,42 @@ defmodule Lightning.Adaptors.SchedulerTest do assert Catalogue.get_adaptor("@openfn/language-http", source) != nil end + test "clears a stored schema when the fetched record has none", %{sup: sup} do + test_pid = self() + source = AdaptorsSupervisor.source(sup) + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + schema_data: ~s({"type":"object"}), + schema_sha256: "sha-1" + ) + ) + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :init_list_adaptors_called) + {:ok, []} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> + {:ok, adaptor_record(schema_data: nil, schema_sha256: nil)} + end + ) + + start_scheduler(sup) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + assert :ok = Scheduler.refresh_package(sched_name, "@openfn/language-http") + + row = Catalogue.get_adaptor("@openfn/language-http", source) + assert row.schema_data == nil + assert row.schema_sha256 == nil + end + test "returns error tuple when fetch_adaptor fails", %{sup: sup} do test_pid = self() diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index 7e19f62816e..9b4ad3a3b4f 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -100,21 +100,6 @@ defmodule Lightning.Adaptors.StoreTest do Cachex.get(cache, {:schema, "@openfn/language-http", source}) end - test "a failed schema fetch returns an error without caching it", - %{sup: sup, cache: cache} do - source = AdaptorsSupervisor.source(sup) - name = "@openfn/language-http" - - {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) - - expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> - {:ok, adaptor_record() |> Map.drop([:schema_data, :schema_sha256])} - end) - - assert {:error, :unavailable} = Store.schema(sup, name) - assert {:ok, nil} = Cachex.get(cache, {:schema, name, source}) - end - test "an adaptor the source confirms has no schema caches an empty one", %{sup: sup, cache: cache} do source = AdaptorsSupervisor.source(sup) @@ -226,94 +211,6 @@ defmodule Lightning.Adaptors.StoreTest do end end - describe "versions/2" do - test "cache miss + DB hit returns projected versions without calling Strategy", - %{sup: sup} do - expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> - :unreachable - end) - - {:ok, _} = - Catalogue.upsert_adaptor( - adaptor_record( - versions: [version_record("1.0.0"), version_record("1.1.0")] - ) - ) - - assert {:ok, versions} = Store.versions(sup, "@openfn/language-http") - assert length(versions) == 2 - assert Enum.all?(versions, &Map.has_key?(&1, :version)) - assert Enum.all?(versions, &Map.has_key?(&1, :deprecated)) - end - - test "known adaptor with no version rows calls Strategy and returns projected versions", - %{ - sup: sup, - cache: cache - } do - source = AdaptorsSupervisor.source(sup) - - {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(versions: [])) - - expect( - Lightning.Adaptors.StrategyMock, - :fetch_adaptor, - 1, - fn "@openfn/language-http" -> - {:ok, - adaptor_record( - versions: [version_record("1.0.0"), version_record("2.0.0")] - )} - end - ) - - assert {:ok, versions} = Store.versions(sup, "@openfn/language-http") - assert length(versions) == 2 - - for v <- versions do - assert Map.keys(v) |> Enum.sort() == - [:deprecated, :integrity, :published_at, :size_bytes, :version] - end - - assert {:ok, nil} = - Cachex.get(cache, {:versions, "@openfn/language-http", source}) - end - - test "a fetched record whose name differs from the requested name is refused", - %{sup: sup} do - source = AdaptorsSupervisor.source(sup) - - {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(versions: [])) - - expect( - Lightning.Adaptors.StrategyMock, - :fetch_adaptor, - 1, - fn "@openfn/language-http" -> - {:ok, adaptor_record(name: "@openfn/language-impostor")} - end - ) - - assert {:error, {:name_mismatch, "@openfn/language-impostor"}} = - Store.versions(sup, "@openfn/language-http") - - assert Catalogue.get_adaptor("@openfn/language-impostor", source) == nil - assert Catalogue.list_versions("@openfn/language-http", source) == [] - end - - test "unknown adaptor returns {:error, :not_found} without calling Strategy or minting a row", - %{sup: sup} do - expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> - :unreachable - end) - - source = AdaptorsSupervisor.source(sup) - - assert {:error, :not_found} = Store.versions(sup, "@openfn/never-existed") - assert Catalogue.get_adaptor("@openfn/never-existed", source) == nil - end - end - describe "packages/1" do test "empty DB returns {:ok, []} but does NOT cache the empty result", %{ sup: sup, @@ -540,8 +437,10 @@ defmodule Lightning.Adaptors.StoreTest do end test "Strategy returns bytes that don't match the row's expected sha", %{ - sup: sup + sup: sup, + cache: cache } do + source = AdaptorsSupervisor.source(sup) name = unique_name("corrupt") {:ok, _} = @@ -559,6 +458,41 @@ defmodule Lightning.Adaptors.StoreTest do end) assert {:error, {:icon_sha_mismatch, _}} = Store.icon(sup, name, :square) + + assert {:ok, {:error, {:icon_sha_mismatch, _}}} = + Cachex.get(cache, {:icon_bytes, source, name, :square}) + + assert {:error, {:icon_sha_mismatch, _}} = Store.icon(sup, name, :square) + end + + test "Strategy returns an extension the row doesn't claim", %{ + sup: sup, + cache: cache + } do + source = AdaptorsSupervisor.source(sup) + name = unique_name("wrong-ext") + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + name: name, + icon_square_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, "EXPECTED_BYTES") + ) + ) + + expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 1, fn ^name, + :square -> + {:ok, %{data: "EXPECTED_BYTES", ext: "svg"}} + end) + + assert {:error, {:ext_mismatch, expected: "png", got: "svg"}} = + Store.icon(sup, name, :square) + + assert {:ok, {:error, {:ext_mismatch, _}}} = + Cachex.get(cache, {:icon_bytes, source, name, :square}) + + assert {:error, {:ext_mismatch, _}} = Store.icon(sup, name, :square) end test "disk miss + Strategy success writes to disk and returns path", %{ @@ -606,7 +540,7 @@ defmodule Lightning.Adaptors.StoreTest do ) ) - expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 1, fn _, _ -> + expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 2, fn _, _ -> {:error, :upstream_5xx} end) @@ -614,6 +548,8 @@ defmodule Lightning.Adaptors.StoreTest do assert {:ok, nil} = Cachex.get(cache, {:icon_bytes, source, name, :square}) + + assert {:error, :upstream_5xx} = Store.icon(sup, name, :square) end test "concurrent first-callers coalesce onto one Strategy fetch", %{ From cc3ce0e6044d7b3c7b71187a4c0e729aebae9edd Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Tue, 8 Sep 2026 18:02:40 +0200 Subject: [PATCH 16/37] Share one adaptor_record fixture, refetch a schema-less row while young - One adaptor_record fixture shared, package metas primed from the real projection; packages cache primed after inserting adaptors in picker tests - Schema decode+digest step shared between Local and NPM - A per-adaptor fetch failure now warns instead of dropping silently - An adaptor whose stored row has no schema is refetched; the lazy schema fill dropped from Store.schema/2 in favour of that refetch - Touch instead of upsert when a refetched schema is still nil - Single-clause `with` replaced in the schema decode paths - Icon sha put in the on-disk cache filename; unused version_record test helper deleted - Two scheduler tests that passed by accident fixed - Every superseded icon file swept regardless of extension - Stale strategy-error claim dropped from Adaptors.schema/2's doc - A schema-less row is only refetched while its version is still young --- lib/lightning/adaptors.ex | 5 +- lib/lightning/adaptors/catalogue.ex | 2 + lib/lightning/adaptors/icon_cache.ex | 85 +++--- lib/lightning/adaptors/local.ex | 34 +-- lib/lightning/adaptors/npm/schema.ex | 13 +- lib/lightning/adaptors/scheduler.ex | 35 ++- lib/lightning/adaptors/store.ex | 58 +--- lib/lightning/adaptors/strategy.ex | 16 ++ lib/mix/tasks/lightning.adaptors.snapshot.ex | 8 +- test/lightning/adaptors/catalogue_test.exs | 27 +- test/lightning/adaptors/icon_cache_test.exs | 152 +++++++---- test/lightning/adaptors/node_monitor_test.exs | 36 +-- test/lightning/adaptors/readiness_test.exs | 32 +-- test/lightning/adaptors/scheduler_test.exs | 252 +++++++++++------- test/lightning/adaptors/store_test.exs | 165 ++---------- test/lightning/adaptors_test.exs | 36 --- .../collaboration/session_readiness_test.exs | 32 +-- .../adaptor_icon_controller_test.exs | 10 +- .../live/credential_live_test.exs | 13 +- .../lightning.adaptors.snapshot_test.exs | 32 +++ test/support/adaptor_test_helpers.ex | 87 ++++-- test/test_helper.exs | 8 +- 22 files changed, 538 insertions(+), 600 deletions(-) diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index 1e2422d7846..76ab2883ec6 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -93,9 +93,8 @@ defmodule Lightning.Adaptors do @doc """ Returns the credential schema of the adaptor named `pkg`, as a JSON - binary. An adaptor with no schema yields `"{}"`; an unknown name is - `{:error, :not_found}` and a schema the source could not be reached - for is the strategy's own `{:error, reason}`. + binary. An adaptor with no schema yields `"{}"` and an unknown name + is `{:error, :not_found}`. """ @spec schema(atom(), String.t()) :: {:ok, String.t()} | {:error, term()} def schema(sup \\ Config.default_instance(), pkg), do: Store.schema(sup, pkg) diff --git a/lib/lightning/adaptors/catalogue.ex b/lib/lightning/adaptors/catalogue.ex index 61adffc97e8..6b9df475857 100644 --- a/lib/lightning/adaptors/catalogue.ex +++ b/lib/lightning/adaptors/catalogue.ex @@ -298,6 +298,8 @@ defmodule Lightning.Adaptors.Catalogue do ) |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + # TODO: 👆would it not be easier to do the group by in the query? That elem,elem group by isn't that easy to understand + Enum.map(adaptors, fn adaptor -> Map.put(adaptor, :versions, Map.get(versions_by_name, adaptor.name, [])) end) diff --git a/lib/lightning/adaptors/icon_cache.ex b/lib/lightning/adaptors/icon_cache.ex index 2e640d68131..3d6afa97145 100644 --- a/lib/lightning/adaptors/icon_cache.ex +++ b/lib/lightning/adaptors/icon_cache.ex @@ -6,27 +6,29 @@ defmodule Lightning.Adaptors.IconCache do `Lightning.Adaptors.Config.icon_path/0`, which returns `ADAPTORS_ICONS_PATH` when set and otherwise resolves the `{:tmp, suffix}` default at call time. - Disk layout is **source-partitioned** and **latest-only**: + Disk layout is **source-partitioned** and **content-addressed**: - ///. + ///.. - Source partitioning means flipping `ADAPTORS_STRATEGY` between restarts - cannot accidentally serve `:npm` bytes from a row that's now resolved - via `:local` (or vice versa). Latest-only means a subsequent - `write!/5` for the same key overwrites, and `cached?/5` only trusts a - file whose bytes still hash to the sha on the adaptor row, so a node - that cached an earlier icon refetches instead of serving it forever. + where `sha8` is the first 8 lowercase hex characters of the icon + sha256 on the adaptor row. Source partitioning means flipping + `ADAPTORS_STRATEGY` between restarts cannot accidentally serve `:npm` + bytes from a row that's now resolved via `:local` (or vice versa). + Putting the sha in the filename means `cached?/5` is a plain + existence check, and a node holding an earlier icon simply misses and + refetches instead of serving it forever; `write!/6` removes the + superseded siblings for that shape. Concurrent first-request fetchers are coalesced upstream by Cachex's courier on `{:icon_bytes, source, name, shape}` inside `Lightning.Adaptors.Store.icon/3`, and all in-flight peers receive the - courier's result for free. Bytes that verify are left uncommitted — - this directory is their cache — but bytes that disagree with the row's - sha or extension are committed as an error, so the disagreement is not - re-fetched from the source on every request until the row moves. - The temp-then-rename in `write!/5` is the belt-and- - braces guarantee for the file-write step itself: readers never observe - a half-written file. + courier's result for free. Bytes that verify are left uncommitted, + since this directory is their cache, but bytes that disagree with the + row's sha or extension are committed as an error, so the disagreement + is not re-fetched from the source on every request until the row moves. + The temp-then-rename in `write!/6` is the belt-and-braces guarantee + for the file-write step itself: readers never observe a half-written + file. """ alias Lightning.Adaptors.Config @@ -43,44 +45,47 @@ defmodule Lightning.Adaptors.IconCache do `@openfn/language-foo`); `Path.join/1` preserves the slash so the scope becomes a real subdirectory. """ - @spec path(source(), name(), shape(), ext()) :: Path.t() - def path(source, name, shape, ext) do - Path.join([Config.icon_path(), to_string(source), name, "#{shape}.#{ext}"]) + @spec path(source(), name(), shape(), ext(), binary()) :: Path.t() + def path(source, name, shape, ext, sha256) do + Path.join([ + Config.icon_path(), + to_string(source), + name, + "#{shape}.#{sha8(sha256)}.#{ext}" + ]) end @doc """ - Whether the icon at `path(source, name, shape, ext)` is on disk with - bytes hashing to `sha256`. + Whether the icon for `sha256` is on disk. The sha is part of the + filename, so existence is the whole check. """ @spec cached?(source(), name(), shape(), ext(), binary()) :: boolean() def cached?(source, name, shape, ext, sha256) do - case File.read(path(source, name, shape, ext)) do - {:ok, bytes} -> :crypto.hash(:sha256, bytes) == sha256 - {:error, _} -> false - end + File.exists?(path(source, name, shape, ext, sha256)) end @doc """ - Atomically write `bytes` to `path(source, name, shape, ext)` and - return the sha256 of the supplied bytes as a 32-byte binary. + Atomically write `bytes` for `sha256` and return the path written. The write is staged in a sibling temp file and then renamed into - place, so concurrent readers never observe a half-written file. + place, so concurrent readers never observe a half-written file. Any + superseded file for the same shape, whatever its extension or + pre-sha naming, is removed first, so a rename never lands on a + directory left empty by its own sweep. """ - @spec write!(source(), name(), shape(), ext(), binary()) :: - {:ok, binary()} - def write!(source, name, shape, ext, bytes) when is_binary(bytes) do - final_path = path(source, name, shape, ext) + @spec write!(source(), name(), shape(), ext(), binary(), binary()) :: + Path.t() + def write!(source, name, shape, ext, bytes, sha256) when is_binary(bytes) do + final_path = path(source, name, shape, ext, sha256) dir = Path.dirname(final_path) File.mkdir_p!(dir) - sha = :crypto.hash(:sha256, bytes) - temp_path = Path.join(dir, ".#{Path.basename(final_path)}.#{random_suffix()}.tmp") try do File.write!(temp_path, bytes) + remove_superseded(dir, shape, final_path) File.rename!(temp_path, final_path) rescue e -> @@ -88,7 +93,19 @@ defmodule Lightning.Adaptors.IconCache do reraise e, __STACKTRACE__ end - {:ok, sha} + final_path + end + + defp remove_superseded(dir, shape, final_path) do + dir + |> Path.join("#{shape}.*") + |> Path.wildcard() + |> Enum.reject(&(&1 == final_path)) + |> Enum.each(&File.rm/1) + end + + defp sha8(sha256) when is_binary(sha256) do + sha256 |> Base.encode16(case: :lower) |> binary_part(0, 8) end @spec random_suffix() :: String.t() diff --git a/lib/lightning/adaptors/local.ex b/lib/lightning/adaptors/local.ex index 9440a808ad6..4b8522aab89 100644 --- a/lib/lightning/adaptors/local.ex +++ b/lib/lightning/adaptors/local.ex @@ -211,23 +211,20 @@ defmodule Lightning.Adaptors.Local do defp build_adaptor_record(record) do pkg = record.latest_package_json - base = %{ - name: record.name, - description: pkg["description"], - homepage: pkg["homepage"], - repository: extract_repository(pkg["repository"]), - license: pkg["license"], - latest_version: record.latest_version, - deprecated: false, - versions: Enum.map(record.versions, &build_version_record/1) - } - - with {:ok, schema_data, schema_sha256} <- read_schema(record.latest_path) do + with {:ok, {schema_data, schema_sha256}} <- read_schema(record.latest_path) do {:ok, - Map.merge(base, %{ + %{ + name: record.name, + description: pkg["description"], + homepage: pkg["homepage"], + repository: extract_repository(pkg["repository"]), + license: pkg["license"], + latest_version: record.latest_version, + deprecated: false, + versions: Enum.map(record.versions, &build_version_record/1), schema_data: schema_data, schema_sha256: schema_sha256 - })} + }} end end @@ -245,12 +242,9 @@ defmodule Lightning.Adaptors.Local do end defp read_schema(dir) do - with {:ok, body} <- File.read(Path.join(dir, @schema_filename)), - {:ok, _} <- Jason.decode(body) do - sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) - {:ok, body, sha} - else - {:error, :enoent} -> {:ok, nil, nil} + case File.read(Path.join(dir, @schema_filename)) do + {:ok, body} -> Lightning.Adaptors.Strategy.digest_schema(body) + {:error, :enoent} -> {:ok, {nil, nil}} {:error, reason} -> {:error, {:schema_fetch_failed, reason}} end end diff --git a/lib/lightning/adaptors/npm/schema.ex b/lib/lightning/adaptors/npm/schema.ex index 801479ce6f3..8a8d935fa9c 100644 --- a/lib/lightning/adaptors/npm/schema.ex +++ b/lib/lightning/adaptors/npm/schema.ex @@ -23,20 +23,17 @@ defmodule Lightning.Adaptors.NPM.Schema do Returns `{:ok, {schema_data, schema_sha256}}` on success, `{:ok, {nil, nil}}` on a genuine 404 (schema removed upstream), and - `{:error, reason}` on any other failure — a transient failure must - not be mistaken for genuine absence by callers that persist the + `{:error, reason}` on any other failure, since a transient failure + must not be mistaken for genuine absence by callers that persist the result. """ @spec schema(String.t(), String.t()) :: {:ok, {String.t(), String.t()}} | {:ok, {nil, nil}} | {:error, term()} def schema(name, version) do - with {:ok, body} <- fetch_schema_bytes(name, version), - {:ok, _} <- Jason.decode(body) do - sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) - {:ok, {body, sha}} - else + case fetch_schema_bytes(name, version) do + {:ok, body} -> Lightning.Adaptors.Strategy.digest_schema(body) {:error, {:http_status, 404}} -> {:ok, {nil, nil}} - {:error, reason} -> {:error, reason} + {:error, _} = err -> err end end diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 7eca2852567..82db252e71a 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -10,9 +10,12 @@ defmodule Lightning.Adaptors.Scheduler do catalogue ticks at once. An interval of `0` disables the timer and leaves only on-demand refreshes. - A tick lists the source, fetches only the adaptors whose - `latest_version` changed, fetches icons in parallel, and upserts each - changed adaptor with its icons. `refresh_package/2` refetches one + A tick lists the source, fetches the adaptors whose `latest_version` + changed or whose stored row has no schema and whose version landed + within the last hour (a refetch that still finds no schema counts as + touched; the window covers jsDelivr's mirroring lag, after which a + missing schema is taken as really missing), fetches icons in parallel, + and upserts each changed adaptor with its icons. `refresh_package/2` refetches one adaptor without icons. """ @@ -26,6 +29,7 @@ defmodule Lightning.Adaptors.Scheduler do require Logger @fetch_max_concurrency 8 + @schema_grace_ms :timer.hours(1) @icons_task_timeout :timer.seconds(60) @doc """ @@ -425,14 +429,24 @@ defmodule Lightning.Adaptors.Scheduler do state ) do existing = Map.get(existing_by_name, name) + same_version? = !is_nil(existing) && existing.latest_version == version - if existing && existing.latest_version == version do + if same_version? and + (not is_nil(existing.schema_data) or older_than_grace?(existing)) do Catalogue.touch_checked_at(name, state.source) :touched else case strategy.fetch_adaptor(name) do - {:ok, %{latest_version: version} = record} -> - Logger.debug("Adaptors[#{state.source}]: fetched #{name}@#{version}") + # Refetched only because the stored schema was nil, and upstream + # still has none: nothing to persist, so don't broadcast a change. + {:ok, %{schema_data: nil}} when same_version? -> + Catalogue.touch_checked_at(name, state.source) + :touched + + {:ok, %{latest_version: fetched_version} = record} -> + Logger.debug( + "Adaptors[#{state.source}]: fetched #{name}@#{fetched_version}" + ) {:fetched, keep_stored_schema(record, existing)} @@ -446,6 +460,11 @@ defmodule Lightning.Adaptors.Scheduler do end end + defp older_than_grace?(%{updated_at: updated_at}) do + DateTime.diff(DateTime.utc_now(), updated_at, :millisecond) > + @schema_grace_ms + end + # jsDelivr 404s for a version it has not mirrored yet, which is # indistinguishable from a schema the source really dropped. On the # periodic path we keep what we have; an operator refresh takes upstream @@ -512,7 +531,7 @@ defmodule Lightning.Adaptors.Scheduler do case Map.get(package_icons, shape) do %{data: bytes, ext: ext, sha256: sha} = entry when is_binary(bytes) -> try do - {:ok, ^sha} = IconCache.write!(source, record.name, shape, ext, bytes) + IconCache.write!(source, record.name, shape, ext, bytes, sha) record |> Map.put(:"icon_#{shape}_ext", ext) @@ -615,7 +634,7 @@ defmodule Lightning.Adaptors.Scheduler do ext_key = :"icon_#{shape}_ext" etag_key = :"icon_#{shape}_etag" - {:ok, ^sha} = IconCache.write!(state.source, row.name, shape, ext, bytes) + IconCache.write!(state.source, row.name, shape, ext, bytes, sha) acc |> Map.put(ext_key, ext) diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index 8955342ae3c..889fc8fc747 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -3,12 +3,10 @@ defmodule Lightning.Adaptors.Store do Cached reads over `Lightning.Adaptors.Catalogue`. Every read checks the instance's Cachex first and falls back to the - catalogue table. `schema/2` also fetches from the strategy when the - row has no data yet and persists what it gets; this only fills gaps on - adaptors already in the catalogue, and an unknown name returns - `{:error, :not_found}`. A lazy fill that lands a value - broadcasts the change like a scheduler write; one whose fetch failed - passes the strategy's error through. `icon/3` returns a path on disk, + catalogue table. Reads never write to the catalogue: the + `Lightning.Adaptors.Scheduler` is the only writer, so a row with no + schema means the source has none and `schema/2` answers `"{}"`, while + an unknown name returns `{:error, :not_found}`. `icon/3` returns a path on disk, fetching the bytes from the strategy on the first miss. `catalogue/1` caches the picker payload already rendered, together with the ETag stamp that describes it. @@ -51,7 +49,8 @@ defmodule Lightning.Adaptors.Store do @doc """ Returns the adaptor's credential schema as a JSON binary, not decoded. - An adaptor with no schema yields `"{}"`. + An adaptor with no schema yields `"{}"`; an unknown name + `{:error, :not_found}`. """ @spec schema(sup(), String.t()) :: {:ok, String.t()} | {:error, term()} def schema(sup, name) do @@ -70,7 +69,7 @@ defmodule Lightning.Adaptors.Store do {:commit, {:ok, data}} _ -> - fetch_and_persist(sup, name, source) + {:commit, {:ok, "{}"}} end end, timeout: Config.cache_timeout_ms() @@ -93,7 +92,7 @@ defmodule Lightning.Adaptors.Store do {:ok, ext} <- ext_for_shape(meta, shape), {:ok, expected_sha} <- sha256_for_shape(meta, shape) do if IconCache.cached?(source, name, shape, ext, expected_sha) do - {:ok, IconCache.path(source, name, shape, ext)} + {:ok, IconCache.path(source, name, shape, ext, expected_sha)} else cache |> Cachex.fetch( @@ -113,8 +112,9 @@ defmodule Lightning.Adaptors.Store do {:ok, %{data: bytes, ext: ^ext}} -> case :crypto.hash(:sha256, bytes) do ^expected_sha -> - {:ok, _sha} = IconCache.write!(source, name, shape, ext, bytes) - {:ignore, {:ok, IconCache.path(source, name, shape, ext)}} + {:ignore, + {:ok, + IconCache.write!(source, name, shape, ext, bytes, expected_sha)}} got -> {:commit, @@ -262,40 +262,6 @@ defmodule Lightning.Adaptors.Store do } end - @spec fetch_and_persist(atom(), String.t(), :npm | :local) :: - {:commit, {:ok, term()}} | {:ignore, {:ok, term()} | {:error, term()}} - defp fetch_and_persist(sup, name, source) do - case AdaptorsSupervisor.strategy(sup).fetch_adaptor(name) do - {:ok, %{name: ^name} = record} -> - record = Map.put(record, :source, source) - {:ok, _} = Catalogue.upsert_adaptor(record) - - case record.schema_data do - # The source has nothing; cache that so the next read stays local. - nil -> - {:commit, {:ok, "{}"}} - - # Landed a value: announce it like a scheduler write. The - # Invalidator drops the stale keys and the next read refills them - # from the row, so there is nothing to commit here. - value -> - Phoenix.PubSub.broadcast( - Lightning.PubSub, - AdaptorsSupervisor.source_topic(sup), - {:changed, name, source} - ) - - {:ignore, {:ok, value}} - end - - {:ok, %{name: other}} -> - {:ignore, {:error, {:name_mismatch, other}}} - - {:error, reason} -> - {:ignore, {:error, reason}} - end - end - @spec project_icon_meta(map()) :: icon_meta() defp project_icon_meta(adaptor) do Map.take(adaptor, [ @@ -332,7 +298,7 @@ defmodule Lightning.Adaptors.Store do # # Every fallback returns an inner `{:ok, _} | {:error, _}`, whichever # wrapper it chooses, so the wrapper tuple's second element is itself - # the public value we want to return — including a committed + # the public value we want to return, including a committed # `{:error, _}`, which comes back as `{:ok, {:error, _}}` on a later # hit. Cachex-side `{:error, _}` passes through unchanged. @spec unwrap(tuple()) :: {:ok, term()} | {:error, term()} diff --git a/lib/lightning/adaptors/strategy.ex b/lib/lightning/adaptors/strategy.ex index e6c273874bb..cec1bc9c71b 100644 --- a/lib/lightning/adaptors/strategy.ex +++ b/lib/lightning/adaptors/strategy.ex @@ -149,4 +149,20 @@ defmodule Lightning.Adaptors.Strategy do @callback list_adaptors() :: {:ok, [%{name: String.t(), latest_version: String.t()}]} | {:error, term()} + + @doc """ + Validate a schema body and pair it with its persisted digest. + + Returns `{:ok, {body, sha256_hex}}` when `body` decodes as JSON, with + `sha256_hex` lowercase hex, matching the `adaptors.schema_sha256` + column format. `{:error, reason}` when it doesn't decode. + """ + @spec digest_schema(binary()) :: + {:ok, {binary(), String.t()}} | {:error, term()} + def digest_schema(body) do + with {:ok, _} <- Jason.decode(body) do + sha = :sha256 |> :crypto.hash(body) |> Base.encode16(case: :lower) + {:ok, {body, sha}} + end + end end diff --git a/lib/mix/tasks/lightning.adaptors.snapshot.ex b/lib/mix/tasks/lightning.adaptors.snapshot.ex index 3de2fdf6c42..85fb353e910 100644 --- a/lib/mix/tasks/lightning.adaptors.snapshot.ex +++ b/lib/mix/tasks/lightning.adaptors.snapshot.ex @@ -62,8 +62,12 @@ defmodule Mix.Tasks.Lightning.Adaptors.Snapshot do defp fetch_full_record(%{name: name}) do case NPM.fetch_adaptor(name) do - {:ok, record} -> Map.put(record, :source, :npm) - {:error, _reason} -> nil + {:ok, record} -> + Map.put(record, :source, :npm) + + {:error, reason} -> + Mix.shell().error("Skipping #{name}: #{inspect(reason)}") + nil end end diff --git a/test/lightning/adaptors/catalogue_test.exs b/test/lightning/adaptors/catalogue_test.exs index 1235de20cc7..6a8b3b2672d 100644 --- a/test/lightning/adaptors/catalogue_test.exs +++ b/test/lightning/adaptors/catalogue_test.exs @@ -1,6 +1,8 @@ defmodule Lightning.Adaptors.CatalogueTest do use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers + alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Catalogue.Adaptor alias Lightning.Adaptors.Catalogue.AdaptorVersion @@ -537,31 +539,6 @@ defmodule Lightning.Adaptors.CatalogueTest do end end - defp adaptor_record(overrides \\ []) do - overrides = Map.new(overrides) - - %{ - name: "@openfn/language-http", - source: :npm, - latest_version: "1.0.0", - description: "HTTP adaptor", - homepage: nil, - repository: nil, - license: "LGPL-3.0", - deprecated: false, - schema_data: nil, - schema_sha256: nil, - icon_square_ext: nil, - icon_rectangle_ext: nil, - icon_square_sha256: nil, - icon_rectangle_sha256: nil, - icon_square_etag: nil, - icon_rectangle_etag: nil, - versions: [version_record("1.0.0")] - } - |> Map.merge(overrides) - end - defp version_record(version, overrides \\ []) do %{ version: version, diff --git a/test/lightning/adaptors/icon_cache_test.exs b/test/lightning/adaptors/icon_cache_test.exs index 61631cbc601..8330e8c6522 100644 --- a/test/lightning/adaptors/icon_cache_test.exs +++ b/test/lightning/adaptors/icon_cache_test.exs @@ -30,107 +30,141 @@ defmodule Lightning.Adaptors.IconCacheTest do {:ok, root: root} end - describe "path/4" do - test "joins Config.icon_path with source/name/shape.ext", %{root: root} do - assert IconCache.path(:npm, "salesforce", :square, "png") == - Path.join([root, "npm", "salesforce", "square.png"]) + @sha :crypto.hash(:sha256, "x") + @sha8 @sha |> Base.encode16(case: :lower) |> binary_part(0, 8) + + describe "path/5" do + test "joins Config.icon_path with source/name/shape.sha8.ext", %{root: root} do + assert IconCache.path(:npm, "salesforce", :square, "png", @sha) == + Path.join([root, "npm", "salesforce", "square.#{@sha8}.png"]) end test "handles names containing a slash like @openfn/language-foo", %{ root: root } do - assert IconCache.path(:npm, "@openfn/language-foo", :square, "png") == + assert IconCache.path(:npm, "@openfn/language-foo", :square, "png", @sha) == Path.join([ root, "npm", "@openfn", "language-foo", - "square.png" + "square.#{@sha8}.png" ]) end test "source-partitions paths for the same name", %{root: root} do - npm_path = IconCache.path(:npm, "salesforce", :square, "png") - local_path = IconCache.path(:local, "salesforce", :square, "png") + npm_path = IconCache.path(:npm, "salesforce", :square, "png", @sha) + local_path = IconCache.path(:local, "salesforce", :square, "png", @sha) - assert npm_path == Path.join([root, "npm", "salesforce", "square.png"]) + assert npm_path == + Path.join([root, "npm", "salesforce", "square.#{@sha8}.png"]) assert local_path == - Path.join([root, "local", "salesforce", "square.png"]) + Path.join([root, "local", "salesforce", "square.#{@sha8}.png"]) refute npm_path == local_path end test "is pure — nothing is created on disk", %{root: root} do - _ = IconCache.path(:npm, "never-written", :rectangle, "svg") + _ = IconCache.path(:npm, "never-written", :rectangle, "svg", @sha) assert File.ls!(root) == [] end end describe "cached?/5" do - @sha :crypto.hash(:sha256, "x") - test "returns false when the file does not exist" do refute IconCache.cached?(:npm, "definitely-missing", :square, "png", @sha) end - test "returns true after write!/5 places bytes with that sha" do - {:ok, sha} = IconCache.write!(:npm, "cached-pkg", :square, "png", "x") - assert sha == @sha + test "returns true after write!/6 places bytes for that sha" do + write("cached-pkg", "x") assert IconCache.cached?(:npm, "cached-pkg", :square, "png", @sha) end - test "returns false when the file on disk has other bytes" do - {:ok, _} = IconCache.write!(:npm, "stale-pkg", :square, "png", "old") + test "returns false when only another sha is on disk" do + write("stale-pkg", "old") refute IconCache.cached?(:npm, "stale-pkg", :square, "png", @sha) end test "stays source-partitioned: a write to :npm doesn't satisfy :local" do - {:ok, _} = IconCache.write!(:npm, "split-pkg", :square, "png", "x") + write("split-pkg", "x") assert IconCache.cached?(:npm, "split-pkg", :square, "png", @sha) refute IconCache.cached?(:local, "split-pkg", :square, "png", @sha) end end - describe "write!/5" do - test "writes bytes and a round-trip read returns them" do + describe "write!/6" do + test "writes bytes and returns the path they can be read back from" do bytes = :crypto.strong_rand_bytes(2_048) - {:ok, _sha} = - IconCache.write!(:npm, "round-trip", :square, "png", bytes) + path = write("round-trip", bytes) + + assert path == + IconCache.path( + :npm, + "round-trip", + :square, + "png", + :crypto.hash(:sha256, bytes) + ) - assert File.read!(IconCache.path(:npm, "round-trip", :square, "png")) == - bytes + assert File.read!(path) == bytes end - test "returns the sha256 of the supplied bytes as a 32-byte binary" do - bytes = "hello, icon" + test "removes the superseded file for the same shape and extension" do + old_path = write("rotated", "first") + new_path = write("rotated", "second") - {:ok, sha} = IconCache.write!(:npm, "sha-test", :square, "png", bytes) + refute old_path == new_path + assert File.read!(new_path) == "second" + refute File.exists?(old_path) + end - assert sha == :crypto.hash(:sha256, bytes) - assert byte_size(sha) == 32 + test "removes the superseded file even when the extension changed" do + old_path = write("re-ext", "first") + + new_path = + IconCache.write!( + :npm, + "re-ext", + :square, + "svg", + "second", + :crypto.hash(:sha256, "second") + ) + + assert String.ends_with?(new_path, ".svg") + assert File.exists?(new_path) + refute File.exists?(old_path) end - test "is latest-only: a subsequent write for the same key overwrites" do - {:ok, _} = IconCache.write!(:npm, "overwrite", :square, "png", "first") - {:ok, _} = IconCache.write!(:npm, "overwrite", :square, "png", "second") + test "removes a pre-sha legacy file for the same shape" do + new_path = write("legacy", "bytes") + legacy = Path.join(Path.dirname(new_path), "square.png") + File.write!(legacy, "old") + + write("legacy", "bytes") - assert File.read!(IconCache.path(:npm, "overwrite", :square, "png")) == - "second" + refute File.exists?(legacy) + assert File.exists?(new_path) + end + + test "leaves the other shape alone when sweeping" do + square = write("two-shapes", "sq") + rectangle = write("two-shapes", "rect", :rectangle) + + assert File.exists?(square) + assert File.exists?(rectangle) end test "creates intermediate directories for scoped names" do - {:ok, _} = - IconCache.write!(:npm, "@openfn/language-http", :square, "png", "abc") + path = write("@openfn/language-http", "abc") - assert File.read!( - IconCache.path(:npm, "@openfn/language-http", :square, "png") - ) == "abc" + assert File.read!(path) == "abc" end test "is atomic: concurrent writers produce no half-written file and no leftover temps", @@ -140,26 +174,38 @@ defmodule Lightning.Adaptors.IconCacheTest do :crypto.strong_rand_bytes(16_384) <> <> end - payloads - |> Enum.map(fn bytes -> - Task.async(fn -> - IconCache.write!(:npm, "concurrent", :square, "png", bytes) + paths = + payloads + |> Enum.map(fn bytes -> + Task.async(fn -> write("concurrent", bytes) end) end) - end) - |> Task.await_many(10_000) + |> Task.await_many(10_000) - final_path = IconCache.path(:npm, "concurrent", :square, "png") - final = File.read!(final_path) + dir = Path.dirname(hd(paths)) + on_disk = File.ls!(dir) - assert final in payloads, - "final file does not match any written payload — write was not atomic" + refute on_disk == [], "the sweep left nothing behind" - dir = Path.dirname(final_path) + for file <- on_disk do + refute String.ends_with?(file, ".tmp"), + "leftover temp files in #{dir}: #{inspect(on_disk)}" - assert dir |> File.ls!() |> Enum.reject(&(&1 == "square.png")) == [], - "leftover temp files in #{dir}: #{inspect(File.ls!(dir))}" + assert File.read!(Path.join(dir, file)) in payloads, + "#{file} matches no written payload — write was not atomic" + end _ = root end end + + defp write(name, bytes, shape \\ :square) do + IconCache.write!( + :npm, + name, + shape, + "png", + bytes, + :crypto.hash(:sha256, bytes) + ) + end end diff --git a/test/lightning/adaptors/node_monitor_test.exs b/test/lightning/adaptors/node_monitor_test.exs index 1e8fa6b47c3..14ab23ed412 100644 --- a/test/lightning/adaptors/node_monitor_test.exs +++ b/test/lightning/adaptors/node_monitor_test.exs @@ -1,6 +1,8 @@ defmodule Lightning.Adaptors.NodeMonitorTest do use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers + import Mox alias Lightning.Adaptors.Catalogue @@ -123,38 +125,4 @@ defmodule Lightning.Adaptors.NodeMonitorTest do Cachex.get(cache, {:packages, source}) end end - - defp adaptor_record(overrides \\ []) do - overrides = Map.new(overrides) - - %{ - name: "@openfn/language-http", - source: :npm, - latest_version: "1.0.0", - description: "HTTP adaptor", - homepage: nil, - repository: nil, - license: "LGPL-3.0", - deprecated: false, - schema_data: nil, - schema_sha256: nil, - icon_square_ext: nil, - icon_rectangle_ext: nil, - icon_square_sha256: nil, - icon_rectangle_sha256: nil, - versions: [ - %{ - version: "1.0.0", - integrity: "sha512-1.0.0", - tarball_url: "https://example.com/x/-/x-1.0.0.tgz", - size_bytes: 1024, - dependencies: %{}, - peer_dependencies: %{}, - published_at: nil, - deprecated: false - } - ] - } - |> Map.merge(overrides) - end end diff --git a/test/lightning/adaptors/readiness_test.exs b/test/lightning/adaptors/readiness_test.exs index 45a58d46cde..ec4a80024a7 100644 --- a/test/lightning/adaptors/readiness_test.exs +++ b/test/lightning/adaptors/readiness_test.exs @@ -8,6 +8,8 @@ defmodule Lightning.Adaptors.ReadinessTest do use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers + import Eventually import Mox @@ -33,36 +35,6 @@ defmodule Lightning.Adaptors.ReadinessTest do {:ok, sup: sup} end - defp adaptor_record(overrides \\ []) do - overrides = Map.new(overrides) - - %{ - name: "@openfn/language-http", - source: :npm, - latest_version: "1.0.0", - description: nil, - homepage: nil, - repository: nil, - license: nil, - deprecated: false, - schema_data: nil, - schema_sha256: nil, - versions: [ - %{ - version: "1.0.0", - integrity: "sha512-abc", - tarball_url: "https://example.com/x-1.0.0.tgz", - size_bytes: 1024, - dependencies: %{}, - peer_dependencies: %{}, - published_at: nil, - deprecated: false - } - ] - } - |> Map.merge(overrides) - end - # Tasks the Scheduler spawns inherit its `$callers`, so allowing the # Scheduler covers them. defp start_scheduler(sup) do diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index 18f984c2065..9e12fd5fffc 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -4,6 +4,8 @@ defmodule Lightning.Adaptors.SchedulerTest do # when tests run serially. use Lightning.DataCase, async: false + import Lightning.AdaptorTestHelpers + import Eventually import Mox @@ -93,36 +95,6 @@ defmodule Lightning.Adaptors.SchedulerTest do end end - defp adaptor_record(overrides \\ []) do - overrides = Map.new(overrides) - - %{ - name: "@openfn/language-http", - source: :npm, - latest_version: "1.0.0", - description: "HTTP adaptor", - homepage: nil, - repository: nil, - license: "LGPL-3.0", - deprecated: false, - schema_data: nil, - schema_sha256: nil, - versions: [ - %{ - version: "1.0.0", - integrity: "sha512-abc", - tarball_url: "https://example.com/x-1.0.0.tgz", - size_bytes: 1024, - dependencies: %{}, - peer_dependencies: %{}, - published_at: nil, - deprecated: false - } - ] - } - |> Map.merge(overrides) - end - describe "start_link/1" do test "raises when :name is missing", %{sup: sup} do assert_raise KeyError, ~r/key :name not found/, fn -> @@ -251,7 +223,14 @@ defmodule Lightning.Adaptors.SchedulerTest do source = AdaptorsSupervisor.source(sup) source_topic = AdaptorsSupervisor.source_topic(sup) - {:ok, existing} = Catalogue.upsert_adaptor(adaptor_record()) + {:ok, existing} = + Catalogue.upsert_adaptor( + adaptor_record( + schema_data: ~s({"type":"object"}), + schema_sha256: "sha-1" + ) + ) + checked_at_before = existing.checked_at expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> @@ -274,13 +253,138 @@ defmodule Lightning.Adaptors.SchedulerTest do assert_receive :list_adaptors_called, 2000 # Allow the spawned task to complete before asserting no broadcast. - refute_receive {:changed, _, _}, 200 + refute_receive {:changed, _, _} row = Catalogue.get_adaptor("@openfn/language-http", source) assert DateTime.compare(row.checked_at, checked_at_before) == :gt assert row.latest_version == "1.0.0" end + test "matching version with no stored schema: refetch and persist it", %{ + sup: sup + } do + test_pid = self() + source = AdaptorsSupervisor.source(sup) + + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record(schema_data: nil, schema_sha256: nil) + ) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + 1, + fn "@openfn/language-http" -> + send(test_pid, :fetch_adaptor_called) + + {:ok, + adaptor_record( + schema_data: ~s({"type":"object"}), + schema_sha256: "sha-1" + )} + end + ) + + start_scheduler(sup) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + Scheduler.refresh_now(sched_name) + + assert_receive :fetch_adaptor_called, 2000 + assert {:ok, %{fetched: 1}} = Scheduler.await_refresh(sched_name, 5_000) + + row = Catalogue.get_adaptor("@openfn/language-http", source) + assert row.latest_version == "1.0.0" + assert row.schema_data == ~s({"type":"object"}) + assert row.schema_sha256 == "sha-1" + end + + test "matching version, still no schema upstream: touch only", %{sup: sup} do + test_pid = self() + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + + {:ok, existing} = + Catalogue.upsert_adaptor( + adaptor_record(schema_data: nil, schema_sha256: nil) + ) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn + "@openfn/language-http" -> + send(test_pid, :fetch_adaptor_called) + {:ok, adaptor_record(schema_data: nil, schema_sha256: nil)} + end) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + Scheduler.refresh_now(sched_name) + + assert_receive :fetch_adaptor_called, 2000 + + assert {:ok, %{fetched: 0, changed: 0}} = + Scheduler.await_refresh(sched_name, 5_000) + + refute_receive {:changed, _, _} + + row = Catalogue.get_adaptor("@openfn/language-http", source) + assert DateTime.compare(row.checked_at, existing.checked_at) == :gt + assert row.updated_at == existing.updated_at + end + + test "matching version, no stored schema, row older than the grace window: touch only", + %{sup: sup} do + source = AdaptorsSupervisor.source(sup) + source_topic = AdaptorsSupervisor.source_topic(sup) + + {:ok, existing} = + Catalogue.upsert_adaptor( + adaptor_record(schema_data: nil, schema_sha256: nil) + ) + + two_hours_ago = DateTime.add(DateTime.utc_now(), -2, :hour) + + {1, _} = + Lightning.Repo.update_all( + from(a in Lightning.Adaptors.Catalogue.Adaptor, + where: a.id == ^existing.id + ), + set: [updated_at: two_hours_ago] + ) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable + end) + + :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) + start_scheduler(sup) + + sched_name = AdaptorsSupervisor.global_scheduler_name(sup) + + assert {:ok, %{fetched: 0, changed: 0, errors: 0}} = + Scheduler.await_refresh(sched_name, 5_000) + + refute_receive {:changed, _, _} + + row = Catalogue.get_adaptor("@openfn/language-http", source) + assert row.schema_data == nil + assert DateTime.compare(row.checked_at, existing.checked_at) == :gt + end + test "changed adaptor: upsert and broadcast per changed name", %{sup: sup} do test_pid = self() source = AdaptorsSupervisor.source(sup) @@ -407,7 +511,7 @@ defmodule Lightning.Adaptors.SchedulerTest do start_scheduler(sup) assert_receive :first_fetch, 2000 - refute_receive {:changed, _, _}, 200 + refute_receive {:changed, _, _} assert Catalogue.get_adaptor(name, source) == nil expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> @@ -437,7 +541,7 @@ defmodule Lightning.Adaptors.SchedulerTest do start_scheduler(sup) assert_receive :list_adaptors_called, 2000 - refute_receive {:changed, _, _}, 200 + refute_receive {:changed, _, _} end test "fetch_adaptor error: logs warning, continues to next adaptor", %{ @@ -587,7 +691,13 @@ defmodule Lightning.Adaptors.SchedulerTest do describe "await_refresh/2 result" do test "carries the cycle's counts on success, with per-adaptor failures as errors", %{sup: sup} do - {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record( + schema_data: ~s({"type":"object"}), + schema_sha256: "sha-1" + ) + ) expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> {:ok, @@ -675,12 +785,12 @@ defmodule Lightning.Adaptors.SchedulerTest do source, "@openfn/language-http", :square, - "png" + "png", + sha ) assert File.exists?(icon_path) assert File.read!(icon_path) == bytes - File.rm!(icon_path) end test "fetch_icons error: records still persist without icons", %{sup: sup} do @@ -721,6 +831,8 @@ defmodule Lightning.Adaptors.SchedulerTest do {:ok, _} = Catalogue.upsert_adaptor( adaptor_record( + schema_data: ~s({"type":"object"}), + schema_sha256: "sha-1", icon_square_ext: "png", icon_square_sha256: old_sha, # Both icon shapes already exist on the row; only the square @@ -733,7 +845,7 @@ defmodule Lightning.Adaptors.SchedulerTest do new_bytes = "NEW_ICON_BYTES" new_sha = :crypto.hash(:sha256, new_bytes) - # Upstream reports the same version, so the diff path marks this + # Same version and a stored schema, so the diff path marks this # adaptor :touched instead of re-fetching it — only the icon changed. expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} @@ -757,7 +869,7 @@ defmodule Lightning.Adaptors.SchedulerTest do start_scheduler(sup) sched_name = AdaptorsSupervisor.global_scheduler_name(sup) - :ok = Scheduler.refresh_now(sched_name) + assert {:ok, %{errors: 0}} = Scheduler.await_refresh(sched_name, 5_000) assert_receive {:changed, "@openfn/language-http", ^source}, 2000 @@ -765,26 +877,22 @@ defmodule Lightning.Adaptors.SchedulerTest do assert row.latest_version == "1.0.0" assert row.icon_square_ext == "png" assert row.icon_square_sha256 == new_sha - - icon_path = - Lightning.Adaptors.IconCache.path( - source, - "@openfn/language-http", - :square, - "png" - ) - - File.rm(icon_path) end test "self-heals iconless rows on the periodic tick", %{sup: sup} do source = AdaptorsSupervisor.source(sup) - # Pre-seed a row that already matches the listed latest_version - # (so the diff path will :touch instead of :fetch). Without - # self-heal this row would stay iconless forever. + # Pre-seed a row that already matches the listed latest_version and + # has a schema (so the diff path will :touch instead of :fetch). + # Without self-heal this row would stay iconless forever. {:ok, _} = - Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/language-stale")) + Catalogue.upsert_adaptor( + adaptor_record( + name: "@openfn/language-stale", + schema_data: ~s({"type":"object"}), + schema_sha256: "sha-1" + ) + ) bytes = "STALE_ICON" sha = :crypto.hash(:sha256, bytes) @@ -813,23 +921,13 @@ defmodule Lightning.Adaptors.SchedulerTest do # The pre-seeded row pushes max_checked_at to "now", so init # delay = full interval — drive the tick explicitly. sched_name = AdaptorsSupervisor.global_scheduler_name(sup) - :ok = Scheduler.refresh_now(sched_name) + assert {:ok, %{errors: 0}} = Scheduler.await_refresh(sched_name, 5_000) assert_receive {:changed, "@openfn/language-stale", ^source}, 2000 row = Catalogue.get_adaptor("@openfn/language-stale", source) assert row.icon_square_ext == "png" assert row.icon_square_sha256 == sha - - icon_path = - Lightning.Adaptors.IconCache.path( - source, - "@openfn/language-stale", - :square, - "png" - ) - - File.rm(icon_path) end end @@ -1061,11 +1159,6 @@ defmodule Lightning.Adaptors.SchedulerTest do current = Catalogue.get_adaptor("@openfn/language-current", source) assert current.icon_square_sha256 == new_sha - - for name <- ["@openfn/language-empty", "@openfn/language-current"] do - Lightning.Adaptors.IconCache.path(source, name, :square, "png") - |> File.rm() - end end test "leaves rows whose shape sha256 already matches unchanged, passing prior etag", @@ -1146,14 +1239,6 @@ defmodule Lightning.Adaptors.SchedulerTest do row = Catalogue.get_adaptor("@openfn/language-rotated", source) assert row.icon_square_sha256 == new_sha assert row.icon_square_etag == new_etag - - Lightning.Adaptors.IconCache.path( - source, - "@openfn/language-rotated", - :square, - "png" - ) - |> File.rm() end test "preserves existing etag when fetched entry's etag is nil or missing", @@ -1219,11 +1304,6 @@ defmodule Lightning.Adaptors.SchedulerTest do row_b = Catalogue.get_adaptor("@openfn/language-no-etag-key", source) assert row_b.icon_square_sha256 == new_sha_b assert row_b.icon_square_etag == prior_etag - - for name <- ["@openfn/language-nil-etag", "@openfn/language-no-etag-key"] do - Lightning.Adaptors.IconCache.path(source, name, :square, "png") - |> File.rm() - end end test "mixed 304 and 200: unchanged row preserves its etag verbatim", @@ -1293,14 +1373,6 @@ defmodule Lightning.Adaptors.SchedulerTest do assert current_row.icon_square_sha256 == current_sha assert current_row.icon_square_etag == current_etag - - Lightning.Adaptors.IconCache.path( - source, - "@openfn/language-stale-etag", - :square, - "png" - ) - |> File.rm() end test "surfaces a strategy fetch error as {:error, reason}", %{sup: sup} do diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index 9b4ad3a3b4f..f6bb41df205 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -1,6 +1,8 @@ defmodule Lightning.Adaptors.StoreTest do use Lightning.DataCase, async: true + import Lightning.AdaptorTestHelpers + import Mox alias Lightning.Adaptors.Catalogue @@ -65,50 +67,15 @@ defmodule Lightning.Adaptors.StoreTest do Store.schema(sup, "@openfn/language-http") end - test "known adaptor with missing schema calls Strategy once, upserts to DB, broadcasts the change", - %{ - sup: sup, - cache: cache - } do - source = AdaptorsSupervisor.source(sup) - - Phoenix.PubSub.subscribe( - Lightning.PubSub, - AdaptorsSupervisor.source_topic(sup) - ) - - {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) - - expect( - Lightning.Adaptors.StrategyMock, - :fetch_adaptor, - 1, - fn "@openfn/language-http" -> - {:ok, adaptor_record(schema_data: ~s({"type":"object"}))} - end - ) - - assert {:ok, ~s({"type":"object"})} = - Store.schema(sup, "@openfn/language-http") - - assert_receive {:changed, "@openfn/language-http", ^source} - - assert %{schema_data: ~s({"type":"object"})} = - Catalogue.get_adaptor("@openfn/language-http", source) - - assert {:ok, nil} = - Cachex.get(cache, {:schema, "@openfn/language-http", source}) - end - - test "an adaptor the source confirms has no schema caches an empty one", + test "a row with no schema answers an empty one without calling Strategy", %{sup: sup, cache: cache} do source = AdaptorsSupervisor.source(sup) name = "@openfn/language-http" {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) - expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> - {:ok, adaptor_record(schema_data: nil)} + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> + :unreachable end) assert {:ok, "{}"} = Store.schema(sup, name) @@ -131,72 +98,7 @@ defmodule Lightning.Adaptors.StoreTest do Cachex.get(cache, {:schema, "@openfn/never-existed", source}) end - test "three concurrent calls coalesce to one Strategy call", %{sup: sup} do - name = "@openfn/language-http" - test_pid = self() - - {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) - - expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn ^name -> - # Brief sleep so the other two tasks queue up in Cachex's courier. - Process.sleep(30) - {:ok, adaptor_record(schema_data: ~s({"type":"object"}))} - end) - - tasks = - Enum.map(1..3, fn _ -> - Task.async(fn -> - receive do - :go -> Store.schema(sup, name) - end - end) - end) - - # Allow all tasks to use the test process's Mox expectations before releasing them. - Enum.each( - tasks, - &Mox.allow(Lightning.Adaptors.StrategyMock, test_pid, &1.pid) - ) - - Enum.each(tasks, &send(&1.pid, :go)) - - results = Task.await_many(tasks, 5_000) - assert Enum.all?(results, &match?({:ok, ~s({"type":"object"})}, &1)) - end - - test "Strategy error returns {:error, _} and is not cached — next call retries", - %{ - sup: sup, - cache: cache - } do - source = AdaptorsSupervisor.source(sup) - - {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(schema_data: nil)) - - expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 1, fn _ -> - {:error, :upstream_error} - end) - - assert {:error, :upstream_error} = - Store.schema(sup, "@openfn/language-http") - - assert {:ok, nil} = - Cachex.get(cache, {:schema, "@openfn/language-http", source}) - - expect( - Lightning.Adaptors.StrategyMock, - :fetch_adaptor, - 1, - fn "@openfn/language-http" -> - {:ok, adaptor_record(schema_data: ~s({"type":"object"}))} - end - ) - - assert {:ok, ~s({"type":"object"})} = - Store.schema(sup, "@openfn/language-http") - end - - test "preserves JSON property order through the persistence round-trip", + test "preserves JSON property order from the stored row", %{sup: sup} do expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> :unreachable @@ -386,14 +288,14 @@ defmodule Lightning.Adaptors.StoreTest do ) ) - {:ok, _} = - Lightning.Adaptors.IconCache.write!( - source, - name, - :square, - "png", - "PRE_WARMED" - ) + Lightning.Adaptors.IconCache.write!( + source, + name, + :square, + "png", + "PRE_WARMED", + :crypto.hash(:sha256, "PRE_WARMED") + ) expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 0, fn _, _ -> :unreachable @@ -418,14 +320,14 @@ defmodule Lightning.Adaptors.StoreTest do ) ) - {:ok, _} = - Lightning.Adaptors.IconCache.write!( - source, - name, - :square, - "png", - "STALE_BYTES" - ) + Lightning.Adaptors.IconCache.write!( + source, + name, + :square, + "png", + "STALE_BYTES", + :crypto.hash(:sha256, "STALE_BYTES") + ) expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 1, fn ^name, :square -> @@ -744,29 +646,6 @@ defmodule Lightning.Adaptors.StoreTest do end end - defp adaptor_record(overrides \\ []) do - overrides = Map.new(overrides) - - %{ - name: "@openfn/language-http", - source: :npm, - latest_version: "1.0.0", - description: "HTTP adaptor", - homepage: nil, - repository: nil, - license: "LGPL-3.0", - deprecated: false, - schema_data: nil, - schema_sha256: nil, - icon_square_ext: nil, - icon_rectangle_ext: nil, - icon_square_sha256: nil, - icon_rectangle_sha256: nil, - versions: [version_record("1.0.0")] - } - |> Map.merge(overrides) - end - defp version_record(version) do %{ version: version, diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index a1d98e308ef..aa00539c20c 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -14,42 +14,6 @@ defmodule Lightning.AdaptorsTest do setup :verify_on_exit! setup :isolated_adaptors - defp adaptor_record(overrides \\ []) do - overrides = Map.new(overrides) - - %{ - name: "@openfn/language-http", - source: :npm, - latest_version: "1.0.0", - description: "HTTP adaptor", - homepage: nil, - repository: nil, - license: "LGPL-3.0", - deprecated: false, - schema_data: nil, - schema_sha256: nil, - icon_square_ext: nil, - icon_rectangle_ext: nil, - icon_square_sha256: nil, - icon_rectangle_sha256: nil, - versions: [version_record("1.0.0")] - } - |> Map.merge(overrides) - end - - defp version_record(version) do - %{ - version: version, - integrity: "sha512-#{version}", - tarball_url: "https://example.com/x/-/x-#{version}.tgz", - size_bytes: 1024, - dependencies: %{}, - peer_dependencies: %{}, - published_at: nil, - deprecated: false - } - end - defp start_scheduler(sup) do original_env = Application.get_env(:lightning, Lightning.Adaptors, []) diff --git a/test/lightning/collaboration/session_readiness_test.exs b/test/lightning/collaboration/session_readiness_test.exs index fcf256425b3..423cc94b543 100644 --- a/test/lightning/collaboration/session_readiness_test.exs +++ b/test/lightning/collaboration/session_readiness_test.exs @@ -9,6 +9,8 @@ defmodule Lightning.Collaboration.SessionReadinessTest do # Scheduler. use Lightning.DataCase, async: false + import Lightning.AdaptorTestHelpers + import Lightning.Factories import Lightning.CollaborationHelpers import Mox @@ -62,36 +64,6 @@ defmodule Lightning.Collaboration.SessionReadinessTest do } end - defp adaptor_record(overrides \\ []) do - overrides = Map.new(overrides) - - %{ - name: "@openfn/language-http", - source: :npm, - latest_version: "1.0.0", - description: nil, - homepage: nil, - repository: nil, - license: nil, - deprecated: false, - schema_data: nil, - schema_sha256: nil, - versions: [ - %{ - version: "1.0.0", - integrity: "sha512-abc", - tarball_url: "https://example.com/x-1.0.0.tgz", - size_bytes: 1024, - dependencies: %{}, - peer_dependencies: %{}, - published_at: nil, - deprecated: false - } - ] - } - |> Map.merge(overrides) - end - test "does not stall a concurrent call into the same session while waiting, and resolves via GenServer.reply on success", %{session: session, user: user} do expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> diff --git a/test/lightning_web/controllers/adaptor_icon_controller_test.exs b/test/lightning_web/controllers/adaptor_icon_controller_test.exs index 78af0a5a6d4..f1cdcbc9808 100644 --- a/test/lightning_web/controllers/adaptor_icon_controller_test.exs +++ b/test/lightning_web/controllers/adaptor_icon_controller_test.exs @@ -45,7 +45,15 @@ defmodule LightningWeb.AdaptorIconControllerTest do end defp write_icon(name, shape, ext, bytes) do - {:ok, _sha} = IconCache.write!(source(), name, shape, ext, bytes) + IconCache.write!( + source(), + name, + shape, + ext, + bytes, + :crypto.hash(:sha256, bytes) + ) + :ok end diff --git a/test/lightning_web/live/credential_live_test.exs b/test/lightning_web/live/credential_live_test.exs index 84a6eaf7923..46367d9245c 100644 --- a/test/lightning_web/live/credential_live_test.exs +++ b/test/lightning_web/live/credential_live_test.exs @@ -11,8 +11,6 @@ defmodule LightningWeb.CredentialLiveTest do import Swoosh.TestAssertions alias Lightning.Accounts.User - alias Lightning.Adaptors.Config - alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor alias Lightning.Credentials alias Lightning.Credentials.Credential @@ -2787,12 +2785,7 @@ defmodule LightningWeb.CredentialLiveTest do schema_data: ~s({"type":"object"}) ) - # `seed_all_credential_schemas/0` primes the packages cache by hand - # (bypassing `Catalogue.list_package_metas/1`), so drop it here to - # force a fresh DB-backed read that can see the row above. - cache = AdaptorsSupervisor.cache_name(Config.default_instance()) - source = AdaptorsSupervisor.source(Config.default_instance()) - Cachex.del(cache, {:packages, source}) + Lightning.AdaptorTestHelpers.prime_packages_cache() {:ok, view, _html} = live(conn, ~p"/credentials") @@ -2813,9 +2806,7 @@ defmodule LightningWeb.CredentialLiveTest do test "omits an adaptor with no configuration schema", %{conn: conn} do insert(:adaptor, name: "@openfn/language-no-schema", schema_data: nil) - cache = AdaptorsSupervisor.cache_name(Config.default_instance()) - source = AdaptorsSupervisor.source(Config.default_instance()) - Cachex.del(cache, {:packages, source}) + Lightning.AdaptorTestHelpers.prime_packages_cache() {:ok, view, _html} = live(conn, ~p"/credentials") diff --git a/test/mix/tasks/lightning.adaptors.snapshot_test.exs b/test/mix/tasks/lightning.adaptors.snapshot_test.exs index 15c4a6f5d28..31593294d1a 100644 --- a/test/mix/tasks/lightning.adaptors.snapshot_test.exs +++ b/test/mix/tasks/lightning.adaptors.snapshot_test.exs @@ -111,6 +111,38 @@ defmodule Mix.Tasks.Lightning.Adaptors.SnapshotTest do versions: [%{version: @latest_version}] } = record end + + test "warns and omits an adaptor whose fetch fails", %{ + tmp_dir: tmp_dir, + registry: registry + } do + Bypass.expect(registry, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, %{@package => "write"}) + end) + + Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> + json_resp(conn, 200, %{ + "objects" => [ + %{"package" => %{"name" => @package, "version" => @latest_version}} + ] + }) + end) + + Bypass.expect(registry, "GET", "/" <> @package, fn conn -> + Plug.Conn.resp(conn, 500, "boom") + end) + + file_path = Path.join([tmp_dir, "cache.json"]) + + output = + capture_io(:stderr, fn -> + Snapshot.run(["--path", file_path]) + end) + + assert output =~ @package + + assert [] = file_path |> File.read!() |> Jason.decode!() + end end defp build_packument do diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex index 44f36eceda9..65aa1dcb704 100644 --- a/test/support/adaptor_test_helpers.ex +++ b/test/support/adaptor_test_helpers.ex @@ -58,6 +58,45 @@ defmodule Lightning.AdaptorTestHelpers do assert_eventually(is_pid(:global.whereis_name(gname)), 2000) end + @doc """ + A `Lightning.Adaptors.Catalogue.upsert_adaptor/1` record for + `@openfn/language-http`, with `overrides` merged in. + """ + @spec adaptor_record(keyword() | map()) :: map() + def adaptor_record(overrides \\ []) do + %{ + name: "@openfn/language-http", + source: :npm, + latest_version: "1.0.0", + description: "HTTP adaptor", + homepage: nil, + repository: nil, + license: "LGPL-3.0", + deprecated: false, + schema_data: nil, + schema_sha256: nil, + icon_square_ext: nil, + icon_rectangle_ext: nil, + icon_square_sha256: nil, + icon_rectangle_sha256: nil, + icon_square_etag: nil, + icon_rectangle_etag: nil, + versions: [ + %{ + version: "1.0.0", + integrity: "sha512-1.0.0", + tarball_url: "https://example.com/x/-/x-1.0.0.tgz", + size_bytes: 1024, + dependencies: %{}, + peer_dependencies: %{}, + published_at: nil, + deprecated: false + } + ] + } + |> Map.merge(Map.new(overrides)) + end + @doc """ Seeds a throwaway adaptor row so the catalogue counts as loaded and saves do not wait on the production Scheduler. @@ -156,7 +195,11 @@ defmodule Lightning.AdaptorTestHelpers do insert(:adaptor, name: "@openfn/language-#{short_name}", source: :npm, - schema_data: schema_body + schema_data: schema_body, + icon_square_ext: "png", + icon_rectangle_ext: "png", + icon_square_sha256: :crypto.hash(:sha256, short_name <> "-square"), + icon_rectangle_sha256: :crypto.hash(:sha256, short_name <> "-rectangle") ) # Cachex fills run in its Courier process, which cannot see the sandbox @@ -175,29 +218,31 @@ defmodule Lightning.AdaptorTestHelpers do def seed_all_credential_schemas do ensure_isolated!() - metas = - Path.wildcard("test/fixtures/schemas/*.json") - |> Enum.reject(fn path -> File.stat!(path).size == 0 end) - |> Enum.map(fn path -> - short_name = path |> Path.basename(".json") - row = seed_credential_schema(short_name) + Path.wildcard("test/fixtures/schemas/*.json") + |> Enum.reject(fn path -> File.stat!(path).size == 0 end) + |> Enum.each(fn path -> + path |> Path.basename(".json") |> seed_credential_schema() + end) - %{ - name: row.name, - latest_version: row.latest_version, - description: nil, - deprecated: false, - icon_square_ext: "png", - icon_rectangle_ext: "png", - icon_square_sha256: :crypto.hash(:sha256, short_name <> "-square"), - icon_rectangle_sha256: - :crypto.hash(:sha256, short_name <> "-rectangle"), - has_schema: true - } - end) + prime_packages_cache() + + :ok + end + + @doc """ + Fills the `{:packages, source}` cache entry from whatever adaptor rows + currently exist. Call this after inserting an adaptor row so the picker + sees it, since `Config.default_instance/0`'s cache fills otherwise run in + a Courier process that can't see the SQL sandbox connection. + """ + @spec prime_packages_cache() :: :ok + def prime_packages_cache do + ensure_isolated!() - cache = AdaptorsSupervisor.cache_name(Config.default_instance()) source = AdaptorsSupervisor.source(Config.default_instance()) + metas = Lightning.Adaptors.Catalogue.list_package_metas(source) + + cache = AdaptorsSupervisor.cache_name(Config.default_instance()) Cachex.put(cache, {:packages, source}, {:ok, metas}) :ok diff --git a/test/test_helper.exs b/test/test_helper.exs index ce4e9365f65..53b11b474d2 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -78,11 +78,9 @@ Application.put_env(:lightning, Lightning.Extensions, ) # Pin the `Lightning.Adaptors.IconCache` on-disk path to a per-OS-PID -# directory and wipe it at startup. Without the wipe, leftover files -# from a prior run can mask a Mox expectation by short-circuiting -# `IconCache.cached?/5`, since `System.unique_integer/1` resets per-VM -# and recycles. Keying by OS PID also keeps concurrent `mix test` runs -# (parallel CI shards, separate tmux panes) from colliding. +# directory and wipe it at startup, so no test ever sees a file a prior +# run left behind. Keying by OS PID also keeps concurrent `mix test` +# runs (parallel CI shards, separate tmux panes) from colliding. icon_dir = Path.join([ System.tmp_dir!(), From df888e6605478943d49d703642666e70be0af9cc Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Tue, 8 Sep 2026 14:43:54 +0200 Subject: [PATCH 17/37] Warn at boot on an empty adaptor catalogue, make the refresh interval configurable - Boot-time warning added when the adaptor catalogue is empty - Adaptor refresh interval now operator-configurable - Both documented in ADAPTORS.md --- .env.example | 4 ++ ADAPTORS.md | 6 +- DEPLOYMENT.md | 1 + lib/lightning/adaptors/scheduler.ex | 58 +++++++++++++------ lib/lightning/adaptors/supervisor.ex | 4 +- lib/lightning/config/bootstrap.ex | 4 +- .../adaptors/channel_broadcaster_test.exs | 5 +- test/lightning/adaptors/invalidator_test.exs | 5 +- test/lightning/adaptors/node_monitor_test.exs | 5 +- test/lightning/adaptors/readiness_test.exs | 15 ++++- test/lightning/adaptors/scheduler_test.exs | 49 +++++++++++++++- test/lightning/adaptors/store_test.exs | 9 ++- test/lightning/config/bootstrap_test.exs | 41 +++++++++++++ .../channels/run_channel_test.exs | 5 +- test/support/adaptor_test_helpers.ex | 12 +++- 15 files changed, 188 insertions(+), 35 deletions(-) diff --git a/.env.example b/.env.example index fed1090c9f2..915b2d1fbad 100644 --- a/.env.example +++ b/.env.example @@ -279,6 +279,10 @@ # # HTTP receive timeout (ms) for registry/schema/icon fetches. Defaults to 30s. # ADAPTORS_NPM_HTTP_TIMEOUT=30000 +# +# How often (ms) the catalogue refreshes from npm. Defaults to one hour. Set +# to 0 to disable scheduled refreshes. +# ADAPTORS_REFRESH_INTERVAL_MS=3600000 # ============================================================================== # <><><> WEBHOOK RETRY SETTINGS <><><> diff --git a/ADAPTORS.md b/ADAPTORS.md index ae3b2b1fcc6..fdb7beecb93 100644 --- a/ADAPTORS.md +++ b/ADAPTORS.md @@ -89,7 +89,9 @@ download not covered here. ## Keeping the catalogue fresh -Lightning refreshes the catalogue hourly; force one, on a source checkout: +Lightning refreshes the catalogue hourly. Set `ADAPTORS_REFRESH_INTERVAL_MS` to +change that interval, or to `0` to disable scheduled refreshes. Force one +manually, on a source checkout: ```sh mix lightning.adaptors.refresh @@ -119,3 +121,5 @@ bin/lightning rpc 'Lightning.Adaptors.refresh_package("@openfn/language-http")' the same name; the log names each shadowed package. - Deprecated-variable boot warning: rename `LOCAL_ADAPTORS=true` to `ADAPTORS_STRATEGY=local` and `OPENFN_ADAPTORS_REPO` to `ADAPTORS_LOCAL_REPO`. +- Workflow save rejected with "adaptor catalogue is not ready yet": see + [Running without internet access](#running-without-internet-access). diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index e71a22eeede..396cf0b0936 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -193,6 +193,7 @@ For SMTP, the following environment variables are required: | `ADAPTORS_NPM_JSDELIVR_URL` | CDN that adaptor credential schemas are fetched from. Defaults to `https://cdn.jsdelivr.net`. | | `ADAPTORS_NPM_REGISTRY_URL` | npm registry the adaptor catalogue is read from. Defaults to `https://registry.npmjs.org`. Set this and the two URLs above to use an internal mirror. See [Adaptors](ADAPTORS.md). | | `ADAPTORS_PATH` | Where you store your locally installed adaptors | +| `ADAPTORS_REFRESH_INTERVAL_MS` | How often, in milliseconds, the adaptor catalogue refreshes from npm. Defaults to one hour. Set to `0` to disable scheduled refreshes. See [Adaptors](ADAPTORS.md). | | `ADAPTORS_STRATEGY` | Where the adaptor catalogue comes from: `npm` (default) or `local`. See [Adaptors](ADAPTORS.md). | | `ALLOW_SIGNUP` | Set to `true` to enable user access to the registration page. Set to `false` to disable new user registrations and block access to the registration page.
Default is `false`. | | `CORS_ORIGIN` | A list of acceptable hosts for browser/cors requests (',' separated) | diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 82db252e71a..63996f0c97f 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -135,37 +135,59 @@ defmodule Lightning.Adaptors.Scheduler do icon_refreshes: %{} } - if interval_ms > 0 do - {:ok, state, {:continue, :schedule_first_tick}} - else - Logger.info("Adaptors[#{source}]: scheduler started interval=0 (disabled)") - {:ok, state} - end + {:ok, state, {:continue, :check_catalogue}} end @impl true - def handle_continue(:schedule_first_tick, state) do - delay = first_tick_delay(state) - Process.send_after(self(), :tick, delay) + def handle_continue(:check_catalogue, state) do + # Read runs, and delay is computed, even when interval_ms == 0 — that's + # the only way an interval=0 (disabled) deployment still gets the + # empty-catalogue warning below. Don't skip it for that branch. + checked_at = + case read_checked_at(state) do + nil -> + Logger.warning( + "Adaptors[#{state.source}]: catalogue is empty at boot — see " <> + "ADAPTORS.md's \"Running without internet access\" section" + ) - Logger.info( - "Adaptors[#{state.source}]: scheduler started interval=#{state.interval_ms}ms " <> - "next_tick_in=#{delay}ms" - ) + nil + + :error -> + nil + + checked_at -> + checked_at + end + + delay = time_until_next_ms(checked_at, state.interval_ms) + + if state.interval_ms > 0 do + Process.send_after(self(), :tick, delay) + + Logger.info( + "Adaptors[#{state.source}]: scheduler started interval=#{state.interval_ms}ms " <> + "next_tick_in=#{delay}ms" + ) + else + Logger.info( + "Adaptors[#{state.source}]: scheduler started interval=0 (disabled)" + ) + end {:noreply, state} end - defp first_tick_delay(state) do - time_until_next_ms(state.checked_at.(state.source), state.interval_ms) + defp read_checked_at(state) do + state.checked_at.(state.source) rescue e in DBConnection.ConnectionError -> Logger.warning( - "Adaptors[#{state.source}]: scheduler could not read max_checked_at, " <> - "ticking immediately: #{Exception.message(e)}" + "Adaptors[#{state.source}]: scheduler could not read max_checked_at: " <> + Exception.message(e) ) - 0 + :error end @impl true diff --git a/lib/lightning/adaptors/supervisor.ex b/lib/lightning/adaptors/supervisor.ex index 1d98d662e96..10415b588f2 100644 --- a/lib/lightning/adaptors/supervisor.ex +++ b/lib/lightning/adaptors/supervisor.ex @@ -24,6 +24,8 @@ defmodule Lightning.Adaptors.Supervisor do defaulting to `Lightning.Adaptors.Config.strategy/0` * `:lock_key` - `HighlanderPG` advisory-lock key, defaulting to `lock_key(name)` + * `:checked_at` - forwarded to the scheduler; see + `Lightning.Adaptors.Scheduler.start_link/1` """ @spec start_link(keyword()) :: Supervisor.on_start() def start_link(opts) do @@ -62,7 +64,7 @@ defmodule Lightning.Adaptors.Supervisor do cache: cache, tasks: tasks, source_topic: source_topic - ] + ] ++ Keyword.take(opts, [:checked_at]) ]} } diff --git a/lib/lightning/config/bootstrap.ex b/lib/lightning/config/bootstrap.ex index 9e86b967270..c05dfa2308a 100644 --- a/lib/lightning/config/bootstrap.ex +++ b/lib/lightning/config/bootstrap.ex @@ -1047,7 +1047,9 @@ defmodule Lightning.Config.Bootstrap do # doesn't bake in a build-time tmp path. An explicit override has # no such concern. icon_path: - env!("ADAPTORS_ICONS_PATH", :string, nil) |> expand_or_nil() + env!("ADAPTORS_ICONS_PATH", :string, nil) |> expand_or_nil(), + refresh_interval: + env!("ADAPTORS_REFRESH_INTERVAL_MS", :integer?, nil) ] |> Enum.reject(fn {_key, value} -> is_nil(value) end) diff --git a/test/lightning/adaptors/channel_broadcaster_test.exs b/test/lightning/adaptors/channel_broadcaster_test.exs index 1eff41bc120..04c480362e3 100644 --- a/test/lightning/adaptors/channel_broadcaster_test.exs +++ b/test/lightning/adaptors/channel_broadcaster_test.exs @@ -15,7 +15,10 @@ defmodule Lightning.Adaptors.ChannelBroadcasterTest do # The supervisor starts the ChannelBroadcaster automatically, registered # under `channel_broadcaster_name(sup)`. start_supervised!( - {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + {AdaptorsSupervisor, + name: sup, + strategy: Lightning.Adaptors.StrategyMock, + checked_at: fn _source -> nil end} ) source_topic = AdaptorsSupervisor.source_topic(sup) diff --git a/test/lightning/adaptors/invalidator_test.exs b/test/lightning/adaptors/invalidator_test.exs index 1b40430e7ae..b1a5e57bbd6 100644 --- a/test/lightning/adaptors/invalidator_test.exs +++ b/test/lightning/adaptors/invalidator_test.exs @@ -9,7 +9,10 @@ defmodule Lightning.Adaptors.InvalidatorTest do # The supervisor starts the Invalidator automatically, registered under # `invalidator_name(sup)`. start_supervised!( - {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + {AdaptorsSupervisor, + name: sup, + strategy: Lightning.Adaptors.StrategyMock, + checked_at: fn _source -> nil end} ) cache = AdaptorsSupervisor.cache_name(sup) diff --git a/test/lightning/adaptors/node_monitor_test.exs b/test/lightning/adaptors/node_monitor_test.exs index 14ab23ed412..ed40f2e3aa1 100644 --- a/test/lightning/adaptors/node_monitor_test.exs +++ b/test/lightning/adaptors/node_monitor_test.exs @@ -16,7 +16,10 @@ defmodule Lightning.Adaptors.NodeMonitorTest do # The supervisor starts the NodeMonitor automatically, registered under # `node_monitor_name(sup)`. start_supervised!( - {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + {AdaptorsSupervisor, + name: sup, + strategy: Lightning.Adaptors.StrategyMock, + checked_at: fn _source -> nil end} ) cache = AdaptorsSupervisor.cache_name(sup) diff --git a/test/lightning/adaptors/readiness_test.exs b/test/lightning/adaptors/readiness_test.exs index ec4a80024a7..f7da798f785 100644 --- a/test/lightning/adaptors/readiness_test.exs +++ b/test/lightning/adaptors/readiness_test.exs @@ -26,7 +26,10 @@ defmodule Lightning.Adaptors.ReadinessTest do sup = :"readiness_test_#{System.unique_integer([:positive])}" start_supervised!( - {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + {AdaptorsSupervisor, + name: sup, + strategy: Lightning.Adaptors.StrategyMock, + checked_at: fn _source -> nil end} ) :ok = @@ -41,12 +44,16 @@ defmodule Lightning.Adaptors.ReadinessTest do pid = start_supervised!({ Scheduler, + # Its boot-time max_checked_at read runs in a process with no + # $callers chain back to this test — the Sandbox.allow/3 below + # races it, so skip the read rather than risk an OwnershipError. name: AdaptorsSupervisor.global_scheduler_name(sup), sup: sup, lock_key: AdaptorsSupervisor.lock_key(sup), cache: AdaptorsSupervisor.cache_name(sup), tasks: AdaptorsSupervisor.tasks_name(sup), - source_topic: AdaptorsSupervisor.source_topic(sup) + source_topic: AdaptorsSupervisor.source_topic(sup), + checked_at: fn _source -> nil end }) Ecto.Adapters.SQL.Sandbox.allow(Lightning.Repo, self(), pid) @@ -119,7 +126,9 @@ defmodule Lightning.Adaptors.ReadinessTest do start_supervised!( Supervisor.child_spec( {AdaptorsSupervisor, - name: local_sup, strategy: Lightning.Adaptors.Local}, + name: local_sup, + strategy: Lightning.Adaptors.Local, + checked_at: fn _source -> nil end}, id: local_sup ) ) diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index 9e12fd5fffc..b2571989e99 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -7,6 +7,7 @@ defmodule Lightning.Adaptors.SchedulerTest do import Lightning.AdaptorTestHelpers import Eventually + import ExUnit.CaptureLog import Mox alias Lightning.Adaptors.Catalogue @@ -27,9 +28,15 @@ defmodule Lightning.Adaptors.SchedulerTest do setup do sup = :"sched_test_#{System.unique_integer([:positive])}" - start_supervised!( - {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} - ) + start_supervised!({ + AdaptorsSupervisor, + # Keeps the auto-started scheduler a true inert no-op ahead of + # start_scheduler/2 below — otherwise its boot-time max_checked_at + # read logs an empty-catalogue warning on every test in this file. + name: sup, + strategy: Lightning.Adaptors.StrategyMock, + checked_at: fn _source -> nil end + }) # Default no-op icons stub for tests that don't care about the icons # pipeline. Individual tests override via `expect` when they need to @@ -213,6 +220,42 @@ defmodule Lightning.Adaptors.SchedulerTest do assert_receive {:DOWN, ^ref, :process, ^pid, reason}, 2000 assert {%Postgrex.Error{}, _stacktrace} = reason end + + test "an empty catalogue logs a boot warning and still ticks when interval > 0", + %{sup: sup} do + test_pid = self() + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :tick_ran) + {:ok, []} + end) + + log = + capture_log(fn -> + start_scheduler(sup, checked_at: fn _source -> nil end, interval: 30) + assert_receive :tick_ran, 2000 + end) + + assert log =~ "catalogue is empty at boot" + end + + test "an empty catalogue logs a boot warning and schedules no tick when interval is 0", + %{sup: sup} do + test_pid = self() + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + send(test_pid, :tick_ran) + {:ok, []} + end) + + log = + capture_log(fn -> + start_scheduler(sup, checked_at: fn _source -> nil end, interval: 0) + refute_receive :tick_ran, 200 + end) + + assert log =~ "catalogue is empty at boot" + end end describe "do_refresh/1 diff logic" do diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index f6bb41df205..e9ee54adadc 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -20,7 +20,10 @@ defmodule Lightning.Adaptors.StoreTest do sup = :"store_test_#{System.unique_integer([:positive])}" start_supervised!( - {AdaptorsSupervisor, name: sup, strategy: Lightning.Adaptors.StrategyMock} + {AdaptorsSupervisor, + name: sup, + strategy: Lightning.Adaptors.StrategyMock, + checked_at: fn _source -> nil end} ) cache = AdaptorsSupervisor.cache_name(sup) @@ -229,7 +232,9 @@ defmodule Lightning.Adaptors.StoreTest do start_supervised!( Supervisor.child_spec( {AdaptorsSupervisor, - name: local_sup, strategy: Lightning.Adaptors.Local}, + name: local_sup, + strategy: Lightning.Adaptors.Local, + checked_at: fn _source -> nil end}, id: local_sup ) ) diff --git a/test/lightning/config/bootstrap_test.exs b/test/lightning/config/bootstrap_test.exs index 54cad2638e8..c04a86ff247 100644 --- a/test/lightning/config/bootstrap_test.exs +++ b/test/lightning/config/bootstrap_test.exs @@ -663,6 +663,47 @@ defmodule Lightning.Config.BootstrapTest do end end + describe "adaptors refresh interval" do + test "ADAPTORS_REFRESH_INTERVAL_MS sets refresh_interval when present" do + Dotenvy.source([%{"ADAPTORS_REFRESH_INTERVAL_MS" => "60000"}]) + + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors)[:refresh_interval] == + 60_000 + end + + test "ADAPTORS_REFRESH_INTERVAL_MS accepts 0 to disable the scheduler" do + Dotenvy.source([%{"ADAPTORS_REFRESH_INTERVAL_MS" => "0"}]) + + Bootstrap.configure() + + assert get_env(:lightning, Lightning.Adaptors)[:refresh_interval] == 0 + end + + test "is not forced when unset, so config/test.exs's 0 is left alone" do + Dotenvy.source([%{}]) + + Bootstrap.configure() + + refute Keyword.has_key?( + get_env(:lightning, Lightning.Adaptors), + :refresh_interval + ) + end + + test "does not set refresh_interval when set but empty" do + Dotenvy.source([%{"ADAPTORS_REFRESH_INTERVAL_MS" => ""}]) + + Bootstrap.configure() + + refute Keyword.has_key?( + get_env(:lightning, Lightning.Adaptors), + :refresh_interval + ) + end + end + describe "adaptors strategy" do test "defaults to the npm strategy when nothing is set" do Dotenvy.source([%{}]) diff --git a/test/lightning_web/channels/run_channel_test.exs b/test/lightning_web/channels/run_channel_test.exs index 480cdf9ff00..ffd5f4d1487 100644 --- a/test/lightning_web/channels/run_channel_test.exs +++ b/test/lightning_web/channels/run_channel_test.exs @@ -313,6 +313,7 @@ defmodule LightningWeb.RunChannelTest do } end + @tag run_state: :claimed test "fetch:plan replies with an error when a job adaptor cannot be resolved", %{project: project} = context do seed_ready_catalogue() @@ -330,7 +331,7 @@ defmodule LightningWeb.RunChannelTest do {:ok, snapshot} = Workflows.Snapshot.create(workflow) - %{socket: socket} = + %{socket: socket, run: run} = context |> Map.merge(%{workflow: workflow, trigger: trigger, snapshot: snapshot}) |> merge_setups([:create_run, :create_socket, :join_run_channel]) @@ -338,6 +339,7 @@ defmodule LightningWeb.RunChannelTest do ref = push(socket, "fetch:plan", %{}) assert_reply ref, :error, %{reason: "adaptor_not_found"} + assert %{state: :claimed} = Lightning.Repo.reload!(run) end @tag project_retention_policy: :erase_all @@ -2865,6 +2867,7 @@ defmodule LightningWeb.RunChannelTest do starting_trigger: trigger, dataclip: dataclip, snapshot: snapshot, + state: Map.get(context, :run_state, :available), options: Lightning.Extensions.MockUsageLimiter.get_run_options(%Context{ project_id: project.id diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex index 65aa1dcb704..8d3f31ea69e 100644 --- a/test/support/adaptor_test_helpers.ex +++ b/test/support/adaptor_test_helpers.ex @@ -37,8 +37,16 @@ defmodule Lightning.AdaptorTestHelpers do ExUnit.Callbacks.start_supervised!( Supervisor.child_spec( - {AdaptorsSupervisor, - name: sup, strategy: Lightning.Adaptors.StrategyMock}, + { + AdaptorsSupervisor, + # The Scheduler's boot-time max_checked_at read runs in a process + # with no $callers chain back to this test, so it can't see an + # `async: true` module's own sandbox connection. Skip the read + # entirely rather than let it crash on an OwnershipError. + name: sup, + strategy: Lightning.Adaptors.StrategyMock, + checked_at: fn _source -> nil end + }, id: sup ) ) From f23bb7143e70a67c0c8d3215b59ff2a235efd096 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 3 Sep 2026 10:05:38 +0200 Subject: [PATCH 18/37] Add docs-style rule, docs-review skill, and replace plan commands with skills - docs-style rule added for shipped guides, plus a user-invoked docs-review skill; later anchored to root docs so it doesn't fire on any *.md - create-plan, implement-plan and tdd skills replace the old plan commands; create-plan made agent-invocable - reshape-pass skill added --- .claude/agents/context-analyzer.md | 21 +- .claude/agents/context-locator.md | 20 +- .claude/commands/create-plan.md | 376 -------------------- .claude/commands/implement-plan.md | 146 -------- .claude/commands/validate-plan.md | 159 --------- .claude/rules/docs-style.md | 60 ++++ .claude/skills/create-plan/SKILL.md | 106 ++++++ .claude/skills/create-plan/plan-template.md | 69 ++++ .claude/skills/docs-review/SKILL.md | 58 +++ .claude/skills/implement-plan/SKILL.md | 26 ++ .claude/skills/reshape-pass/SKILL.md | 205 +++++++++++ .claude/skills/tdd/SKILL.md | 68 ++++ CHANGELOG.md | 5 + CLAUDE.md | 4 +- 14 files changed, 631 insertions(+), 692 deletions(-) delete mode 100644 .claude/commands/create-plan.md delete mode 100644 .claude/commands/implement-plan.md delete mode 100644 .claude/commands/validate-plan.md create mode 100644 .claude/rules/docs-style.md create mode 100644 .claude/skills/create-plan/SKILL.md create mode 100644 .claude/skills/create-plan/plan-template.md create mode 100644 .claude/skills/docs-review/SKILL.md create mode 100644 .claude/skills/implement-plan/SKILL.md create mode 100644 .claude/skills/reshape-pass/SKILL.md create mode 100644 .claude/skills/tdd/SKILL.md diff --git a/.claude/agents/context-analyzer.md b/.claude/agents/context-analyzer.md index bc6ab3236dd..3e4d630836c 100644 --- a/.claude/agents/context-analyzer.md +++ b/.claude/agents/context-analyzer.md @@ -1,7 +1,7 @@ --- name: context-analyzer description: The research equivalent of codebase-analyzer. Use this subagent_type when wanting to deep dive on context documents. Not commonly needed otherwise. -tools: Read, Grep, Glob +tools: Read, Grep, Glob, Bash model: sonnet effort: high --- @@ -13,9 +13,19 @@ You are a specialist at extracting HIGH-VALUE insights from context documents. Y ### Step 1: Read with Purpose - Read the entire document first - Identify the document's main goal -- Note the date and context +- Take the date from `git -C .context log -1 --format=%cs -- `, not + the filename - Understand what question it was answering +### Step 1b: Check the code has not moved on +For each file, module or function the document cites, check it still exists +and whether it changed after the document's date: +`git log -1 --format=%cs -- ` in the code repo. A cited path that is gone +or was rewritten after the doc's date marks that section **superseded**: report +it as history (what was decided and why), never as a description of the +current code. Decisions, requirements and rejected options do not rot this +way; only claims about how the code works do. + ### Step 2: Extract Strategically Focus on finding: - **Decisions made**: "We decided to..." @@ -45,7 +55,8 @@ Structure your analysis like this: ### Document Context - **Date**: [When written] - **Purpose**: [Why this document exists] -- **Status**: [Is this still relevant/implemented/superseded?] +- **Status**: [current | partly superseded | superseded]. List each cited + path that has moved or changed since the doc date. ### Key Decisions 1. **[Decision Topic]**: [Specific decision made] @@ -74,7 +85,9 @@ Structure your analysis like this: - [Decisions that were deferred] ### Relevance Assessment -[1-2 sentences on whether this information is still applicable and why] +[1-2 sentences on whether this information is still applicable and why. A +plan whose cited code has changed since it was written: "approach is history, +re-verify before copying".] ``` ## Example Transformation diff --git a/.claude/agents/context-locator.md b/.claude/agents/context-locator.md index c9edf367961..8cf43d3f8d3 100644 --- a/.claude/agents/context-locator.md +++ b/.claude/agents/context-locator.md @@ -1,7 +1,7 @@ --- name: context-locator description: Discovers relevant documents in .context/ directory - the context equivalent of codebase-locator for finding project documentation, notes, and historical context -tools: Grep, Glob +tools: Grep, Glob, Bash model: haiku effort: low --- @@ -35,7 +35,9 @@ their contents in depth. 3. **Return organized results** - Group by document type - Include brief one-line description from title/header - - Note document dates if visible in filename + - Date every hit from git, not the filename: run + `git -C .context log -1 --format=%cs -- ` (`.context` is its own + repo). Put the date on the line. ## Search Strategy @@ -62,6 +64,12 @@ to best categorize the findings for the user. └── *.md # Various root-level documentation files ``` +### Scope + +Start with `shared/` and the topic directory for the feature at hand. Widen to +personal directories and root files only when that turns up nothing useful. +Skip `archive/` unless the request asks for it by name. + ### Search Patterns - Use grep for content searching @@ -102,14 +110,14 @@ Structure your findings like this: ## Context Documents about [Topic] ### GitHub Issues -- `.context/shared/issues/issue-3635-save-button.md` - Save button implementation -- `.context/shared/issues/issue-3624-workflow-editor-header.md` - Workflow editor header +- `.context/shared/issues/issue-3635-save-button.md` (2024-09-30) - Save button implementation +- `.context/shared/issues/issue-3624-workflow-editor-header.md` (2024-09-28) - Workflow editor header ### Research Documents - `.context/shared/research/2024-10-01-yjs-integration.md` - Research on Yjs collaborative editing ### Implementation Plans -- `.context/shared/plans/save-implementation.md` - Detailed plan for save functionality +- `.context/shared/plans/save-implementation.md` (2024-10-03) - Detailed plan for save functionality ### Architecture & Design - `.context/shared/architecture/store-structure.md` - Store architecture documentation @@ -137,6 +145,8 @@ Total: 12 relevant documents found - **Check multiple locations** - Shared, personal, and root level - **Don't read full file contents** - Just scan for relevance - **Preserve exact paths** - Show where documents live +- **Date every hit** - The reader ranks by age relative to the code; a hit + without a date is unranked - **Be thorough** - Check subdirectories AND root level - **Group logically** - Make categories meaningful - **Note patterns** - Help user understand naming conventions diff --git a/.claude/commands/create-plan.md b/.claude/commands/create-plan.md deleted file mode 100644 index fec1f041888..00000000000 --- a/.claude/commands/create-plan.md +++ /dev/null @@ -1,376 +0,0 @@ ---- -argument-hint: [issue-file-path] -description: Create detailed implementation plan ---- - -# Implementation Plan - -You are tasked with creating detailed implementation plans through an interactive, iterative process. You should be skeptical, thorough, and work collaboratively with the user to produce high-quality technical specifications. - -**Usage**: `/create-plan $ARGUMENTS` - -If `$ARGUMENTS` is provided with an issue file path, read it and begin research immediately. -If no argument provided, present the initial prompt and ask the user for context. - -## Initial Response - -When this command is invoked: - -1. **If `$ARGUMENTS` is provided**: - - Immediately read the file at `$ARGUMENTS` - - Begin the research process without waiting for user input - - Proceed directly to Step 1: Context Gathering & Initial Analysis - -2. **If `$ARGUMENTS` is empty**, respond with: -``` -I'll help you create a detailed implementation plan. Let me start by understanding what we're building. - -Please provide: -1. The task/ticket description (or reference to a ticket/issue file) -2. Any relevant context, constraints, or specific requirements -3. Links to related research or previous implementations - -I'll analyze this information and work with you to create a comprehensive plan. - -Tip: You can also invoke this command with an issue file directly: `/create-plan .context/shared/issues/issue-1234.md` -``` - -Then wait for the user's input. - -## Process Steps - -### Step 1: Context Gathering & Initial Analysis - -1. **Read all mentioned files immediately**: - - Issue/ticket files (e.g., `.context/shared/issues/issue-1234.md`) - - Research documents - - Related implementation plans - - Any JSON/data files mentioned - - Read mentioned files fully (no limit/offset) in the main context before spawning sub-tasks. - -2. **Spawn initial research tasks to gather context**: - Before asking the user any questions, use specialized agents to research in parallel: - - - Use the **codebase-locator** agent to find all files related to the ticket/task - - Use the **codebase-analyzer** agent to understand how the current implementation works - - If relevant, use the **context-locator** agent to find any existing context documents about this feature - - These agents will: - - Find relevant source files, configs, and tests - - Identify the specific directories to focus on (e.g., frontend work in assets/, backend work in lib/) - - Trace data flow and key functions - - Return detailed explanations with file:line references - -3. **Read all files identified by research tasks**: - - After research tasks complete, read ALL files they identified as relevant - - Read them FULLY into the main context - - This ensures you have complete understanding before proceeding - -4. **Analyze and verify understanding**: - - Cross-reference the ticket requirements with actual code - - Identify any discrepancies or misunderstandings - - Note assumptions that need verification - - Determine true scope based on codebase reality - -5. **Present informed understanding and focused questions**: - ``` - Based on the ticket and my research of the codebase, I understand we need to [accurate summary]. - - I've found that: - - [Current implementation detail with file:line reference] - - [Relevant pattern or constraint discovered] - - [Potential complexity or edge case identified] - - Questions that my research couldn't answer: - - [Specific technical question that requires human judgment] - - [Business logic clarification] - - [Design preference that affects implementation] - ``` - - Only ask questions that you genuinely cannot answer through code investigation. - -### Step 2: Research & Discovery - -After getting initial clarifications: - -1. **If the user corrects any misunderstanding**: - - DO NOT just accept the correction - - Spawn new research tasks to verify the correct information - - Read the specific files/directories they mention - - Only proceed once you've verified the facts yourself - -2. **Create a research todo list** using TodoWrite to track exploration tasks - -3. **Spawn parallel sub-tasks for comprehensive research**: - - Create multiple Task agents to research different aspects concurrently - - See [CLAUDE.md §Available Agents](../../CLAUDE.md#available-agents) for the canonical agent roster. - -3. **Wait for ALL sub-tasks to complete** before proceeding - -4. **Present findings and design options**: - ``` - Based on my research, here's what I found: - - **Current State:** - - [Key discovery about existing code] - - [Pattern or convention to follow] - - **Design Options:** - 1. [Option A] - [pros/cons] - 2. [Option B] - [pros/cons] - - **Open Questions:** - - [Technical uncertainty] - - [Design decision needed] - - Which approach aligns best with your vision? - ``` - -### Step 3: Plan Structure Development - -Once aligned on approach: - -1. **Create initial plan outline**: - ``` - Here's my proposed plan structure: - - ## Overview - [1-2 sentence summary] - - ## Implementation Phases: - 1. [Phase name] - [what it accomplishes] - 2. [Phase name] - [what it accomplishes] - 3. [Phase name] - [what it accomplishes] - - Does this phasing make sense? Should I adjust the order or granularity? - ``` - -2. **Get feedback on structure** before writing details - -### Step 4: Detailed Plan Writing - -After structure approval: - -1. **Identify agent assignments for each phase**: - - For each implementation phase, determine which specialized agent should handle it. See [CLAUDE.md §Available Agents](../../CLAUDE.md#available-agents) for the canonical roster. - - Each phase uses a fresh agent instance, which isolates it: a later phase cannot be misled by an earlier phase's abandoned attempts. This is the default for any plan with more than one phase; a single-phase plan may be implemented directly. - - The agent assignment tells the implementation coordinator which agent to spawn for that phase - -2. **Write the plan** to `.context/shared/plans/YYYY-MM-DD-XXXX-description.md` - - - Format: `YYYY-MM-DD-XXXX-description.md` where: - - YYYY-MM-DD is today's date - - XXXX is the ticket number (omit if no ticket) - - description is a brief kebab-case description - - Examples: - - With ticket: `2025-01-08-1478-parent-child-tracking.md` - - Without ticket: `2025-01-08-improve-error-handling.md` -3. **Use this template structure**: - -````markdown -# [Feature/Task Name] Implementation Plan - -## Overview - -[Brief description of what we're implementing and why] - -## Current State Analysis - -[What exists now, what's missing, key constraints discovered] - -## Desired End State - -[A Specification of the desired end state after this plan is complete, and how to verify it] - -### Key Discoveries: -- [Important finding with file:line reference] -- [Pattern to follow] -- [Constraint to work within] - -## What We're NOT Doing - -[Explicitly list out-of-scope items to prevent scope creep] - -## Implementation Approach - -[High-level strategy and reasoning] - -## Phase 1: [Descriptive Name] - -**Implementation Agent**: `[agent-type]` - - -### Overview -[What this phase accomplishes] - -### Changes Required: - -#### 1. [Component/File Group] -**File**: `path/to/file.ext` -**Changes**: [Summary of changes] - -```[language] -// Specific code to add/modify -``` - -### Success Criteria: - -#### Automated Verification: -- [ ] Migration applies cleanly: `` -- [ ] Unit tests pass: `` -- [ ] Type checking passes: `` -- [ ] Linting passes: `` -- [ ] Integration tests pass: `` -- [ ] API endpoint returns 200: `curl localhost:/api/new-endpoint` - -#### Manual Verification: -- [ ] Feature works as expected when tested via UI -- [ ] Performance is acceptable under load -- [ ] Edge case handling verified manually -- [ ] No regressions in related features - ---- - -## Phase 2: [Descriptive Name] - -[Similar structure with both automated and manual success criteria...] - ---- - -## Testing Strategy - -### Unit Tests: -- [What to test] -- [Key edge cases] - -### Integration Tests: -- [End-to-end scenarios] - -### Manual Testing Steps: -1. [Specific step to verify feature] -2. [Another verification step] -3. [Edge case to test manually] - -## Performance Considerations - -[Any performance implications or optimizations needed] - -## Migration Notes - -[If applicable, how to handle existing data/systems] - -## References - -- Original issue: `.context/shared/issues/issue-XXXX.md` -- Related research: `.context/shared/research/[relevant].md` -- Similar implementation: `[file:line]` -```` - -### Step 5: Review and Iterate - -1. **Present the draft plan location**: - ``` - I've created the initial implementation plan at: - `.context/shared/plans/YYYY-MM-DD-XXXX-description.md` - - Please review it and let me know: - - Are the phases properly scoped? - - Are the success criteria specific enough? - - Any technical details that need adjustment? - - Missing edge cases or considerations? - ``` - -2. **Iterate based on feedback** - be ready to: - - Add missing phases - - Adjust technical approach - - Clarify success criteria (both automated and manual) - - Add/remove scope items - -3. **Continue refining** until the user is satisfied - -## Important Guidelines - -1. Include specific file paths and line numbers. Automated verification steps should use project-specific commands (e.g., `mix verify`, `npm test`). - -2. **Track progress** with TodoWrite for non-trivial plans. - -3. **No open questions in the final plan**: resolve them before finalizing. The implementation plan must be complete and actionable. - -## Success Criteria Guidelines - -**Always separate success criteria into two categories:** - -1. **Automated Verification** (can be run by execution agents): - - Shell commands the project provides for tests, linting, type-checking, etc. - - Specific files that should exist - - Code compilation/type checking - - Automated test suites - -2. **Manual Verification** (requires human testing): - - UI/UX functionality - - Performance under real conditions - - Edge cases that are hard to automate - - User acceptance criteria - -Review the CHANGELOG entry against the final implementation. Lightning uses -Keep-a-Changelog; a user-visible change merging to `main` needs an accurate entry with -an issue or PR link. Prefer broadening an existing entry over adding a second one. -"No change needed" is a fine outcome — the review is what matters. - -## Common Patterns - -### For Database Changes: -- Start with schema/migration -- Add store methods -- Update business logic -- Expose via API -- Update clients - -### For New Features: -- Research existing patterns first -- Start with data model -- Build backend logic -- Add API endpoints -- Implement UI last - -### For Refactoring: -- Document current behavior -- Plan incremental changes -- Maintain backwards compatibility -- Include migration strategy - -## Sub-task Spawning Best Practices - -When spawning research sub-tasks: - -1. **Spawn multiple tasks in parallel** for efficiency -2. **Each task should be focused** on a specific area -3. **Provide detailed instructions** including: - - Exactly what to search for - - Which directories to focus on - - What information to extract - - Expected output format -4. **Be specific about directories** — name the exact path (e.g., `assets/`, `lib/`): - - Include the full path context in your prompts -5. **Request specific file:line references** in responses -6. **Wait for all tasks to complete** before synthesizing -7. **Verify sub-task results**: - - If a sub-task returns unexpected results, spawn follow-up tasks - - Cross-check findings against the actual codebase - - Don't accept results that seem incorrect - -## Example Interaction Flow - -``` -User: /create-plan -Assistant: I'll help you create a detailed implementation plan... - -User: We need to add workflow save functionality. See .context/shared/issues/issue-3635.md -Assistant: Let me read that issue file completely first... - -[Reads file fully] - -Based on the issue, I understand we need to implement save functionality for the collaborative workflow editor. Before I start planning, I have some questions... - -[Interactive process continues...] -``` diff --git a/.claude/commands/implement-plan.md b/.claude/commands/implement-plan.md deleted file mode 100644 index 9ab3a3a072a..00000000000 --- a/.claude/commands/implement-plan.md +++ /dev/null @@ -1,146 +0,0 @@ ---- -argument-hint: [plan-file-path] -description: Implement approved technical plan ---- - -# Implement Plan - -You are tasked with implementing an approved technical plan from `.context/shared/plans/`. These plans contain phases with specific changes and success criteria. - -**Usage**: `/implement-plan $ARGUMENTS` - -If `$ARGUMENTS` is empty, ask user for plan path. Otherwise, read the plan at `$ARGUMENTS` and begin implementation. - -## Getting Started - -When `$ARGUMENTS` contains a plan path: -- Read the plan at `$ARGUMENTS` completely and check for any existing checkmarks (- [x]) -- **Identify the agent assignment** for each phase (marked as `**Implementation Agent**: ...`) -- Read the original ticket and all files mentioned in the plan -- Read files fully (no limit/offset). -- Create a todo list to track your progress across all phases - -If `$ARGUMENTS` is empty, ask for one. - -## Available Agent Types - -See [CLAUDE.md §Available Agents](../../CLAUDE.md#available-agents) for the canonical roster and scopes. Pick the agent whose scope matches the phase's work type. - -## Agent-Based Phase Implementation - -**This is the core of the implementation process**: - -1. **For each phase**, spawn a FRESH agent of the type specified in the plan. This is the default for any plan with more than one phase; a single-phase plan may be implemented directly. - -2. **Each agent gets a focused task**: - ``` - You are implementing Phase [N] of this plan: [plan path] - - Read the plan file completely, then implement ONLY this phase: - - ## Phase [N]: [Phase Name] - - [Copy the full phase details from the plan] - - After implementation: - 1. Run all automated verification steps listed in the success criteria - 2. Fix any issues that arise - 3. Update the plan file to check off completed items - 4. Report back with what was completed and any manual verification steps remaining - ``` - -3. **Wait for each phase to complete** before spawning the next agent - - A fresh agent isolates each phase: a later phase cannot be misled by an earlier phase's abandoned attempts - - Each agent focuses solely on their phase - -4. **Between phases**: verify the previous phase's work (check the plan, review agent output, coordinate manual verification), then move on with a new fresh agent. - -## Your Role as Coordinator - -As the main agent running this command, you are the **coordinator**, not the implementer: -- You read the plan and understand the full scope -- You track overall progress across all phases -- You handle issues and communicate with the user -- You coordinate manual verification between phases - -## Implementation Philosophy - -Plans are carefully designed, but reality can be messy. The job is to: -- Follow the plan's intent while adapting to what is found -- Implement each phase fully before moving to the next -- Verify work makes sense in the broader codebase context -- Update checkboxes in the plan as sections are completed - -When things don't match the plan exactly, think about why and communicate clearly. The plan is the guide, but judgment matters too. - -If an agent encounters a mismatch: -- The agent should STOP and report the issue -- Present the issue clearly to the user: - ``` - Issue in Phase [N]: - Expected: [what the plan says] - Found: [actual situation] - Why this matters: [explanation] - - How should I proceed? - ``` -- Wait for user guidance before continuing -- May need to spawn a new agent with updated instructions - -## Verification Approach - -Each phase agent is responsible for: -- Running all automated verification steps in the success criteria (see [CLAUDE.md §Common Commands](../../CLAUDE.md#common-commands) for the project's quality gates) -- Fixing any issues before reporting completion -- Updating checkboxes in the plan file using Edit -- Reporting what manual verification steps remain - -As coordinator, you should: -- Verify the agent completed their automated checks -- Coordinate any manual verification with the user -- Ensure quality before moving to the next phase -- Review the CHANGELOG entry against the final implementation. Lightning uses - Keep-a-Changelog; a user-visible change merging to `main` needs an accurate entry with - an issue or PR link. Prefer broadening an existing entry over adding a second one. - "No change needed" is a fine outcome — the review is what matters. - -## If an Agent Gets Stuck - -When an agent reports something isn't working as expected: -- Review what the agent tried -- Consider if the codebase has evolved since the plan was written -- Present the mismatch clearly to the user -- Get guidance before spawning a new agent with updated instructions - -If an agent is stuck, don't try to fix it yourself - either: -1. Guide the user to help resolve the issue, then spawn a new agent -2. Spawn a debugging/research agent to understand the issue -3. Update the plan and spawn a new implementation agent - -## Resuming Work - -If the plan has existing checkmarks: -- Identify which phase to start from (first unchecked phase) -- Trust that completed work is done -- Spawn an agent for the next incomplete phase -- Verify previous work only if something seems off - -## Example Flow - -``` -You (coordinator): Reading plan... I see 3 phases: - - Phase 1: Database Schema (phoenix-elixir-expert) ✅ Done - - Phase 2: API Endpoints (phoenix-elixir-expert) ⬜ Next - - Phase 3: React Components (react-collab-editor) ⬜ Pending - -I'll spawn a fresh phoenix-elixir-expert agent for Phase 2... - -[Agent implements Phase 2, runs tests, updates checkboxes] - -Agent completed Phase 2! All automated checks passed. -Manual verification needed: Test the API endpoints with curl. - -[Wait for user to verify or proceed] - -Now spawning a fresh react-collab-editor agent for Phase 3... -``` diff --git a/.claude/commands/validate-plan.md b/.claude/commands/validate-plan.md deleted file mode 100644 index 70cb58d9c1a..00000000000 --- a/.claude/commands/validate-plan.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -argument-hint: [plan-file-path] -description: Validate implementation against plan ---- - -# Validate Plan - -You are tasked with validating that an implementation plan was correctly executed, verifying all success criteria and identifying any deviations or issues. - -**Usage**: -- `/validate-plan` - Auto-discover plan from context/commits -- `/validate-plan ` - Validate specific plan file - -## Initial Setup - -When invoked: -1. **Determine plan location**: - - If `$ARGUMENTS` is provided, use that plan path - - Otherwise, search recent commits for plan references or ask user - -2. **Determine context** - Are you in an existing conversation or starting fresh? - - If existing: Review what was implemented in this session - - If fresh: Need to discover what was done through git and codebase analysis - -3. **Gather implementation evidence**: - ```bash - # Check recent commits - git log --oneline -n 20 - git diff HEAD~N..HEAD # Where N covers implementation commits - ``` - - Run `mix verify` (see [CLAUDE.md §Common Commands](../../CLAUDE.md#common-commands)). - -## Validation Process - -### Step 1: Context Discovery - -If starting fresh or need more context: - -1. **Read the implementation plan** completely -2. **Identify what should have changed**: - - List all files that should be modified - - Note all success criteria (automated and manual) - - Identify key functionality to verify - -3. **Spawn parallel research tasks** to discover implementation: - ``` - Task 1 - Verify database changes: - Research if migration [N] was added and schema changes match plan. - Check: migration files, schema version, table structure - Return: What was implemented vs what plan specified - - Task 2 - Verify code changes: - Find all modified files related to [feature]. - Compare actual changes to plan specifications. - Return: File-by-file comparison of planned vs actual - - Task 3 - Verify test coverage: - Check if tests were added/modified as specified. - Run test commands and capture results. - Return: Test status and any missing coverage - ``` - -### Step 2: Systematic Validation - -For each phase in the plan: - -1. **Check completion status**: - - Look for checkmarks in the plan (- [x]) - - Verify the actual code matches claimed completion - -2. **Run automated verification**: - - Execute each command from "Automated Verification" - - Document pass/fail status - - If failures, investigate root cause - -3. **Assess manual criteria**: - - List what needs manual testing - - Provide clear steps for user verification - -4. **Think deeply about edge cases**: - - Were error conditions handled? - - Are there missing validations? - - Could the implementation break existing functionality? - -### Step 3: Generate Validation Report - -Create comprehensive validation summary: - -```markdown -## Validation Report: [Plan Name] - -### Implementation Status -✓ Phase 1: [Name] - Fully implemented -✓ Phase 2: [Name] - Fully implemented -⚠️ Phase 3: [Name] - Partially implemented (see issues) - -### Automated Verification Results -✓ Quality gate passes: `mix verify` -✓ Tests pass: `mix test` -✗ Lint warnings: 3 (see output) - -### Code Review Findings - -#### Matches Plan: -- Database migration correctly adds [table] -- API endpoints implement specified methods -- Error handling follows plan - -#### Deviations from Plan: -- Used different variable names in [file:line] -- Added extra validation in [file:line] (improvement) - -#### Potential Issues: -- Missing index on foreign key could impact performance -- No rollback handling in migration - -### Manual Testing Required: -1. UI functionality: - - [ ] Verify [feature] appears correctly - - [ ] Test error states with invalid input - -2. Integration: - - [ ] Confirm works with existing [component] - - [ ] Check performance with large datasets - -### Recommendations: -- Address linting warnings before merge -- Consider adding integration test for [scenario] -- Document new API endpoints -``` - -## Working with Existing Context - -If you were part of the implementation: -- Review the conversation history -- Check your todo list for what was completed -- Focus validation on work done in this session -- Be honest about any shortcuts or incomplete items - -## Important Guidelines - -1. **Report all deviations from the plan.** Flag severity where useful, but don't pre-filter — the reader triages what matters - -## Validation Checklist - -- [ ] Code follows existing patterns -- [ ] No regressions introduced -- [ ] Error handling is robust -- [ ] Documentation updated if needed - -## Relationship to Other Commands - -Recommended workflow: -1. `/implement-plan` - Execute the implementation -2. `/commit` - Create atomic commits for changes -3. `/validate-plan` - Verify implementation correctness - -The validation works best after commits are made, as it can analyze the git history to understand what was implemented. diff --git a/.claude/rules/docs-style.md b/.claude/rules/docs-style.md new file mode 100644 index 00000000000..c0117297268 --- /dev/null +++ b/.claude/rules/docs-style.md @@ -0,0 +1,60 @@ +--- +paths: + - "/ADAPTORS.md" + - "/CHANGELOG.md" + - "/DEPLOYMENT.md" + - "/PROVISIONING.md" + - "/README.md" + - "/RUNNINGLOCAL.md" + - "/SECURITY.md" + - "/WORKERS.md" + - "tooling/**/README.md" +--- + +# User-facing docs + +The root `*.md` files ship in exdoc. The reader is a person with a job to do: +an operator deploying Lightning, or a developer running it locally. They are +not here to learn how the code is organised. Write for the job. + +## What goes in + +- Lead each section with the action or setting, then at most two sentences on + why. If the reader can act after the first line, the section is done. +- Order sections by how often someone needs them, common first. +- Use the reader's words for things. "Local adaptors", not "the Local + strategy". The module name can appear once, as a link, for people who want + the code. +- Real commands and real values in fenced blocks. A worked example beats a + description of one. +- Troubleshooting belongs in the guide for the thing that broke, as a short + "if X, check Y" list. + +## What stays out + +- No Overview, Architecture, How it works, History, Background, or Key + concepts sections. Internals live in moduledocs. Link to them. +- No paragraph that could sit unchanged in another project's docs. +- No commit SHAs, PR numbers, line references, or "as of" dates in the prose. + Git has those. +- No repeating a fact that lives in another doc. Link once and stop. + +## Environment variables + +`DEPLOYMENT.md`'s table is the one place every env var gets its one-line +meaning. A guide uses variables in context, in examples, and may explain a +setting at length, but does not build a second table of them. When you add or +rename a variable, the `DEPLOYMENT.md` row and `.env.example` change in the +same commit. + +## Style + +- Sentence-case headings. Plain words. Short sentences. +- No em dashes. No bold for emphasis mid-sentence. +- British English. +- Before finishing, run the `unslop` skill over what you wrote, then reread it + as a newcomer: could you do the task from this page alone, in a minute, with + nothing left over you didn't need? + +`WORKERS.md`'s History section is the pattern to avoid: it explains how the +old runtime worked, which no reader of that page needs. diff --git a/.claude/skills/create-plan/SKILL.md b/.claude/skills/create-plan/SKILL.md new file mode 100644 index 00000000000..a8e1e208ebd --- /dev/null +++ b/.claude/skills/create-plan/SKILL.md @@ -0,0 +1,106 @@ +--- +name: create-plan +description: Turn a feature request, issue file, or Linear ticket into a phased implementation plan that /implement-plan executes. The lead session steers and decides, cheap agents gather facts, and an independent reviewer checks the draft against the code before it is final. Usage /create-plan [issue path | Linear ID | one-line brief] +disable-model-invocation: false +--- + +# Create plan + +You are the lead. You hold the design, make the recommendations, and talk to +the user. Workers read the codebase and return conclusions; you read only what +the user names and what workers hand back. Protect this context window. + +## Roles + +| Job | Who | Model | +|---|---|---| +| Steer, decide, write the plan, talk to the user | this session | whatever the session is running | +| Find files; find `.context/` docs | `codebase-locator`, `context-locator` | haiku (pinned in agent) | +| Explain how a component works; find a pattern to copy | `codebase-analyzer`, `codebase-pattern-finder` | sonnet (pinned) | +| Read `.context/` history | `context-analyzer` | sonnet (pinned) | +| Library docs, prior art | `web-search-researcher`, only when the user asks | sonnet (pinned) | +| Generate alternatives when the approach is open | `idea-machine` | pass `model: sonnet` | +| Review the draft against the code | `general-purpose` | pass `model: opus` | + +`/model` changes this session only. An agent whose file does not pin a model +gets an explicit `model:` on every dispatch. + +## Process + +### 1. Load the brief + +- `$ARGUMENTS` is a path: read it in full. A Linear ID (`ABC-123`): fetch the + issue and its comments. Empty: ask in one line what we are planning. +- Read every file the user names yourself, in full. +- In one message, dispatch `codebase-locator` and `context-locator` with the + brief. Then send `codebase-analyzer` at each component the change touches; + one agent per area, in parallel. Ask each for conclusions with `file:line` + citations, not file dumps. +- `.context/` is large and rarely pruned. Send `context-analyzer` only at the + hits whose date is close to the code they describe, or that record a decision + rather than an implementation. Treat anything it marks superseded as history: + it explains why, it does not describe what is there now. Never copy an older + plan's approach without re-verifying against the code. + +### 2. Resolve open questions + +- **Facts are your job.** Never ask the user something an agent can look up. +- Present every open **decision** at once, numbered, each with your + recommended answer and a one-line why. Ask again when new decisions appear. + Stop when there are none left. +- When the user states a fact about the code, verify it with an agent before you + build on it. Say so if it does not hold. +- When the approach is genuinely open, dispatch `idea-machine` and bring back + two or three options with a recommendation. Do not present a survey. + +### 3. Structure sign-off + +Show, in plain prose: the goal; what we are not doing; the **seams** where +tests will go (the highest existing seam that proves the behaviour, per +`/tdd`); the phases as one-liners, each naming its implementation agent. One +approval gate. Write no detail before it. + +### 4. Write the plan + +- Path: `.context/shared/plans/YYYY-MM-DD[-XXXX]-slug.md`, `XXXX` the ticket + number when there is one. Template: [plan-template.md](plan-template.md). +- Phases are **tracer bullets**: each lands a thin working slice through every + layer it touches, with its tests, sized to one fresh context window. + Exception: a wide mechanical refactor goes expand, migrate in batches, + contract, so CI stays green throughout. +- Each phase names an **Implementation Agent** from CLAUDE.md §Available + Agents and the model it should run on. `/implement-plan` starts a fresh + agent per phase. +- Automated success criteria are commands that exist in this repo. Manual + criteria are listed separately. +- The CHANGELOG review is a checklist item in the last phase, not a footnote. +- No open questions survive into the file. + +### 5. Independent review + +Dispatch one fresh `general-purpose` agent, `model: opus`, with the plan path +and this checklist: every cited `file:line` exists and says what the plan +claims; each phase depends only on earlier phases; every automated criterion +is a runnable command in this repo; nothing in "What we're not doing" is +needed by a phase; each phase fits one context window. Findings only, no +edits. Fix what holds. Where you disagree, keep your version and say why. + +### 6. Hand back + +Give the user the path, the phase list, and any reviewer finding you rejected. +Iterate until the user is happy. If the brief came from Linear, offer to attach +the plan to the issue; after any Linear write, re-fetch to confirm it landed. End +with the next step: `/implement-plan `. + +## Scaling up + +When the change spans many areas, offer a Workflow and run it only if the user +opts in: locate (haiku) → analyse each area in parallel (sonnet) → you +synthesise and write → review (opus). Load `workflow-authoring` before +writing the script. + +## Rules + +- You grep once at most; the second search is a locator's job. +- Recommend, do not survey. A choice you can default, you default and say so. +- Plain language to the user. The plan path is the only file path in your prose. diff --git a/.claude/skills/create-plan/plan-template.md b/.claude/skills/create-plan/plan-template.md new file mode 100644 index 00000000000..53163d892e7 --- /dev/null +++ b/.claude/skills/create-plan/plan-template.md @@ -0,0 +1,69 @@ +# [Feature] Implementation Plan + +## Overview + +[What we are building and why, two or three sentences.] + +## Current State + +[What exists now, what is missing, constraints found. Cite `file:line`.] + +## Desired End State + +[What is true when the plan is done, and how we can tell.] + +## What We're NOT Doing + +- [Out-of-scope item] + +## Approach + +[The strategy in a paragraph, and why it beat the alternatives considered.] + +## Test Seams + +[The seams agreed at sign-off: which public interface each phase proves its +behaviour at, and the existing test harness that exercises it. `/tdd` runs here.] + +## Phase 1: [Name] + +**Implementation Agent**: `[agent-type]` on `[model]` + + +[What this slice delivers end to end.] + +### Changes + +#### 1. [Component] +**File**: `path/to/file.ext` +**Changes**: [Summary; code only where it encodes a real decision.] + +### Success Criteria + +#### Automated +- [ ] `mix test path/to/test.exs` +- [ ] `mix verify` +- [ ] `cd assets && npm test` (when JS changes) + +#### Manual +- [ ] [What a person checks, and how] + +--- + +## Phase N: [Name] + +[As above. The last phase also carries:] + +- [ ] CHANGELOG reviewed: Keep-a-Changelog entry with issue/PR link for any + user-visible change; broaden an existing entry over adding a second; + "no change needed" is a valid outcome + +## Migration Notes + +[Only when existing data or deployed systems need handling.] + +## References + +- Issue: [Linear ID or `.context/shared/issues/issue-XXXX.md`] +- Research: `.context/shared/research/...` +- Pattern followed: `file:line` diff --git a/.claude/skills/docs-review/SKILL.md b/.claude/skills/docs-review/SKILL.md new file mode 100644 index 00000000000..91b48e1272a --- /dev/null +++ b/.claude/skills/docs-review/SKILL.md @@ -0,0 +1,58 @@ +--- +name: docs-review +description: Review a user-facing doc the way a reader would. Two fresh agents, one trying to do real tasks from the page alone, one cutting words. Usage /docs-review [task; task; ...] +disable-model-invocation: true +--- + +# Docs review + +`$ARGUMENTS` is a doc path, optionally followed by a semicolon-separated list +of reader tasks. If no tasks are given, derive three from the doc's headings +before you start and state them in the report. + +Run both agents in parallel, `subagent_type: general-purpose`, +`model: sonnet`. Neither gets this conversation's context. That is the point: +they read the page cold, like a user. + +## Agent 1: newcomer test + +Prompt, with the path and tasks filled in: + +> Read `` and nothing else. Do not open source code or other docs unless +> the page links you there, and if it does, note that you had to leave. +> +> For each task below, describe in two or three lines the steps you would take +> using only what the page told you. Then say honestly which of these happened: +> you found it within the first screen; you had to hunt; you had to guess; you +> could not do it from this page. Quote the sentence that finally answered +> you, or say none did. +> +> Tasks: +> +> +> Then list, as one line each, any paragraph you skipped because it wasn't +> helping you do anything, and any sentence you had to read twice. +> +> Report under 40 lines. No praise, no suggestions for new sections. + +## Agent 2: cut pass + +Prompt: + +> Read ``. Produce a version at most three quarters of the length that +> loses no command, setting, value, or instruction. You may cut explanation, +> repetition, history, hedging, and anything a reader could not act on. Keep +> the headings unless a section empties out, in which case remove it and say +> so. Also apply the `unslop` skill's checklist while you cut. +> +> Write the result to `.scratch/.cut.md`. Then report, in under 20 +> lines: word count before and after, each section you shortened by more than +> half with a one-line reason, and any sentence you were unsure was safe to +> cut. Do not edit the original. + +## Report back + +Relay both results in plain prose. Lead with the tasks the newcomer could not +do or had to guess at. Then the paragraphs both agents flagged, since those +are the surest cuts. Point at the `.cut.md` file for the trimmed draft and +leave the decision to Stu. Do not apply changes yourself. diff --git a/.claude/skills/implement-plan/SKILL.md b/.claude/skills/implement-plan/SKILL.md new file mode 100644 index 00000000000..0b094269698 --- /dev/null +++ b/.claude/skills/implement-plan/SKILL.md @@ -0,0 +1,26 @@ +--- +name: implement-plan +description: Execute a plan written by /create-plan, one phase at a time, on the agent and model each phase names. Usage /implement-plan +disable-model-invocation: true +--- + +# Implement plan + +Read the plan at `$ARGUMENTS` in full. If empty, ask for the path. Check +`git log` for phases already landed before starting. + +Plans from `/create-plan` carry three things a fresh session should honour: + +- **Test Seams** names where tests go. Use `/tdd` at those seams only; a test + at a seam the plan does not name needs the user's agreement first. +- Each phase names an **Implementation Agent** and model. Dispatch one fresh + agent per phase with that type and an explicit `model:`, handing it the plan + path and phase number rather than a paraphrase. Phases run in order. +- **Success Criteria** split automated from manual. The phase agent runs the + automated ones; run them again yourself before moving on. Manual ones are + reported to the user at the end, never ticked on their behalf. + +When the code disagrees with the plan, stop and ask before adapting. + +When every phase passes: `/code-review`, fix what holds, then `/commit`. +Report any manual criteria still outstanding. diff --git a/.claude/skills/reshape-pass/SKILL.md b/.claude/skills/reshape-pass/SKILL.md new file mode 100644 index 00000000000..1ef0d42dbb9 --- /dev/null +++ b/.claude/skills/reshape-pass/SKILL.md @@ -0,0 +1,205 @@ +--- +name: reshape-pass +description: + Review the code a branch adds or changes for design that is working around + itself, and reshape it: shapes converted back and forth, guards for cases a + better shape makes impossible, one thing under several names, comments that + explain a design fact as a workaround. Use before merging a long-lived + feature branch, or when a nearly-done implementation feels like it was fitted + around what was there. Not a bug hunt and not a comment pass. +argument-hint: + '[path or module to scope to] [review notes file] [--fix to apply as ordered + commits]' +disable-model-invocation: true +--- + +Review the code the current branch adds or changes, and find where the design +is working around its own shape instead of changing it. + +The tell is accommodation. Code written to fit what was already there, rather +than asking whether what was there should still exist. Each accommodation is +small and defensible on its own. Together they mean the branch merges already +needing a refactor. + +**Default is propose-only.** Present the findings and wait; touch nothing. With +`--fix` (or `--apply`), apply the changes to the working tree, one commit per +finding in the order given, using the project's commit convention or skill, and +show what changed. + +**Never post anything.** This skill edits files in the working tree and nothing +else. No PR comments, no pushes, no public surface. Pass this constraint to +every subagent. + +## Three roles + +- **Mapper**: a subagent that builds the map of the area and returns a report. +- **You**: read the map and the code it points at, form and order the + findings, and in `--fix` mode make the changes. +- **Verifier**: a fresh subagent that did not see the findings being formed, + running the checks under "Verify" against the real code. + +The mapper never judges and the verifier never edits. Findings are yours. + +## Read the local rules and the plan first + +Read the project's CLAUDE.md and anything it points at for module layout, +naming and testing conventions. A project rule beats this skill. Respect any +boundary the project deliberately holds (a facade, a context module, a strategy +behaviour): reshape inside it, and say so explicitly if a finding would move +something across it. + +If a plan, spec or ADR exists for the branch, read it before mapping. It says +what the shape was meant to be, and what was deliberately deferred. A finding +that contradicts the plan has to say so. A deferral defers work, not a bug: if +a bug sits on deferred ground, find the fix that doesn't need the deferred +work, and if there isn't one, report the bug as blocking and say why. + +## Scope + +The area is what you read; the diff is what makes a finding in scope. + +- Diff against the merge-base with the default branch, plus staged and unstaged + changes. On the default branch with no branch diff, use the uncommitted + changes; if none, ask what to review. +- If a path or module was given, restrict findings to that area, but still read + its callers and callees so a reshape doesn't break a consumer you never + opened. +- Pre-existing code is in scope when the branch built on it in a way that + exposed the problem. "It was there before" is not a defence: nothing on a + branch is in main yet, and the branch is the cheapest moment to fix it. +- If the user supplied review notes or a list of smells, treat them as seed + findings. Map around them, and in the report say which the map confirmed, + which it reframed (the smell was real, the cause was elsewhere), and what it + found that the notes didn't. + +## Map before you judge + +Do not start from the diff hunks. Start from the shapes. The mapper returns a +report of at most a few hundred lines, quoting only converters, guards and +cache writes, never whole files: + +- Every struct, typespec, Ecto schema or type alias in the area: fields, owner, + who constructs it, who consumes it. +- The data flow across module boundaries, and every point where one shape is + converted into another, quoted verbatim. +- Every place that strips `__struct__`, calls `Map.from_struct`, builds one + struct from another with `struct/2`, or pattern-matches a map and a struct + for the same logical data. +- Every guard for a nil, empty or missing value, and where that value comes + from. +- Every cache write: what shape goes in, what a fresh read of the same record + returns, and what invalidates the key. +- Public functions in the area and which of them anything outside actually + calls. +- Tests the branch added or changed, and the fixture state each relies on + (factory defaults especially). + +Read the map, then read the code the map points at. Only then form findings. + +Write the map to disk (the scratchpad, or the branch's scratch directory if it +has one) and give the path. Findings go in the same file. That file is the +handoff if the work continues in another session. + +## Bugs come first + +This is not a bug hunt, but mapping shapes finds bugs: a cache that pins a +transient failure, a guard that catches the wrong case, a test that passes for +the wrong reason. A bug is blocking if you would not ship main with it. Report +blocking bugs before any design finding, and in `--fix` mode commit each one +first and separately. + +## What to look for + +Each of these is a place where a shape should change and code should +disappear. The fix is always upstream of the symptom. + +- **Two shapes for one thing at a boundary.** A function that accepts both a + lean map and a full struct for the same logical record, and normalises them + with `Map.delete(:__struct__)`, `Map.get/3` defaults, or `struct/2` that + silently drops fields. Find the caller that feeds two shapes and make it + feed one. The converter becomes a field copy or disappears. +- **One record under several names.** Three typespecs projecting the same row, + two modules declaring a type with the same name and different fields. Decide + what the record is as the rest of the app sees it, name it once, and make the + other shapes a render step on top. +- **Downstream guards for upstream facts.** A `{:ok, nil}` branch, a + `when not is_nil` in a consumer, a default for a missing key. Ask what + produced the nil. If the producer is in the area, make it produce a + well-formed value (an empty schema, an empty list, an error tuple) and delete + the guard. A producer that can return nil pushes the same guard into every + consumer it will ever have. +- **Comments that explain the data instead of the code.** A comment saying + "X may be nil even when Y, because Z" is a design fact stated as a + workaround. It marks the spot where the shape should change. Fix the shape, + then delete the comment. +- **The same normalisation in several places.** Encoding, decoding, trimming, + defaulting done at three layers for the same field because each layer did + not trust the one before. Pick the layer that owns the contract, tighten the + contract in its typespec, and delete the rest. +- **A concept without a home.** The branch introduced a responsibility (a + cache, a projection, a lifecycle, a permission) and spread it across + existing modules as parameters, flags and helper functions instead of + naming it. The fix is bigger than the ticket said. Say so, cost it, and + propose the module or struct it should become. +- **Dead surface.** A struct declared and never constructed, a public function + no one outside calls, an option no caller passes. Delete it. +- **A cache that can disagree with its source.** A cached value shaped + differently from a fresh read of the same record, or a cache key with no + invalidation path when the source changes. Report it even when fixing it is + out of scope. + +## What to leave alone + +Never touch: + +- A boundary the project deliberately holds. Reshape inside it. +- Working code the branch did not build on and that no finding depends on. +- A guard at a trust boundary (user input, external API, deserialised data). + Those defend against the outside world, not against the code's own shape. +- Anything a `# SAFETY:` or "we deliberately do NOT" comment protects. + +When unsure whether a reshape is safe, propose it with the doubt stated and let +the user decide. A wrong refactor on a nearly-done branch costs more than the +accommodation it removes. + +## Order the findings + +Findings depend on each other. A single projection function removes the need +for a converter, which removes a default, which removes a comment. Order the +list so each step shrinks the ones after it, and say which findings a given +step makes unnecessary. + +Split the list in two. **Before merge**: anything that changes behaviour, +touches a cache or a producer's contract, or that a later finding depends on. +**Can follow**: renames, type consolidation and deletions with no behaviour +change and nothing depending on them. Say why each item landed where it did. + +## Verify before declaring done + +The verifier checks, against the real code: + +- For every deleted guard: which producer now guarantees the value, and does + every path into that producer honour it? Trace them. +- For every merged shape: does anything outside the area pattern-match on the + old shape? Grep the whole codebase, including tests. +- For every converter removed: is the cached value still identical to a fresh + read? +- For every filter or guard added, moved or removed: do the tests for the + excluded case still exclude it for the stated reason, or does a different + filter now get there first? A fixture with a nil field can make a + deprecation test pass without ever testing deprecation. +- If any finding changes user-visible behaviour, say so and check whether the + CHANGELOG needs a line. Reshapes usually don't; a producer that stops + returning nil sometimes does. +- Run the tests for the area and any consumer touched. In `--fix` mode, run + them after each commit, not only at the end. + +## Output + +Blocking bugs first. Then findings in the order they should be applied, split +into before merge and can follow. For each: the smell (from the list above), +the files and functions involved, what the shape should become, what code the +change deletes, and which later findings it makes unnecessary. If review notes +were supplied, say what happened to each. Note anything left alone out of +caution and why. End with the totals, the path of the map file, and in propose +mode how to apply (rerun with `--fix`, or pick items by hand). diff --git a/.claude/skills/tdd/SKILL.md b/.claude/skills/tdd/SKILL.md new file mode 100644 index 00000000000..9a8d8143b19 --- /dev/null +++ b/.claude/skills/tdd/SKILL.md @@ -0,0 +1,68 @@ +--- +name: tdd +description: Test-driven development. Use when the user wants to build features or fix bugs test-first, mentions "red-green-refactor", or wants integration tests. Covers both stacks — ExUnit and Vitest/Playwright. +--- + +# Test-Driven Development + +TDD is the red → green loop. This skill is the reference that makes that loop produce tests worth keeping: what a good test is, where tests go, the anti-patterns, and the rules of the loop. Every section applies on every cycle — consult them before and during the loop, not after. + +When exploring the codebase, match the domain language already in use so test names and interface vocabulary line up with it: the context modules in `lib/lightning/` are the glossary (workflow, job, trigger, edge, snapshot, run, step, work order — these are distinct things, don't blur them). CLAUDE.md §Key Contexts is the map. + +## What a good test is + +Tests verify behavior through public interfaces, not implementation details. Code can change entirely; tests shouldn't. A good test reads like a specification — "user can checkout with valid cart" tells you exactly what capability exists — and survives refactors because it doesn't care about internal structure. + +`.claude/guidelines/testing-essentials.md` is the reference for what a keepable test looks like here. Two rules from it that bear on every cycle: + +- **Group related assertions.** Multiple assertions about one operation belong in one test. One-property-per-test produces files where the shape is buried in noise. In Elixir, pattern-match the whole struct instead of asserting field by field. +- **400 lines is the ceiling for a test file.** Past that, consolidate and extract setup into helpers. + +## Seams — where tests go + +A **seam** is the public boundary you test at: the interface where you observe behavior without reaching inside. Tests live at seams, never against internals. + +**Test only at pre-agreed seams.** Before writing any test, write down the seams under test and confirm them with the user. No test is written at an unconfirmed seam. You can't test everything — agreeing the seams up front is how testing effort lands on the critical paths and complex logic instead of every edge case. + +Ask: "What's the public interface, and which seams should we test?" + +The seams this codebase actually offers, and what you test them through: + +| Seam | Test through | Case module | +| --- | --- | --- | +| Context module public API (`Lightning.Workflows`, `Lightning.Runs`, …) | direct function calls | `DataCase` | +| HTTP endpoint / controller | `Phoenix.ConnTest` | `ConnCase` | +| LiveView page | `Phoenix.LiveViewTest` — render and interact, don't call `handle_event` | `ConnCase` | +| Phoenix channel | join and push | `ChannelCase` | +| Oban worker | `perform/1` with a real job struct | `DataCase` | +| React store | `getSnapshot()` and the store's commands | Vitest | +| React component | React Testing Library, through the DOM | Vitest | +| Whole user journey (LiveView + React + DB) | Playwright | `npm run test:e2e` | + +Schema modules and `changeset/2` are a seam too, but a shallow one — prefer testing validation through the context function that calls it, unless the changeset is itself complex enough to earn its own test. + +## Mocking + +There is no repo guideline for this, so: Mox is the default and by a wide margin the house style. Grep `test/` for `Mox`, `Mimic` and `with_mock` to see the current split. + +- **Mox** for collaborators behind a behaviour that gets injected — HTTP via `Tesla.Adapter`, the extension hooks, `Lightning.Config`. Mocks are declared once in `test/test_helper.exs`; add new ones there. Stays `async: true`. +- **Mimic** only when the collaborator genuinely can't be injected: `File`, `IO`, `:hackney` are already `Mimic.copy`'d in `test/test_helper.exs`. +- **`:mock`'s `with_mock`** — don't add new uses. It swaps the module globally, so almost every file using it runs `async: false`. +- **Bypass** for a real HTTP server when you're testing the request that goes out on the wire (`test/support/bypass_helpers.ex`). +- **Stub modules** over expectation-based mocks when you only need a canned answer and don't care that the call happened: see `test/support/stub_rate_limiter.ex` and `stub_usage_limiter.ex`. + +Mocking your own module is the smell, not the tool. If a test needs Mimic to reach past a boundary you own, the boundary is in the wrong place. + +## Anti-patterns + +- **Implementation-coupled** — mocks internal collaborators, tests private methods, or verifies through a side channel (querying the database instead of using the interface). The tell: the test breaks when you refactor but behavior hasn't changed. Local forms: a `Repo.get` assertion where the context function would have told you, or reading a Y.Doc's internal arrays where the store's snapshot would have. The one sanctioned exception is counting store `notify()` calls, where the count *is* the behaviour — see testing-essentials.md §Test behavior not implementation. +- **Tautological** — the assertion recomputes the expected value the way the code does (`expect(add(a, b)).toBe(a + b)`, a snapshot derived by hand the same way, a constant asserted equal to itself), so it passes by construction and can never disagree with the code. Expected values must come from an independent source of truth — a known-good literal, a worked example, the spec. +- **Horizontal slicing** — writing all tests first, then all implementation. Bulk tests verify _imagined_ behavior: you test the _shape_ of things rather than user-facing behavior, the tests go insensitive to real changes, and you commit to test structure before understanding the implementation. Work in **vertical slices** instead — one test → one implementation → repeat, each test a **tracer bullet** that responds to what the last cycle taught you. + +## Rules of the loop + +- **Red before green.** Write the failing test first, then only enough code to pass it. Don't anticipate future tests or add speculative features. +- **One slice at a time.** One seam, one test, one minimal implementation per cycle. +- **Run only the slice.** `mix test path/to/test.exs:42` or `npm test -- useSession.test.ts` (from `assets/`). Full-suite runs belong at the end of the session, not inside the loop. +- **Green means green.** `warnings_as_errors: true`, so a warning is a red. Don't move on with one outstanding. +- **Refactoring is not part of the loop.** It belongs to the review stage — `/code-review` for defects, `/simplify` for cleanup — not the red → green implementation cycle. diff --git a/CHANGELOG.md b/CHANGELOG.md index 39f5e78fe34..2a8ea7a438c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,11 @@ and this project adheres to does not normalise anything today, but #4577 adds it on every name, so the runtime moves first. +- Lightning now keeps its own adaptor registry instead of fetching the list from + npm at startup, so new adaptors and versions show up without a rebuild or + redeploy. See [ADAPTORS.md](ADAPTORS.md). + [#4801](https://github.com/OpenFn/lightning/pull/4801) + ### Removed - The AI assistant's "Send code" tickbox. The assistant reads your workflow to diff --git a/CLAUDE.md b/CLAUDE.md index 91d38c01363..3fecbfb1879 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -159,8 +159,8 @@ External Node.js workers (@openfn/ws-worker) execute JavaScript jobs: Claude Code injects every agent's name and description each session, so the roster is not restated here; the agent files themselves are in -`.claude/agents/`. Command files (`create-plan.md`, `implement-plan.md`, -`research-codebase.md`) cross-ref this section rather than repeating it. +`.claude/agents/`. The `create-plan` skill and the `research-codebase` command +cross-ref this section rather than repeating it. One convention that no agent file carries: dispatch `web-search-researcher` on request, not by default. From 205bf0ead38b5d9dc349f1afaaa18aa4fb576427 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 10 Sep 2026 09:13:53 +0200 Subject: [PATCH 19/37] Assert the save timeout in GitHubSyncModal's push expectations - save_workflow and save_and_sync now push a timeout as a third argument; the four `save_and_sync` expectations pinned the call to two and failed --- .../components/GitHubSyncModal.test.tsx | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/assets/test/collaborative-editor/components/GitHubSyncModal.test.tsx b/assets/test/collaborative-editor/components/GitHubSyncModal.test.tsx index c549c20c919..573120b8c8a 100644 --- a/assets/test/collaborative-editor/components/GitHubSyncModal.test.tsx +++ b/assets/test/collaborative-editor/components/GitHubSyncModal.test.tsx @@ -534,7 +534,8 @@ describe('GitHubSyncModal - Save & Sync Action', () => { commit_message: expect.stringContaining( 'initiated a sync from Lightning' ), - }) + }), + expect.any(Number) ); }); }); @@ -679,7 +680,8 @@ describe('GitHubSyncModal - Save & Sync Action', () => { 'save_and_sync', expect.objectContaining({ commit_message: 'Test commit message', - }) + }), + expect.any(Number) ); }); }); @@ -727,7 +729,11 @@ describe('GitHubSyncModal - Keyboard Shortcuts', () => { await user.type(textarea, '{Control>}{Enter}{/Control}'); await waitFor(() => { - expect(pushSpy).toHaveBeenCalledWith('save_and_sync', expect.any(Object)); + expect(pushSpy).toHaveBeenCalledWith( + 'save_and_sync', + expect.any(Object), + expect.any(Number) + ); }); }); @@ -768,7 +774,11 @@ describe('GitHubSyncModal - Keyboard Shortcuts', () => { await user.type(textarea, '{Meta>}{Enter}{/Meta}'); await waitFor(() => { - expect(pushSpy).toHaveBeenCalledWith('save_and_sync', expect.any(Object)); + expect(pushSpy).toHaveBeenCalledWith( + 'save_and_sync', + expect.any(Object), + expect.any(Number) + ); }); }); From 55c83bb5188c5627dbc96ec86f8eb5ae324447a8 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 10 Sep 2026 09:14:01 +0200 Subject: [PATCH 20/37] Pin the locale for the JS test run - Components format numbers and dates with the viewer's own locale, so a test asserting a formatted literal depended on the developer's LANG: ChatInput's counter reads "9,600" under en-US and "9 600" under en-ZA - LC_ALL set in vitest.config.ts rather than the npm scripts, so `npx vitest` behaves the same and the reason can be written down - A note at the one assertion that relies on it, warning against the tempting fix of hardcoding a locale in the component --- .../collaborative-editor/components/ChatInput.test.tsx | 6 ++++++ assets/vitest.config.ts | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/assets/test/collaborative-editor/components/ChatInput.test.tsx b/assets/test/collaborative-editor/components/ChatInput.test.tsx index d93e8295a84..7c6ef64935c 100644 --- a/assets/test/collaborative-editor/components/ChatInput.test.tsx +++ b/assets/test/collaborative-editor/components/ChatInput.test.tsx @@ -292,6 +292,12 @@ describe('ChatInput', () => { render(); await type('x'.repeat(9600)); + // The comma grouping comes from the locale vitest.config.ts pins for the + // run, not from the component: ChatInput formats with the viewer's own + // locale, so a real en-ZA or de-DE user sees "9 600" / "9.600". If this + // assertion fails on your machine, the locale pin is not in effect — + // don't pass a fixed locale to `toLocaleString()` in the component to + // make it pass. expect(screen.getByTestId('chat-input-length')).toHaveTextContent( '9,600 / 10,000' ); diff --git a/assets/vitest.config.ts b/assets/vitest.config.ts index 0172a27b00a..fe04ee9ccf3 100644 --- a/assets/vitest.config.ts +++ b/assets/vitest.config.ts @@ -4,6 +4,14 @@ import react from '@vitejs/plugin-react'; import tsconfigPaths from 'vite-tsconfig-paths'; import { defineConfig } from 'vitest/config'; +// Number and date formatting in the app uses the viewer's own locale (bare +// `toLocaleString()`), which is correct for users but makes any test asserting a +// formatted literal depend on the developer's `LANG`. Pin one locale for the run +// so the suite is deterministic; set here rather than in the npm scripts so a +// bare `npx vitest` behaves the same. Must be set before the worker processes +// fork, since Node resolves its default locale at startup. +process.env.LC_ALL = 'en_US.UTF-8'; + export default defineConfig({ plugins: [react(), tsconfigPaths()], define: { From 72a0196662efd7cb100f77dc039b7e7de400795d Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 10 Sep 2026 09:14:15 +0200 Subject: [PATCH 21/37] Destroy session stores in tests, drop the max-listeners cap - PhoenixChannelProvider registers a process 'exit' handler that only destroy() removes, so a test leaving a session initialised leaked one listener plus a Y.Doc, awareness and channel per test; the worst files reached 50 and 34 against a cap of 24 - createTestSessionStore added to sessionStoreHelpers: createSessionStore plus onTestFinished(destroy). The 10 raw call sites and the trigger inspector harness use it - process.setMaxListeners(24) deleted from the test setup, so the next leak surfaces as a warning instead of being absorbed. Node writes that warning to raw stderr, which is why it never reached the junit report --- assets/test/_setup.ts | 4 ---- .../collaborative-editor/__helpers__/index.ts | 1 + .../__helpers__/sessionStoreHelpers.ts | 20 ++++++++++++++++++- .../__helpers__/triggerInspectorHelpers.tsx | 8 +++++++- .../components/GitHubSyncModal.test.tsx | 4 ++-- .../components/ReadOnlyWarning.test.tsx | 4 ++-- .../components/inspector/EdgeForm.test.tsx | 4 ++-- .../inspector/EdgeInspector.test.tsx | 8 +++++--- .../inspector/JobInspector.test.tsx | 8 +++++--- .../inspector/TriggerInspector.test.tsx | 8 +++++--- .../hooks/useRunRetry.test.tsx | 8 ++++---- .../hooks/useWorkflowReadOnly.test.tsx | 12 +++++------ .../collaborative-editor/useAdaptors.test.tsx | 4 ++-- .../collaborative-editor/useSession.test.tsx | 6 +++--- 14 files changed, 63 insertions(+), 36 deletions(-) diff --git a/assets/test/_setup.ts b/assets/test/_setup.ts index db2de43348b..5de5e96e935 100644 --- a/assets/test/_setup.ts +++ b/assets/test/_setup.ts @@ -4,10 +4,6 @@ import { enableMapSet } from 'immer'; // Enable Immer MapSet plugin for tests that use Set in Immer state enableMapSet(); -// Increase max listeners to avoid warning during test runs -// This is safe for tests as multiple test files add cleanup handlers -process.setMaxListeners(24); - // Suppress debug logs during tests console.debug = () => {}; diff --git a/assets/test/collaborative-editor/__helpers__/index.ts b/assets/test/collaborative-editor/__helpers__/index.ts index c99d4833a47..f66d5ed2313 100644 --- a/assets/test/collaborative-editor/__helpers__/index.ts +++ b/assets/test/collaborative-editor/__helpers__/index.ts @@ -53,6 +53,7 @@ export { // Session store helpers export { createMockSocket, + createTestSessionStore, triggerProviderSync, triggerProviderStatus, applyProviderUpdate, diff --git a/assets/test/collaborative-editor/__helpers__/sessionStoreHelpers.ts b/assets/test/collaborative-editor/__helpers__/sessionStoreHelpers.ts index fe9e7dc7810..eb414f594f6 100644 --- a/assets/test/collaborative-editor/__helpers__/sessionStoreHelpers.ts +++ b/assets/test/collaborative-editor/__helpers__/sessionStoreHelpers.ts @@ -15,8 +15,9 @@ import { Doc as YDoc, applyUpdate, encodeStateAsUpdate } from 'yjs'; import type { PhoenixChannelProvider } from 'y-phoenix-channel'; -import { expect } from 'vitest'; +import { expect, onTestFinished } from 'vitest'; +import { createSessionStore } from '../../../js/collaborative-editor/stores/createSessionStore'; import type { SessionStore } from '../../../js/collaborative-editor/stores/createSessionStore'; import type { SessionState } from '../../../js/collaborative-editor/stores/createSessionStore'; @@ -29,6 +30,23 @@ import { // Re-export commonly used utilities export { waitForAsync }; +/** + * A session store that tears itself down when the test ends. + * + * PhoenixChannelProvider registers a `process.on('exit')` handler that only + * `destroy()` removes, so a store left initialised leaks one listener — plus a + * Y.Doc, awareness and channel — per test. + */ +export function createTestSessionStore(): SessionStore { + const store = createSessionStore(); + + onTestFinished(() => { + store.destroy(); + }); + + return store; +} + /** * Creates a mock Phoenix socket for session store tests * diff --git a/assets/test/collaborative-editor/__helpers__/triggerInspectorHelpers.tsx b/assets/test/collaborative-editor/__helpers__/triggerInspectorHelpers.tsx index 6d2900db9f5..443ea66b05d 100644 --- a/assets/test/collaborative-editor/__helpers__/triggerInspectorHelpers.tsx +++ b/assets/test/collaborative-editor/__helpers__/triggerInspectorHelpers.tsx @@ -16,7 +16,7 @@ import type React from 'react'; import { act } from 'react'; -import { vi } from 'vitest'; +import { onTestFinished, vi } from 'vitest'; import { LiveViewActionsProvider } from '../../../js/collaborative-editor/contexts/LiveViewActionsContext'; import { KeyboardProvider } from '../../../js/collaborative-editor/keyboard/KeyboardProvider'; @@ -131,6 +131,12 @@ export async function createTriggerTestHarness( { connect: true } ); + // PhoenixChannelProvider registers a process 'exit' handler that only + // destroy() removes, so an undestroyed session leaks one per test. + onTestFinished(() => { + sessionStore.destroy(); + }); + // 2. Allow the mock PhoenixChannelProvider to create its channel. await new Promise(resolve => setTimeout(resolve, 50)); diff --git a/assets/test/collaborative-editor/components/GitHubSyncModal.test.tsx b/assets/test/collaborative-editor/components/GitHubSyncModal.test.tsx index 573120b8c8a..a116b633f75 100644 --- a/assets/test/collaborative-editor/components/GitHubSyncModal.test.tsx +++ b/assets/test/collaborative-editor/components/GitHubSyncModal.test.tsx @@ -24,7 +24,6 @@ import { createAdaptorStore } from '../../../js/collaborative-editor/stores/crea import { createAwarenessStore } from '../../../js/collaborative-editor/stores/createAwarenessStore'; import { createCredentialStore } from '../../../js/collaborative-editor/stores/createCredentialStore'; import { createSessionContextStore } from '../../../js/collaborative-editor/stores/createSessionContextStore'; -import { createSessionStore } from '../../../js/collaborative-editor/stores/createSessionStore'; import { createUIStore } from '../../../js/collaborative-editor/stores/createUIStore'; import { createWorkflowStore } from '../../../js/collaborative-editor/stores/createWorkflowStore'; import type { Session } from '../../../js/collaborative-editor/types/session'; @@ -32,6 +31,7 @@ import { createGithubConnectedContext, createSessionContext, } from '../__helpers__/sessionContextFactory'; +import { createTestSessionStore } from '../__helpers__/sessionStoreHelpers'; import { createMockPhoenixChannel, createMockPhoenixChannelProvider, @@ -58,7 +58,7 @@ function createTestSetup(options: WrapperOptions = {}) { } = options; // Create all stores - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const sessionContextStore = createSessionContextStore(false); const workflowStore = createWorkflowStore(); const adaptorStore = createAdaptorStore(); diff --git a/assets/test/collaborative-editor/components/ReadOnlyWarning.test.tsx b/assets/test/collaborative-editor/components/ReadOnlyWarning.test.tsx index eac3c0092be..77d68405b7e 100644 --- a/assets/test/collaborative-editor/components/ReadOnlyWarning.test.tsx +++ b/assets/test/collaborative-editor/components/ReadOnlyWarning.test.tsx @@ -19,10 +19,10 @@ import { SessionContext } from '../../../js/collaborative-editor/contexts/Sessio import type { StoreContextValue } from '../../../js/collaborative-editor/contexts/StoreProvider'; import { StoreContext } from '../../../js/collaborative-editor/contexts/StoreProvider'; import { createSessionContextStore } from '../../../js/collaborative-editor/stores/createSessionContextStore'; -import { createSessionStore } from '../../../js/collaborative-editor/stores/createSessionStore'; import { createWorkflowStore } from '../../../js/collaborative-editor/stores/createWorkflowStore'; import type { Session } from '../../../js/collaborative-editor/types/session'; import { createSessionContext } from '../__helpers__/sessionContextFactory'; +import { createTestSessionStore } from '../__helpers__/sessionStoreHelpers'; import { createMockURLState, getURLStateMockValue, @@ -61,7 +61,7 @@ function createTestSetup(options: WrapperOptions = {}) { isNewWorkflow = false, } = options; - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const sessionContextStore = createSessionContextStore(isNewWorkflow); const workflowStore = createWorkflowStore(); diff --git a/assets/test/collaborative-editor/components/inspector/EdgeForm.test.tsx b/assets/test/collaborative-editor/components/inspector/EdgeForm.test.tsx index 4eac828e9c2..5dcec2da7e9 100644 --- a/assets/test/collaborative-editor/components/inspector/EdgeForm.test.tsx +++ b/assets/test/collaborative-editor/components/inspector/EdgeForm.test.tsx @@ -25,7 +25,6 @@ import type { CredentialStoreInstance } from '../../../../js/collaborative-edito import { createCredentialStore } from '../../../../js/collaborative-editor/stores/createCredentialStore'; import type { SessionContextStoreInstance } from '../../../../js/collaborative-editor/stores/createSessionContextStore'; import { createSessionContextStore } from '../../../../js/collaborative-editor/stores/createSessionContextStore'; -import { createSessionStore } from '../../../../js/collaborative-editor/stores/createSessionStore'; import type { WorkflowStoreInstance } from '../../../../js/collaborative-editor/stores/createWorkflowStore'; import { createWorkflowStore } from '../../../../js/collaborative-editor/stores/createWorkflowStore'; import type { Session } from '../../../../js/collaborative-editor/types/session'; @@ -33,6 +32,7 @@ import { createMockPhoenixChannel, createMockPhoenixChannelProvider, } from '../../__helpers__/channelMocks'; +import { createTestSessionStore } from '../../__helpers__/sessionStoreHelpers'; import { createMockSocket } from '../../mocks/phoenixSocket'; import { createWorkflowYDoc } from '../../__helpers__/workflowFactory'; @@ -61,7 +61,7 @@ function createWrapper( awarenessStore: AwarenessStoreInstance ): React.ComponentType<{ children: React.ReactNode }> { // Create session store and initialize with mock socket - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const mockSocket = createMockSocket(); sessionStore.initializeSession(mockSocket as any, 'test:room', null, { connect: true, // Ensure connected state diff --git a/assets/test/collaborative-editor/components/inspector/EdgeInspector.test.tsx b/assets/test/collaborative-editor/components/inspector/EdgeInspector.test.tsx index 99fa2abfd91..6d2b03e52cf 100644 --- a/assets/test/collaborative-editor/components/inspector/EdgeInspector.test.tsx +++ b/assets/test/collaborative-editor/components/inspector/EdgeInspector.test.tsx @@ -15,7 +15,6 @@ import { SessionContext } from '../../../../js/collaborative-editor/contexts/Ses import { LiveViewActionsProvider } from '../../../../js/collaborative-editor/contexts/LiveViewActionsContext'; import type { StoreContextValue } from '../../../../js/collaborative-editor/contexts/StoreProvider'; import { StoreContext } from '../../../../js/collaborative-editor/contexts/StoreProvider'; -import { createSessionStore } from '../../../../js/collaborative-editor/stores/createSessionStore'; import type { AdaptorStoreInstance } from '../../../../js/collaborative-editor/stores/createAdaptorStore'; import { createAdaptorStore } from '../../../../js/collaborative-editor/stores/createAdaptorStore'; import type { AwarenessStoreInstance } from '../../../../js/collaborative-editor/stores/createAwarenessStore'; @@ -31,7 +30,10 @@ import { createMockPhoenixChannelProvider, } from '../../__helpers__/channelMocks'; import { createWorkflowYDoc } from '../../__helpers__/workflowFactory'; -import { createMockSocket } from '../../__helpers__/sessionStoreHelpers'; +import { + createMockSocket, + createTestSessionStore, +} from '../../__helpers__/sessionStoreHelpers'; /** * Helper to create and connect a workflow store with Y.Doc @@ -71,7 +73,7 @@ function createWrapper( redirect: vi.fn(), }; - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const mockSocket = createMockSocket(); sessionStore.initializeSession( mockSocket, diff --git a/assets/test/collaborative-editor/components/inspector/JobInspector.test.tsx b/assets/test/collaborative-editor/components/inspector/JobInspector.test.tsx index e9c4e15d53c..ab13bbdc554 100644 --- a/assets/test/collaborative-editor/components/inspector/JobInspector.test.tsx +++ b/assets/test/collaborative-editor/components/inspector/JobInspector.test.tsx @@ -21,7 +21,6 @@ import { SessionContext } from '../../../../js/collaborative-editor/contexts/Ses import { KeyboardProvider } from '../../../../js/collaborative-editor/keyboard'; import type { StoreContextValue } from '../../../../js/collaborative-editor/contexts/StoreProvider'; import { StoreContext } from '../../../../js/collaborative-editor/contexts/StoreProvider'; -import { createSessionStore } from '../../../../js/collaborative-editor/stores/createSessionStore'; import type { AdaptorStoreInstance } from '../../../../js/collaborative-editor/stores/createAdaptorStore'; import { createAdaptorStore } from '../../../../js/collaborative-editor/stores/createAdaptorStore'; import type { AwarenessStoreInstance } from '../../../../js/collaborative-editor/stores/createAwarenessStore'; @@ -39,7 +38,10 @@ import { getURLStateMockValue, } from '../../__helpers__'; import { createWorkflowYDoc } from '../../__helpers__/workflowFactory'; -import { createMockSocket } from '../../__helpers__/sessionStoreHelpers'; +import { + createMockSocket, + createTestSessionStore, +} from '../../__helpers__/sessionStoreHelpers'; // Mock useURLState hook const urlState = createMockURLState(); @@ -86,7 +88,7 @@ function createWrapper( redirect: vi.fn(), }; - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); // Initialize session with proper mock socket so isSynced works const mockSocket = createMockSocket(); sessionStore.initializeSession( diff --git a/assets/test/collaborative-editor/components/inspector/TriggerInspector.test.tsx b/assets/test/collaborative-editor/components/inspector/TriggerInspector.test.tsx index 2eb11b0d6c6..0b23e712880 100644 --- a/assets/test/collaborative-editor/components/inspector/TriggerInspector.test.tsx +++ b/assets/test/collaborative-editor/components/inspector/TriggerInspector.test.tsx @@ -26,7 +26,6 @@ import type { CredentialStoreInstance } from '../../../../js/collaborative-edito import { createCredentialStore } from '../../../../js/collaborative-editor/stores/createCredentialStore'; import type { SessionContextStoreInstance } from '../../../../js/collaborative-editor/stores/createSessionContextStore'; import { createSessionContextStore } from '../../../../js/collaborative-editor/stores/createSessionContextStore'; -import { createSessionStore } from '../../../../js/collaborative-editor/stores/createSessionStore'; import type { UIStoreInstance } from '../../../../js/collaborative-editor/stores/createUIStore'; import { createUIStore } from '../../../../js/collaborative-editor/stores/createUIStore'; import type { WorkflowStoreInstance } from '../../../../js/collaborative-editor/stores/createWorkflowStore'; @@ -35,7 +34,10 @@ import { createMockPhoenixChannel, createMockPhoenixChannelProvider, } from '../../__helpers__/channelMocks'; -import { createMockSocket } from '../../__helpers__/sessionStoreHelpers'; +import { + createMockSocket, + createTestSessionStore, +} from '../../__helpers__/sessionStoreHelpers'; import { createMockURLState, getURLStateMockValue, @@ -100,7 +102,7 @@ function createWrapper( redirect: vi.fn(), }; - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const mockSocket = createMockSocket(); sessionStore.initializeSession( mockSocket, diff --git a/assets/test/collaborative-editor/hooks/useRunRetry.test.tsx b/assets/test/collaborative-editor/hooks/useRunRetry.test.tsx index e15034f7c2c..6e0ce27e085 100644 --- a/assets/test/collaborative-editor/hooks/useRunRetry.test.tsx +++ b/assets/test/collaborative-editor/hooks/useRunRetry.test.tsx @@ -11,7 +11,6 @@ import { } from '../../../js/collaborative-editor/hooks/useRunRetry'; import type { Dataclip } from '../../../js/collaborative-editor/api/dataclips'; import * as dataclipApi from '../../../js/collaborative-editor/api/dataclips'; -import { createSessionStore } from '../../../js/collaborative-editor/stores/createSessionStore'; import type { RunDetail, StepDetail, @@ -22,6 +21,7 @@ import { createMockURLState, getURLStateMockValue, } from '../__helpers__'; +import { createTestSessionStore } from '../__helpers__/sessionStoreHelpers'; import { createMockSocket } from '../mocks/phoenixSocket'; import { createMockSessionContextStore, @@ -72,7 +72,7 @@ function setMockActiveRun(run: RunDetail | null) { */ function createWrapper(): React.ComponentType<{ children: React.ReactNode }> { // Create session store and initialize it - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const mockSocket = createMockSocket(); sessionStore.initializeSession(mockSocket, 'test:room', { id: 'user-1', @@ -719,7 +719,7 @@ describe('useRunRetry - handleRetry', () => { }; // Create wrapper with getLimits mock using standardized factories - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const mockSocket = createMockSocket(); sessionStore.initializeSession(mockSocket, 'test:room', { id: 'user-1', @@ -801,7 +801,7 @@ describe('useRunRetry - handleRetry', () => { }; // Create wrapper with getLimits mock using standardized factories - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const mockSocket = createMockSocket(); sessionStore.initializeSession(mockSocket, 'test:room', { id: 'user-1', diff --git a/assets/test/collaborative-editor/hooks/useWorkflowReadOnly.test.tsx b/assets/test/collaborative-editor/hooks/useWorkflowReadOnly.test.tsx index 1176cf0270c..f04cbc2fef9 100644 --- a/assets/test/collaborative-editor/hooks/useWorkflowReadOnly.test.tsx +++ b/assets/test/collaborative-editor/hooks/useWorkflowReadOnly.test.tsx @@ -16,7 +16,6 @@ import { StoreContext } from '../../../js/collaborative-editor/contexts/StorePro import { useWorkflowReadOnly } from '../../../js/collaborative-editor/hooks/useWorkflow'; import type { SessionContextStoreInstance } from '../../../js/collaborative-editor/stores/createSessionContextStore'; import { createSessionContextStore } from '../../../js/collaborative-editor/stores/createSessionContextStore'; -import { createSessionStore } from '../../../js/collaborative-editor/stores/createSessionStore'; import type { WorkflowStoreInstance } from '../../../js/collaborative-editor/stores/createWorkflowStore'; import { createWorkflowStore } from '../../../js/collaborative-editor/stores/createWorkflowStore'; import type { Session } from '../../../js/collaborative-editor/types/session'; @@ -24,6 +23,7 @@ import { createSessionContext, mockPermissions, } from '../__helpers__/sessionContextFactory'; +import { createTestSessionStore } from '../__helpers__/sessionStoreHelpers'; import { createMockURLState, getURLStateMockValue, @@ -82,7 +82,7 @@ function createWrapper(options: WrapperOptions = {}): [ } = options; // Create stores - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const sessionContextStore = createSessionContextStore(); const workflowStore = createWorkflowStore(); @@ -529,7 +529,7 @@ describe('useWorkflowReadOnly - Valid Editing', () => { describe('useWorkflowReadOnly - Edge Cases', () => { test('handles null workflow gracefully', async () => { - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const sessionContextStore = createSessionContextStore(); const workflowStore = createWorkflowStore(); @@ -605,7 +605,7 @@ describe('useWorkflowReadOnly - Edge Cases', () => { }); test('handles null permissions gracefully (loading state - not read-only)', async () => { - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const sessionContextStore = createSessionContextStore(); const workflowStore = createWorkflowStore(); @@ -779,7 +779,7 @@ describe('useWorkflowReadOnly - Priority Order', () => { describe('useWorkflowReadOnly - Unsaved New Workflow', () => { test('returns read-only true for new workflow with content (from template or AI)', async () => { - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); // Pass isNewWorkflow: true when creating the store const sessionContextStore = createSessionContextStore(true); const workflowStore = createWorkflowStore(); @@ -858,7 +858,7 @@ describe('useWorkflowReadOnly - Unsaved New Workflow', () => { }); test('returns not read-only for new workflow without content (empty canvas)', async () => { - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); // Pass isNewWorkflow: true when creating the store const sessionContextStore = createSessionContextStore(true); const workflowStore = createWorkflowStore(); diff --git a/assets/test/collaborative-editor/useAdaptors.test.tsx b/assets/test/collaborative-editor/useAdaptors.test.tsx index 6a66b56a0a1..71c08d4ff72 100644 --- a/assets/test/collaborative-editor/useAdaptors.test.tsx +++ b/assets/test/collaborative-editor/useAdaptors.test.tsx @@ -17,7 +17,6 @@ import { useAdaptorsInUse, useAdaptorsLoading, } from '../../js/collaborative-editor/hooks/useAdaptors'; -import { createSessionStore } from '../../js/collaborative-editor/stores/createSessionStore'; import { SessionContext } from '../../js/collaborative-editor/contexts/SessionProvider'; import { StoreContext } from '../../js/collaborative-editor/contexts/StoreProvider'; @@ -26,6 +25,7 @@ import { createAwarenessStore } from '../../js/collaborative-editor/stores/creat import { createCredentialStore } from '../../js/collaborative-editor/stores/createCredentialStore'; import { createSessionContextStore } from '../../js/collaborative-editor/stores/createSessionContextStore'; import { createWorkflowStore } from '../../js/collaborative-editor/stores/createWorkflowStore'; +import { createTestSessionStore } from './__helpers__/sessionStoreHelpers'; import { mockAdaptorsList, mockAdaptor, @@ -39,7 +39,7 @@ import { createMockSocket } from './mocks/phoenixSocket'; // ============================================================================= function createWrapper() { - const sessionStore = createSessionStore(); + const sessionStore = createTestSessionStore(); const adaptorStore = createAdaptorStore(); const credentialStore = createCredentialStore(); const awarenessStore = createAwarenessStore(); diff --git a/assets/test/collaborative-editor/useSession.test.tsx b/assets/test/collaborative-editor/useSession.test.tsx index 8653cceda13..615f05a66eb 100644 --- a/assets/test/collaborative-editor/useSession.test.tsx +++ b/assets/test/collaborative-editor/useSession.test.tsx @@ -18,9 +18,9 @@ import { useSession, } from '../../js/collaborative-editor/hooks/useSession'; import type { SessionStoreInstance } from '../../js/collaborative-editor/stores/createSessionStore'; -import { createSessionStore } from '../../js/collaborative-editor/stores/createSessionStore'; import { SessionContext } from '../../js/collaborative-editor/contexts/SessionProvider'; +import { createTestSessionStore } from './__helpers__/sessionStoreHelpers'; import { createMockSocket } from './mocks/phoenixSocket'; // ============================================================================= @@ -32,7 +32,7 @@ import { createMockSocket } from './mocks/phoenixSocket'; * Returns both the wrapper and the store instance for test manipulation */ function createWrapper() { - const store = createSessionStore(); + const store = createTestSessionStore(); const mockSocket = createMockSocket(); // Initialize the session store @@ -58,7 +58,7 @@ function createWrapper() { * Useful for testing pre-initialization state */ function createUninitializedWrapper() { - const store = createSessionStore(); + const store = createTestSessionStore(); const wrapper = ({ children }: { children: React.ReactNode }) => ( Date: Thu, 10 Sep 2026 09:25:26 +0200 Subject: [PATCH 22/37] Update sobelow to 0.15.0 --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 1e654d1428e..dfc2fe9e2bb 100644 --- a/mix.exs +++ b/mix.exs @@ -164,7 +164,7 @@ defmodule Lightning.MixProject do {:retry, "~> 0.18"}, {:scrivener, "~> 2.7"}, {:sentry, "~> 13.2.0"}, - {:sobelow, "~> 0.14.1", only: [:test, :dev]}, + {:sobelow, "~> 0.15.0", only: [:test, :dev]}, {:sweet_xml, "~> 0.7.1", only: [:test]}, {:swoosh, "~> 1.26"}, {:gen_smtp, "~> 1.1"}, diff --git a/mix.lock b/mix.lock index f2140fa1964..6208b12e2a0 100644 --- a/mix.lock +++ b/mix.lock @@ -124,7 +124,7 @@ "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, "sentry": {:hex, :sentry, "13.2.0", "edef8afdbe3bbdae141c2a1a18661c214d9af57308ad6bd41b2182c6e9506382", [:mix], [{:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: true]}, {:hackney, ">= 1.8.0 and < 5.0.0", [hex: :hackney, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:opentelemetry, ">= 0.0.0", [hex: :opentelemetry, repo: "hexpm", optional: true]}, {:opentelemetry_api, ">= 0.0.0", [hex: :opentelemetry_api, repo: "hexpm", optional: true]}, {:opentelemetry_exporter, ">= 0.0.0", [hex: :opentelemetry_exporter, repo: "hexpm", optional: true]}, {:opentelemetry_semantic_conventions, ">= 0.0.0", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "f3397760ba0f0a2d8abb969c3e9f95e909839f7c45957249ee229c0e9738a3b4"}, "sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"}, - "sobelow": {:hex, :sobelow, "0.14.1", "2f81e8632f15574cba2402bcddff5497b413c01e6f094bc0ab94e83c2f74db81", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8fac9a2bd90fdc4b15d6fca6e1608efb7f7c600fa75800813b794ee9364c87f2"}, + "sobelow": {:hex, :sobelow, "0.15.0", "b067d7f8522a9d758fa89cb2bfcbab7ad72c45a0993cb958c989c6fd956fdd56", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "24a800e2d7fa8c3bd21561b6ad8ad4745ed726a09fd606598981d9048708da98"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, "statistics": {:hex, :statistics, "0.6.3", "7fb182e7c1cab2980e392c7efef7ce326539f081f9defda4099550e9c2c7cb0f", [:mix], [], "hexpm", "a43d87726d240205e9ef47f29650a6e3132b4e4061e05512f32fa8120784a8e0"}, From 2dd8eb007d8f707e62f85f7b985ad4d5db092e72 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 10 Sep 2026 09:25:27 +0200 Subject: [PATCH 23/37] Give the icon column names one home Five modules were mapping a shape to its icon columns: the catalogue schema, the store's projections, the scheduler's record merges, the controller and the URL builder. The schema did it under a different vocabulary again, taking :icon_square where everything else says :square. IconField holds the mapping, so an unknown shape is a FunctionClauseError at the boundary rather than a freshly minted atom. --- lib/lightning/adaptors/catalogue_adaptor.ex | 16 ++++++------- lib/lightning/adaptors/icon_field.ex | 25 +++++++++++++++++++++ lib/lightning/adaptors/scheduler.ex | 17 +++++++------- lib/lightning/adaptors/store.ex | 5 +++-- 4 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 lib/lightning/adaptors/icon_field.ex diff --git a/lib/lightning/adaptors/catalogue_adaptor.ex b/lib/lightning/adaptors/catalogue_adaptor.ex index 0115bd3c677..dd07c454cc2 100644 --- a/lib/lightning/adaptors/catalogue_adaptor.ex +++ b/lib/lightning/adaptors/catalogue_adaptor.ex @@ -9,6 +9,8 @@ defmodule Lightning.Adaptors.Catalogue.Adaptor do import Ecto.Changeset + alias Lightning.Adaptors.IconField + defmodule JSONBinary do @moduledoc """ Ecto type for `schema_data`: a JSON binary in a `text` column. @@ -106,18 +108,16 @@ defmodule Lightning.Adaptors.Catalogue.Adaptor do |> validate_format(:name, Lightning.Adaptors.PackageName.name_format()) |> validate_inclusion(:icon_square_ext, ~w(png svg)) |> validate_inclusion(:icon_rectangle_ext, ~w(png svg)) - |> validate_icon_sha256_pair(:icon_square) - |> validate_icon_sha256_pair(:icon_rectangle) + |> validate_icon_sha256_pair(:square) + |> validate_icon_sha256_pair(:rectangle) |> unique_constraint([:name, :source]) end - @spec validate_icon_sha256_pair( - Ecto.Changeset.t(), - :icon_square | :icon_rectangle - ) :: Ecto.Changeset.t() + @spec validate_icon_sha256_pair(Ecto.Changeset.t(), IconField.shape()) :: + Ecto.Changeset.t() defp validate_icon_sha256_pair(changeset, shape) do - ext_field = :"#{shape}_ext" - sha_field = :"#{shape}_sha256" + ext_field = IconField.ext(shape) + sha_field = IconField.sha256(shape) case {get_field(changeset, ext_field), get_field(changeset, sha_field)} do {nil, nil} -> diff --git a/lib/lightning/adaptors/icon_field.ex b/lib/lightning/adaptors/icon_field.ex new file mode 100644 index 00000000000..7159e89e9da --- /dev/null +++ b/lib/lightning/adaptors/icon_field.ex @@ -0,0 +1,25 @@ +defmodule Lightning.Adaptors.IconField do + @moduledoc """ + Schema column names for an icon shape. + + Every module that reaches for an icon column — the catalogue schema, + the store's projections, the scheduler's record merges, the controller + and the URL builder — goes through here, so `:square` and + `:rectangle` mean the same columns everywhere and no column name is + built by interpolating an atom. + """ + + @type shape :: :square | :rectangle + + @spec ext(shape()) :: atom() + def ext(:square), do: :icon_square_ext + def ext(:rectangle), do: :icon_rectangle_ext + + @spec sha256(shape()) :: atom() + def sha256(:square), do: :icon_square_sha256 + def sha256(:rectangle), do: :icon_rectangle_sha256 + + @spec etag(shape()) :: atom() + def etag(:square), do: :icon_square_etag + def etag(:rectangle), do: :icon_rectangle_etag +end diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 63996f0c97f..b684b15e98b 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -24,6 +24,7 @@ defmodule Lightning.Adaptors.Scheduler do alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Config alias Lightning.Adaptors.IconCache + alias Lightning.Adaptors.IconField alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor require Logger @@ -556,8 +557,8 @@ defmodule Lightning.Adaptors.Scheduler do IconCache.write!(source, record.name, shape, ext, bytes, sha) record - |> Map.put(:"icon_#{shape}_ext", ext) - |> Map.put(:"icon_#{shape}_sha256", sha) + |> Map.put(IconField.ext(shape), ext) + |> Map.put(IconField.sha256(shape), sha) |> maybe_put_etag(shape, Map.get(entry, :etag)) rescue e -> @@ -585,7 +586,7 @@ defmodule Lightning.Adaptors.Scheduler do defp maybe_put_etag(record, _shape, nil), do: record defp maybe_put_etag(record, shape, etag) when is_binary(etag) do - Map.put(record, :"icon_#{shape}_etag", etag) + Map.put(record, IconField.etag(shape), etag) end defp reapply_icons(existing_rows, icons, state) do @@ -628,8 +629,8 @@ defmodule Lightning.Adaptors.Scheduler do end defp accumulate_icon_change(acc, shape, row, package_icons, state) do - sha_key = :"icon_#{shape}_sha256" - etag_key = :"icon_#{shape}_etag" + sha_key = IconField.sha256(shape) + etag_key = IconField.etag(shape) case Map.get(package_icons, shape) do %{data: bytes, ext: ext, sha256: sha} = entry when is_binary(bytes) -> @@ -652,9 +653,9 @@ defmodule Lightning.Adaptors.Scheduler do end defp accumulate_fetched_icon(acc, shape, row, entry, ext, sha, bytes, state) do - sha_key = :"icon_#{shape}_sha256" - ext_key = :"icon_#{shape}_ext" - etag_key = :"icon_#{shape}_etag" + sha_key = IconField.sha256(shape) + ext_key = IconField.ext(shape) + etag_key = IconField.etag(shape) IconCache.write!(state.source, row.name, shape, ext, bytes, sha) diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index 889fc8fc747..05c7864e0c7 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -15,6 +15,7 @@ defmodule Lightning.Adaptors.Store do alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Config alias Lightning.Adaptors.IconCache + alias Lightning.Adaptors.IconField alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor alias LightningWeb.AdaptorIconURL @@ -275,7 +276,7 @@ defmodule Lightning.Adaptors.Store do @spec ext_for_shape(icon_meta(), :square | :rectangle) :: {:ok, String.t()} | {:error, :not_found} defp ext_for_shape(meta, shape) do - case Map.get(meta, :"icon_#{shape}_ext") do + case Map.get(meta, IconField.ext(shape)) do nil -> {:error, :not_found} ext -> {:ok, ext} end @@ -284,7 +285,7 @@ defmodule Lightning.Adaptors.Store do @spec sha256_for_shape(icon_meta(), :square | :rectangle) :: {:ok, binary()} | {:error, :not_found} defp sha256_for_shape(meta, shape) do - case Map.get(meta, :"icon_#{shape}_sha256") do + case Map.get(meta, IconField.sha256(shape)) do nil -> {:error, :not_found} sha -> {:ok, sha} end From 7bbfae7e71e2d9b06541aa0ffa474a98de471021 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 10 Sep 2026 09:25:42 +0200 Subject: [PATCH 24/37] Serve icon content types from literals, convert the shape once put_resp_content_type took a value computed from the request, which sobelow flags at medium and which CI fails on. Two clauses, one per extension, make the content type a literal at the call site and give the function a catch-all 404 instead of a FunctionClauseError. The shape string now becomes an atom once in show/2 rather than at each of the three places that needed it, so the controller reads its columns through IconField like everything else and has_icon?/2 and ext_for_shape_param/2 are gone. send_file keeps a sobelow_skip. Sobelow flags any send_file whose path traces back to a param, which is true of every file server; the path is built by Adaptors.icon/2 from a catalogue row, so an unknown adaptor 404s before any of it happens. --- .../controllers/adaptor_icon_controller.ex | 68 ++++++++----------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/lib/lightning_web/controllers/adaptor_icon_controller.ex b/lib/lightning_web/controllers/adaptor_icon_controller.ex index 55d33f56c0c..10e525e2390 100644 --- a/lib/lightning_web/controllers/adaptor_icon_controller.ex +++ b/lib/lightning_web/controllers/adaptor_icon_controller.ex @@ -13,10 +13,12 @@ defmodule LightningWeb.AdaptorIconURL do Returns `nil` when `meta` has no ext or sha256 for the requested shape — i.e. when no icon is available. """ - @spec build(String.t(), map(), :square | :rectangle) :: String.t() | nil + alias Lightning.Adaptors.IconField + + @spec build(String.t(), map(), IconField.shape()) :: String.t() | nil def build(name, meta, shape) do - with ext when not is_nil(ext) <- ext_for_shape(meta, shape), - sha when not is_nil(sha) <- sha_for_shape(meta, shape) do + with ext when not is_nil(ext) <- Map.get(meta, IconField.ext(shape)), + sha when not is_nil(sha) <- Map.get(meta, IconField.sha256(shape)) do sha8 = sha |> binary_part(0, 4) |> Base.encode16(case: :lower) "/adaptors/icons/#{URI.encode(name, &URI.char_unreserved?/1)}/" <> @@ -25,14 +27,6 @@ defmodule LightningWeb.AdaptorIconURL do _ -> nil end end - - defp ext_for_shape(meta, :square), do: Map.get(meta, :icon_square_ext) - defp ext_for_shape(meta, :rectangle), do: Map.get(meta, :icon_rectangle_ext) - - defp sha_for_shape(meta, :square), do: Map.get(meta, :icon_square_sha256) - - defp sha_for_shape(meta, :rectangle), - do: Map.get(meta, :icon_rectangle_sha256) end defmodule LightningWeb.AdaptorIconController do @@ -55,6 +49,7 @@ defmodule LightningWeb.AdaptorIconController do use LightningWeb, :controller alias Lightning.Adaptors + alias Lightning.Adaptors.IconField @immutable_cache "public, max-age=31536000, immutable" @@ -85,16 +80,17 @@ defmodule LightningWeb.AdaptorIconController do %{"name" => name, "shape" => shape, "sha8" => sha8, "ext" => ext} ) when shape in ~w(square rectangle) do + shape = String.to_existing_atom(shape) + case Adaptors.icon_meta(name) do {:error, :not_found} -> send_resp(conn, 404, "") {:ok, meta} -> - cond do - ext_for_shape_param(meta, shape) != ext -> - send_resp(conn, 404, "") + stored_ext = Map.get(meta, IconField.ext(shape)) - not has_icon?(meta, shape) -> + cond do + is_nil(stored_ext) or stored_ext != ext -> send_resp(conn, 404, "") sha_matches?(meta, shape, sha8) -> @@ -108,11 +104,23 @@ defmodule LightningWeb.AdaptorIconController do def show(conn, _params), do: send_resp(conn, 404, "") - defp serve_bytes(conn, name, shape, ext) do - case Adaptors.icon(name, String.to_existing_atom(shape)) do + defp serve_bytes(conn, name, shape, "png") do + conn |> put_resp_content_type("image/png") |> send_icon(name, shape) + end + + defp serve_bytes(conn, name, shape, "svg") do + conn |> put_resp_content_type("image/svg+xml") |> send_icon(name, shape) + end + + defp serve_bytes(conn, _name, _shape, _ext), do: send_resp(conn, 404, "") + + # `path` is a cache path built by `Adaptors.icon/2` from a catalogue row, + # reached only after `icon_meta/1` confirmed the adaptor exists. + # sobelow_skip ["Traversal.SendFile"] + defp send_icon(conn, name, shape) do + case Adaptors.icon(name, shape) do {:ok, path} -> conn - |> put_resp_content_type(content_type_for(ext)) |> put_resp_header("cache-control", @immutable_cache) |> merge_resp_headers(LightningWeb.Utils.sandboxed_asset_headers()) |> send_file(200, path) @@ -123,12 +131,7 @@ defmodule LightningWeb.AdaptorIconController do end defp redirect_to_current(conn, name, meta, shape) do - url = - LightningWeb.AdaptorIconURL.build( - name, - meta, - String.to_existing_atom(shape) - ) + url = LightningWeb.AdaptorIconURL.build(name, meta, shape) conn |> put_resp_header("cache-control", "no-store") @@ -136,24 +139,11 @@ defmodule LightningWeb.AdaptorIconController do |> send_resp(302, "") end - defp has_icon?(meta, shape), do: not is_nil(ext_for_shape_param(meta, shape)) - - defp ext_for_shape_param(meta, "square"), do: Map.get(meta, :icon_square_ext) - - defp ext_for_shape_param(meta, "rectangle"), - do: Map.get(meta, :icon_rectangle_ext) - - defp sha_matches?(meta, "square", sha8), - do: sha_prefix_matches?(Map.get(meta, :icon_square_sha256), sha8) - - defp sha_matches?(meta, "rectangle", sha8), - do: sha_prefix_matches?(Map.get(meta, :icon_rectangle_sha256), sha8) + defp sha_matches?(meta, shape, sha8), + do: sha_prefix_matches?(Map.get(meta, IconField.sha256(shape)), sha8) defp sha_prefix_matches?(<>, sha8), do: Base.encode16(prefix, case: :lower) == String.downcase(sha8) defp sha_prefix_matches?(_, _sha8), do: false - - defp content_type_for("png"), do: "image/png" - defp content_type_for("svg"), do: "image/svg+xml" end From f52e3db24e36b4ab2cd3298894144d9421ef18d6 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 10 Sep 2026 09:25:43 +0200 Subject: [PATCH 25/37] Reject adaptor names that could escape the icon cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adaptor names become directory segments in IconCache.path/5, and the name format allowed "..": [\w.-]+ matches it, and the format permits one slash, so "../.." resolved two levels above the cache root. write!/6 then ran mkdir_p!, write! and a File.rm sweep wherever it landed. The name check also ran too late to help. Icons are written from the strategy's response in Scheduler.merge_icon/4, well before the row reaches CatalogueAdaptor.changeset/2 and its validate_format — and the changeset is the only place a name is ever validated, since none of the update_all paths touch that column. So the guard goes in path/5, which every File call in the module routes through, and it raises. The scheduler already rescues around write!/6, so a hostile registry entry now degrades to a logged warning. The name format is tightened alongside it: a segment must not start with "." or "_", which is npm's own rule and keeps "." and ".." out. --- lib/lightning/adaptors/icon_cache.ex | 18 +++++++++++++++++- lib/lightning/adaptors/package_name.ex | 10 ++++++++-- test/lightning/adaptors/icon_cache_test.exs | 14 ++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/lib/lightning/adaptors/icon_cache.ex b/lib/lightning/adaptors/icon_cache.ex index 3d6afa97145..707ffdaf382 100644 --- a/lib/lightning/adaptors/icon_cache.ex +++ b/lib/lightning/adaptors/icon_cache.ex @@ -32,6 +32,7 @@ defmodule Lightning.Adaptors.IconCache do """ alias Lightning.Adaptors.Config + alias Lightning.Adaptors.PackageName @type source :: :npm | :local @type name :: String.t() @@ -39,14 +40,25 @@ defmodule Lightning.Adaptors.IconCache do @type ext :: String.t() @doc """ - Disk path for an icon. Pure — nothing is checked or created. + Disk path for an icon. Nothing is created. `name` may contain a `/` (scoped npm packages like `@openfn/language-foo`); `Path.join/1` preserves the slash so the scope becomes a real subdirectory. + + Raises `ArgumentError` on a name `PackageName` would reject. Icons are + written straight from a strategy's response, before the row reaches + `CatalogueAdaptor.changeset/2` and its name validation, so this is the + only thing standing between a hostile registry entry and a write + outside the cache root. """ @spec path(source(), name(), shape(), ext(), binary()) :: Path.t() def path(source, name, shape, ext, sha256) do + unless Regex.match?(PackageName.name_format(), name) do + raise ArgumentError, + "unsafe adaptor name for an icon path: #{inspect(name)}" + end + Path.join([ Config.icon_path(), to_string(source), @@ -73,6 +85,9 @@ defmodule Lightning.Adaptors.IconCache do pre-sha naming, is removed first, so a rename never lands on a directory left empty by its own sweep. """ + # Every path here comes from `path/5`, which rejects a name that is not + # a safe path segment, so nothing escapes `Config.icon_path/0`. + # sobelow_skip ["Traversal.FileModule"] @spec write!(source(), name(), shape(), ext(), binary(), binary()) :: Path.t() def write!(source, name, shape, ext, bytes, sha256) when is_binary(bytes) do @@ -96,6 +111,7 @@ defmodule Lightning.Adaptors.IconCache do final_path end + # sobelow_skip ["Traversal.FileModule"] defp remove_superseded(dir, shape, final_path) do dir |> Path.join("#{shape}.*") diff --git a/lib/lightning/adaptors/package_name.ex b/lib/lightning/adaptors/package_name.ex index c98569f09bc..afa5d32b43b 100644 --- a/lib/lightning/adaptors/package_name.ex +++ b/lib/lightning/adaptors/package_name.ex @@ -4,9 +4,15 @@ defmodule Lightning.Adaptors.PackageName do """ # `\A…\z` rather than `^…$`: `$` matches before a trailing newline. - @strict_format ~r{\A(@?[\w.-]+(?:/[\w.-]+)?)(?:@([\w.-]+))?\z} + # + # A segment may not begin with `.` or `_`, which is npm's own rule. That + # keeps `.` and `..` out, so a name is always safe to use as a path + # segment — see `Lightning.Adaptors.IconCache`. + @segment "[a-zA-Z0-9-][\\w.-]*" - @name_format ~r{\A@?[\w.-]+(?:/[\w.-]+)?\z} + @strict_format ~r{\A(@?#{@segment}(?:/#{@segment})?)(?:@([\w.-]+))?\z} + + @name_format ~r{\A@?#{@segment}(?:/#{@segment})?\z} @language_prefix "@openfn/language-" diff --git a/test/lightning/adaptors/icon_cache_test.exs b/test/lightning/adaptors/icon_cache_test.exs index 8330e8c6522..41efdab4ee1 100644 --- a/test/lightning/adaptors/icon_cache_test.exs +++ b/test/lightning/adaptors/icon_cache_test.exs @@ -198,6 +198,20 @@ defmodule Lightning.Adaptors.IconCacheTest do end end + describe "path/5 name validation" do + test "refuses names that would escape the cache root", %{root: root} do + for name <- ["..", "../..", "@openfn/..", ".hidden", "@../evil"] do + assert_raise ArgumentError, ~r/unsafe adaptor name/, fn -> + IconCache.path(:npm, name, :square, "png", <<0::256>>) + end + + assert_raise ArgumentError, fn -> write(name, "bytes") end + end + + assert File.ls!(root) == [] + end + end + defp write(name, bytes, shape \\ :square) do IconCache.write!( :npm, From 986b8c65d2af327ad188a79425d6d71dc3e9d550 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 10 Sep 2026 09:25:52 +0200 Subject: [PATCH 26/37] Annotate the traversal checks on operator-supplied paths dump_to_file/2 and seed_from_file/2 take a mix-task or release-command argument, and the Local strategy reads under the paths an operator configured for it. None of them see request input. --- lib/lightning/adaptors/dump.ex | 3 +++ lib/lightning/adaptors/local.ex | 6 ++++++ lib/lightning/adaptors/seed.ex | 3 +++ 3 files changed, 12 insertions(+) diff --git a/lib/lightning/adaptors/dump.ex b/lib/lightning/adaptors/dump.ex index b211ba702f5..8404181be66 100644 --- a/lib/lightning/adaptors/dump.ex +++ b/lib/lightning/adaptors/dump.ex @@ -25,6 +25,9 @@ defmodule Lightning.Adaptors.Dump do * `:source` - `:npm` (default) or `:local` """ + # `path` is a mix-task argument (`mix lightning.adaptors.dump`) or a + # release-command argument — an operator's own filesystem, not a request. + # sobelow_skip ["Traversal.FileModule"] @spec dump_to_file(Path.t(), keyword()) :: {:ok, non_neg_integer()} def dump_to_file(path, opts \\ []) do source = Keyword.get(opts, :source, :npm) diff --git a/lib/lightning/adaptors/local.ex b/lib/lightning/adaptors/local.ex index 4b8522aab89..2081288ed49 100644 --- a/lib/lightning/adaptors/local.ex +++ b/lib/lightning/adaptors/local.ex @@ -166,6 +166,10 @@ defmodule Lightning.Adaptors.Local do records end + # `dir` is under a path an operator configured for this strategy + # (`Config.strategy_opts(__MODULE__)[:paths]`), never request input. + # Applies to `read_schema/1` and `read_icon/2` below too. + # sobelow_skip ["Traversal.FileModule"] defp read_package_dir(dir) do pkg_json_path = Path.join(dir, "package.json") @@ -241,6 +245,7 @@ defmodule Lightning.Adaptors.Local do } end + # sobelow_skip ["Traversal.FileModule"] defp read_schema(dir) do case File.read(Path.join(dir, @schema_filename)) do {:ok, body} -> Lightning.Adaptors.Strategy.digest_schema(body) @@ -249,6 +254,7 @@ defmodule Lightning.Adaptors.Local do end end + # sobelow_skip ["Traversal.FileModule"] defp read_icon(dir, shape) do Enum.find_value(@icon_exts, {:error, :not_found}, fn ext -> case File.read(icon_path(dir, shape, ext)) do diff --git a/lib/lightning/adaptors/seed.ex b/lib/lightning/adaptors/seed.ex index 4df67aade9b..1862c63fa36 100644 --- a/lib/lightning/adaptors/seed.ex +++ b/lib/lightning/adaptors/seed.ex @@ -27,6 +27,9 @@ defmodule Lightning.Adaptors.Seed do * `:sup` - supervisor instance whose topic the broadcasts go to, defaulting to `Lightning.Adaptors.Config.default_instance/0` """ + # `path` is a mix-task argument (`mix lightning.adaptors.import`) or a + # release-command argument — an operator's own filesystem, not a request. + # sobelow_skip ["Traversal.FileModule"] @spec seed_from_file(Path.t(), keyword()) :: {:ok, non_neg_integer()} def seed_from_file(path, opts \\ []) do source = Keyword.get(opts, :source, :npm) From e35e499685b12e42055ae04afeaac3da94b2c9ef Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 10 Sep 2026 09:25:53 +0200 Subject: [PATCH 27/37] Read the version group-by without elem/2 Destructuring says which half is the name and which the version, so the TODO asking for that no longer has anything to ask. --- lib/lightning/adaptors/catalogue.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/lightning/adaptors/catalogue.ex b/lib/lightning/adaptors/catalogue.ex index 6b9df475857..4342ef5b48a 100644 --- a/lib/lightning/adaptors/catalogue.ex +++ b/lib/lightning/adaptors/catalogue.ex @@ -296,9 +296,9 @@ defmodule Lightning.Adaptors.Catalogue do order_by: [asc: v.inserted_at, asc: v.version], select: {a.name, v.version} ) - |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) - - # TODO: 👆would it not be easier to do the group by in the query? That elem,elem group by isn't that easy to understand + |> Enum.group_by(fn {name, _version} -> name end, fn {_name, version} -> + version + end) Enum.map(adaptors, fn adaptor -> Map.put(adaptor, :versions, Map.get(versions_by_name, adaptor.name, [])) From 82b2ed04d897741d7ed69df2b22b973bb43408fb Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 10 Sep 2026 12:11:25 +0200 Subject: [PATCH 28/37] Take the scheduler's interval and empty-catalogue warning as opts An empty catalogue only needs an operator's attention when no timer will fill it: with an interval set the first tick is already due immediately, so that case logs at info and the warning is left for interval=0. The suite's own boot hits exactly that state, so warn_when_empty turns the warning off for the instance application.ex starts. Both it and refresh_interval now reach the scheduler as required opts, read once from config by the supervisor the way strategy already is, which lets the three test helpers that start a scheduler directly pass their own values instead of writing to the application env and restoring it. --- config/test.exs | 6 ++- lib/lightning/adaptors/config.ex | 10 +++++ lib/lightning/adaptors/scheduler.ex | 41 ++++++++++++++---- lib/lightning/adaptors/supervisor.ex | 15 ++++++- lib/lightning/config/bootstrap.ex | 2 +- test/lightning/adaptors/readiness_test.exs | 2 + test/lightning/adaptors/scheduler_test.exs | 49 ++++++++++++---------- test/lightning/adaptors_test.exs | 33 +++++---------- 8 files changed, 101 insertions(+), 57 deletions(-) diff --git a/config/test.exs b/config/test.exs index 43c8a014885..56a4d5df376 100644 --- a/config/test.exs +++ b/config/test.exs @@ -108,7 +108,11 @@ config :lightning, Lightning.Mailer, adapter: Swoosh.Adapters.Test config :lightning, Lightning.Adaptors, strategy: Lightning.Adaptors.StrategyMock, - refresh_interval: 0 + refresh_interval: 0, + # The instance application.ex starts comes up on an empty catalogue with + # refreshes off, which is exactly the state the operator warning is for. + # Test-owned instances pass their own opts instead. + warn_when_empty: false # The reconciler runs against the shared production catalogue table, which # tests seed freely; each test that needs it starts its own named instance. diff --git a/lib/lightning/adaptors/config.ex b/lib/lightning/adaptors/config.ex index 79de389c7ee..8bee241e145 100644 --- a/lib/lightning/adaptors/config.ex +++ b/lib/lightning/adaptors/config.ex @@ -67,6 +67,16 @@ defmodule Lightning.Adaptors.Config do get(:refresh_interval, @default_refresh_interval) end + @doc """ + Whether a scheduler booting on an empty catalogue that no timer will + fill should warn an operator. Defaults to true. Read once by + `Lightning.Adaptors.Supervisor` at boot and passed to the scheduler. + """ + @spec warn_when_empty?() :: boolean() + def warn_when_empty? do + get(:warn_when_empty, true) + end + @doc """ How long a read waits for a cache fill, in milliseconds. Defaults to 15 seconds. diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index b684b15e98b..006a9e80d08 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -35,7 +35,10 @@ defmodule Lightning.Adaptors.Scheduler do @doc """ Starts the Scheduler. Required opts: `:name`, `:sup`, `:lock_key`, - `:cache`, `:tasks`, `:source_topic`. Optional: `:checked_at` (1-arity fn, + `:cache`, `:tasks`, `:source_topic`, `:refresh_interval` (tick interval + in milliseconds; `0` disables the timer) and `:warn_when_empty` (whether + booting on an empty catalogue with the timer disabled logs a warning). + Optional: `:checked_at` (1-arity fn, default `&Catalogue.max_checked_at/1`) reads the source's last-checked timestamp; called once at boot to schedule the delay before the scheduler's initial tick. @@ -48,6 +51,8 @@ defmodule Lightning.Adaptors.Scheduler do _ = Keyword.fetch!(opts, :cache) _ = Keyword.fetch!(opts, :tasks) _ = Keyword.fetch!(opts, :source_topic) + _ = Keyword.fetch!(opts, :refresh_interval) + _ = Keyword.fetch!(opts, :warn_when_empty) GenServer.start_link(__MODULE__, opts, name: name) end @@ -120,12 +125,13 @@ defmodule Lightning.Adaptors.Scheduler do checked_at = Keyword.get(opts, :checked_at, &Catalogue.max_checked_at/1) source = AdaptorsSupervisor.source(sup) - interval_ms = Config.refresh_interval() + interval_ms = Keyword.fetch!(opts, :refresh_interval) state = %{ sup: sup, source: source, interval_ms: interval_ms, + warn_when_empty: Keyword.fetch!(opts, :warn_when_empty), source_topic: source_topic, cache: cache, tasks: tasks, @@ -142,16 +148,12 @@ defmodule Lightning.Adaptors.Scheduler do @impl true def handle_continue(:check_catalogue, state) do # Read runs, and delay is computed, even when interval_ms == 0 — that's - # the only way an interval=0 (disabled) deployment still gets the - # empty-catalogue warning below. Don't skip it for that branch. + # the only way an interval=0 (disabled) deployment still learns its + # catalogue is empty. Don't skip it for that branch. checked_at = case read_checked_at(state) do nil -> - Logger.warning( - "Adaptors[#{state.source}]: catalogue is empty at boot — see " <> - "ADAPTORS.md's \"Running without internet access\" section" - ) - + log_empty_catalogue(state) nil :error -> @@ -179,6 +181,27 @@ defmodule Lightning.Adaptors.Scheduler do {:noreply, state} end + # An empty catalogue only needs an operator's attention when no timer will + # fill it; with an interval set the first tick is already due immediately. + defp log_empty_catalogue(state) do + cond do + state.interval_ms > 0 -> + Logger.info( + "Adaptors[#{state.source}]: catalogue is empty at boot — refreshing now" + ) + + state.warn_when_empty -> + Logger.warning( + "Adaptors[#{state.source}]: catalogue is empty and refreshes are " <> + "disabled (interval=0) — see ADAPTORS.md's \"Running without " <> + "internet access\" section" + ) + + true -> + :ok + end + end + defp read_checked_at(state) do state.checked_at.(state.source) rescue diff --git a/lib/lightning/adaptors/supervisor.ex b/lib/lightning/adaptors/supervisor.ex index 10415b588f2..077f619015c 100644 --- a/lib/lightning/adaptors/supervisor.ex +++ b/lib/lightning/adaptors/supervisor.ex @@ -24,6 +24,11 @@ defmodule Lightning.Adaptors.Supervisor do defaulting to `Lightning.Adaptors.Config.strategy/0` * `:lock_key` - `HighlanderPG` advisory-lock key, defaulting to `lock_key(name)` + * `:refresh_interval` - scheduler tick interval in milliseconds, + defaulting to `Lightning.Adaptors.Config.refresh_interval/0` + * `:warn_when_empty` - whether the scheduler warns when it boots on an + empty catalogue that no timer will fill, defaulting to + `Lightning.Adaptors.Config.warn_when_empty?/0` * `:checked_at` - forwarded to the scheduler; see `Lightning.Adaptors.Scheduler.start_link/1` """ @@ -39,6 +44,12 @@ defmodule Lightning.Adaptors.Supervisor do strategy = Keyword.get(opts, :strategy, Config.strategy()) lock_key = Keyword.get(opts, :lock_key, lock_key(name)) + refresh_interval = + Keyword.get(opts, :refresh_interval, Config.refresh_interval()) + + warn_when_empty = + Keyword.get(opts, :warn_when_empty, Config.warn_when_empty?()) + # Per-instance config for stateless callers that hold only the name. # Children take theirs from the child spec (see lock_key), not from here. :persistent_term.put(meta_key(name), %{ @@ -63,7 +74,9 @@ defmodule Lightning.Adaptors.Supervisor do lock_key: lock_key, cache: cache, tasks: tasks, - source_topic: source_topic + source_topic: source_topic, + refresh_interval: refresh_interval, + warn_when_empty: warn_when_empty ] ++ Keyword.take(opts, [:checked_at]) ]} } diff --git a/lib/lightning/config/bootstrap.ex b/lib/lightning/config/bootstrap.ex index c05dfa2308a..ba114600ca7 100644 --- a/lib/lightning/config/bootstrap.ex +++ b/lib/lightning/config/bootstrap.ex @@ -235,7 +235,7 @@ defmodule Lightning.Config.Bootstrap do adaptors_path: env!("ADAPTORS_PATH", :string, "./priv/openfn") # Comma-separated to match the ws-worker parser, so the picker view and - # @local resolution agree on the same repo list. See RUNNINGLOCAL.md. + # @local resolution agree on the same repo list. See ADAPTORS.md. local_adaptors_repos = parse_repo_list(env!("OPENFN_ADAPTORS_REPO", :string, nil)) diff --git a/test/lightning/adaptors/readiness_test.exs b/test/lightning/adaptors/readiness_test.exs index f7da798f785..c4d653fab5c 100644 --- a/test/lightning/adaptors/readiness_test.exs +++ b/test/lightning/adaptors/readiness_test.exs @@ -53,6 +53,8 @@ defmodule Lightning.Adaptors.ReadinessTest do cache: AdaptorsSupervisor.cache_name(sup), tasks: AdaptorsSupervisor.tasks_name(sup), source_topic: AdaptorsSupervisor.source_topic(sup), + refresh_interval: 0, + warn_when_empty: false, checked_at: fn _source -> nil end }) diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index b2571989e99..f6ee027692c 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -49,9 +49,7 @@ defmodule Lightning.Adaptors.SchedulerTest do end # Replace the supervisor's inert auto-started (HighlanderPG-wrapped) - # Scheduler with a controlled one under test ownership. Application - # env is restored immediately after start_supervised!/1 returns - # because the Scheduler captures interval_ms in init/1. + # Scheduler with a controlled one under test ownership. # # The test-owned Scheduler bypasses HighlanderPG entirely: we # register the GenServer directly under the same `{:global, …}` name @@ -59,15 +57,6 @@ defmodule Lightning.Adaptors.SchedulerTest do # `AdaptorsSupervisor.global_scheduler_name/1` exactly as production # callers do. defp start_scheduler(sup, opts \\ []) do - interval = Keyword.get(opts, :interval, 99_999_999) - original_env = Application.get_env(:lightning, Lightning.Adaptors, []) - - Application.put_env( - :lightning, - Lightning.Adaptors, - Keyword.put(original_env, :refresh_interval, interval) - ) - global_name = AdaptorsSupervisor.global_scheduler_name(sup) source_topic = AdaptorsSupervisor.source_topic(sup) @@ -84,14 +73,12 @@ defmodule Lightning.Adaptors.SchedulerTest do lock_key: AdaptorsSupervisor.lock_key(sup), cache: AdaptorsSupervisor.cache_name(sup), tasks: AdaptorsSupervisor.tasks_name(sup), - source_topic: source_topic + source_topic: source_topic, + refresh_interval: Keyword.get(opts, :interval, 99_999_999), + warn_when_empty: Keyword.get(opts, :warn_when_empty, false) ] ++ Keyword.take(opts, [:checked_at]) - pid = start_supervised!({Scheduler, scheduler_opts}) - - Application.put_env(:lightning, Lightning.Adaptors, original_env) - - pid + start_supervised!({Scheduler, scheduler_opts}) end defp drain_tick_ran do @@ -221,7 +208,7 @@ defmodule Lightning.Adaptors.SchedulerTest do assert {%Postgrex.Error{}, _stacktrace} = reason end - test "an empty catalogue logs a boot warning and still ticks when interval > 0", + test "an empty catalogue ticks at once, without warning, when interval > 0", %{sup: sup} do test_pid = self() @@ -236,10 +223,10 @@ defmodule Lightning.Adaptors.SchedulerTest do assert_receive :tick_ran, 2000 end) - assert log =~ "catalogue is empty at boot" + refute log =~ "catalogue is empty" end - test "an empty catalogue logs a boot warning and schedules no tick when interval is 0", + test "an empty catalogue warns an operator and schedules no tick when interval is 0", %{sup: sup} do test_pid = self() @@ -248,13 +235,29 @@ defmodule Lightning.Adaptors.SchedulerTest do {:ok, []} end) + # start_scheduler/2 defaults the warning off; pose as a real + # deployment to see it. log = capture_log(fn -> - start_scheduler(sup, checked_at: fn _source -> nil end, interval: 0) + start_scheduler(sup, + checked_at: fn _source -> nil end, + interval: 0, + warn_when_empty: true + ) + refute_receive :tick_ran, 200 end) - assert log =~ "catalogue is empty at boot" + assert log =~ "catalogue is empty and refreshes are disabled" + end + + test "an empty catalogue stays quiet when the warning is off", %{sup: sup} do + log = + capture_log(fn -> + start_scheduler(sup, checked_at: fn _source -> nil end, interval: 0) + end) + + refute log =~ "catalogue is empty" end end diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index aa00539c20c..13729831ad3 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -15,14 +15,6 @@ defmodule Lightning.AdaptorsTest do setup :isolated_adaptors defp start_scheduler(sup) do - original_env = Application.get_env(:lightning, Lightning.Adaptors, []) - - Application.put_env( - :lightning, - Lightning.Adaptors, - Keyword.put(original_env, :refresh_interval, 99_999_999) - ) - # Stop the supervisor's auto-started HighlanderPG (and its wrapped # Scheduler) so we can start a replacement under the controlled # interval without name collision. The test-owned Scheduler registers @@ -30,20 +22,17 @@ defmodule Lightning.AdaptorsTest do :ok = Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) - pid = - start_supervised!({ - Scheduler, - name: AdaptorsSupervisor.global_scheduler_name(sup), - sup: sup, - lock_key: AdaptorsSupervisor.lock_key(sup), - cache: AdaptorsSupervisor.cache_name(sup), - tasks: AdaptorsSupervisor.tasks_name(sup), - source_topic: AdaptorsSupervisor.source_topic(sup) - }) - - Application.put_env(:lightning, Lightning.Adaptors, original_env) - - pid + start_supervised!({ + Scheduler, + name: AdaptorsSupervisor.global_scheduler_name(sup), + sup: sup, + lock_key: AdaptorsSupervisor.lock_key(sup), + cache: AdaptorsSupervisor.cache_name(sup), + tasks: AdaptorsSupervisor.tasks_name(sup), + source_topic: AdaptorsSupervisor.source_topic(sup), + refresh_interval: 99_999_999, + warn_when_empty: false + }) end describe "packages/1" do From 8716e772e8e38951b68939146253f371a375fc04 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Mon, 14 Sep 2026 08:22:48 +0200 Subject: [PATCH 29/37] Run the adaptors subsystem outside the web app - Express the adaptor refresh interval in seconds - Give the adaptor icon cache a fixed path and a volume - Start the adaptors subsystem from the out-of-band setup commands - Start Finch when the adaptors supervisor starts on its own --- .env.example | 6 +- ADAPTORS.md | 11 +- DEPLOYMENT.md | 118 ++++++++++---------- Dockerfile | 5 + docker-compose.yml | 3 + lib/lightning/adaptors/supervisor.ex | 60 ++++++++-- lib/lightning/application.ex | 2 +- lib/lightning/config/bootstrap.ex | 6 +- lib/lightning/demo.ex | 4 +- lib/lightning/setup.ex | 31 ++++- lib/mix/tasks/kickstart.ex | 8 +- test/lightning/adaptors/supervisor_test.exs | 21 ++-- test/lightning/config/bootstrap_test.exs | 10 +- 13 files changed, 181 insertions(+), 104 deletions(-) diff --git a/.env.example b/.env.example index 915b2d1fbad..ff177b8dd2f 100644 --- a/.env.example +++ b/.env.example @@ -280,9 +280,9 @@ # HTTP receive timeout (ms) for registry/schema/icon fetches. Defaults to 30s. # ADAPTORS_NPM_HTTP_TIMEOUT=30000 # -# How often (ms) the catalogue refreshes from npm. Defaults to one hour. Set -# to 0 to disable scheduled refreshes. -# ADAPTORS_REFRESH_INTERVAL_MS=3600000 +# How often (seconds) the catalogue refreshes from npm. Defaults to one hour. +# Set to 0 to disable scheduled refreshes. +# ADAPTORS_REFRESH_INTERVAL_SECONDS=3600 # ============================================================================== # <><><> WEBHOOK RETRY SETTINGS <><><> diff --git a/ADAPTORS.md b/ADAPTORS.md index fdb7beecb93..9d6d6bbc8b7 100644 --- a/ADAPTORS.md +++ b/ADAPTORS.md @@ -46,8 +46,9 @@ mix lightning.adaptors.dump --path snapshot.json tar czf icons.tar.gz -C "$ADAPTORS_ICONS_PATH" . ``` -`ADAPTORS_ICONS_PATH` defaults to `lightning/adaptor_icons` under the temp -directory. On a release image (no Mix), dump with: +`ADAPTORS_ICONS_PATH` is `/app/priv/adaptor_icons` in the official image, and +otherwise defaults to `lightning/adaptor_icons` under the temp directory. On a +release image (no Mix), dump with: ```sh bin/lightning eval 'Lightning.Release.dump_adaptors("/path/to/snapshot.json")' @@ -89,9 +90,9 @@ download not covered here. ## Keeping the catalogue fresh -Lightning refreshes the catalogue hourly. Set `ADAPTORS_REFRESH_INTERVAL_MS` to -change that interval, or to `0` to disable scheduled refreshes. Force one -manually, on a source checkout: +Lightning refreshes the catalogue hourly. Set +`ADAPTORS_REFRESH_INTERVAL_SECONDS` to change that interval, or to `0` to +disable scheduled refreshes. Force one manually, on a source checkout: ```sh mix lightning.adaptors.refresh diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 396cf0b0936..fce2ec897b6 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -183,65 +183,65 @@ For SMTP, the following environment variables are required: ### Other config -| **Variable** | Description | -| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ADAPTORS_ICONS_PATH` | Directory the on-disk adaptor icon cache is written to. Defaults to a subdirectory of the system temp directory, which most container platforms wipe on restart; point it at a persistent volume so cached icons survive a restart. | -| `ADAPTORS_LOCAL_REPO` | Path to an OpenFn adaptors checkout, used when `ADAPTORS_STRATEGY` is `local`. Comma-separate several. The first checkout that has a package wins. See [Adaptors](ADAPTORS.md). | -| `ADAPTORS_NPM_GITHUB_REF` | Git ref of `OpenFn/adaptors` that adaptor icons are read from. Defaults to `main`. | -| `ADAPTORS_NPM_GITHUB_URL` | Raw GitHub host that adaptor icons are fetched from. Defaults to `https://raw.githubusercontent.com`. | -| `ADAPTORS_NPM_HTTP_TIMEOUT` | Receive timeout in milliseconds for npm registry, schema and icon requests. Defaults to `30000`. | -| `ADAPTORS_NPM_JSDELIVR_URL` | CDN that adaptor credential schemas are fetched from. Defaults to `https://cdn.jsdelivr.net`. | -| `ADAPTORS_NPM_REGISTRY_URL` | npm registry the adaptor catalogue is read from. Defaults to `https://registry.npmjs.org`. Set this and the two URLs above to use an internal mirror. See [Adaptors](ADAPTORS.md). | -| `ADAPTORS_PATH` | Where you store your locally installed adaptors | -| `ADAPTORS_REFRESH_INTERVAL_MS` | How often, in milliseconds, the adaptor catalogue refreshes from npm. Defaults to one hour. Set to `0` to disable scheduled refreshes. See [Adaptors](ADAPTORS.md). | -| `ADAPTORS_STRATEGY` | Where the adaptor catalogue comes from: `npm` (default) or `local`. See [Adaptors](ADAPTORS.md). | -| `ALLOW_SIGNUP` | Set to `true` to enable user access to the registration page. Set to `false` to disable new user registrations and block access to the registration page.
Default is `false`. | -| `CORS_ORIGIN` | A list of acceptable hosts for browser/cors requests (',' separated) | -| `DISABLE_DB_SSL` | In production, the use of an SSL connection to Postgres is required by default.
Setting this to `"true"` allows unencrypted connections to the database. This is strongly discouraged in a real production environment. | -| `DISABLE_DB_SSL_CERT_VERIFY` | When a SSL connection is used to connect to Postgres, the server's certificate will be verified by default.
Setting this to `"true"` disables certificate verification. This is strongly discouraged in a real production environment. | -| `EMAIL_ADMIN` | This is used as the sender email address for system emails. It is also displayed in the menu as the support email. | -| `EMAIL_SENDER_NAME` | This is displayed in the email client as the sender name for emails sent by the application. | -| `ERLANG_NODE_DISCOVERY_VIA_POSTGRES_CHANNEL_NAME` | The name of the Postgresql channel that is used when Erlang node discovery via Postgres is enabled. Defaults to `lightning-cluster` if not set. | -| `ERLANG_NODE_DISCOVERY_VIA_POSTGRES_ENABLED` | If set to `true`, Lightning will use Postgres to discover Erlang nodes. This strategy will be used in addition to other strategies that are in use. Default value is `false` | -| `IDLE_TIMEOUT` | The number of seconds that must pass without data being received before the Lightning web server kills the connection. | -| `IS_RESETTABLE_DEMO` | If set to `yes`, it allows this instance to be reset to the initial "Lightning Demo" state. Note that this will destroy _most_ of what you have in your database! | -| `K8S_HEADLESS_SERVICE` | This environment variable is automatically set if you're running on GKE and it is used to establish an Erlang node cluster. Note that if you're _not_ using Kubernetes, the "gossip" strategy is used to establish clusters. | -| `LISTEN_ADDRESS` | The address the web server should bind to. Defaults to `127.0.0.1` to block access from other machines. | -| `LOG_LEVEL` | How noisy you want the logs to be (e.g., `debug`, `info`) | -| `METRICS_RUN_PERFORMANCE_AGE_SECONDS` | The oldest a run can be to be included in Run performance metrics. | -| `METRICS_RUN_QUEUE_AGE_SECONDS` | The polling period for run queue metrics. | -| `METRICS_STALLED_RUN_THRESHOLD_SECONDS` | The length of time a Run must be in the `available` state before it is considered stalled. | -| `METRICS_UNCLAIMED_RUN_THRESHOLD_SECONDS` | The length of time a Run must be in the `available` state before it counts towards an impeded project. | -| `MIX_ENV` | Your mix env, likely `prod` for deployment | -| `NODE_ENV` | Node env, likely `production` for deployment | -| `ORIGINS` | The allowed origins for web traffic to the backend | -| `PER_WORKFLOW_CLAIM_LIMIT` | The maximum number of runs per workflow to consider during run claiming. This prevents any single workflow from dominating the processing queue while ensuring fairness across workflows.
Default is `50`. | -| `CLAIM_WORK_MEM` | PostgreSQL `work_mem` setting for the run claim query. Helps optimize complex sorting operations. Set to a valid PostgreSQL memory value (e.g., `32MB`, `64MB`, `1GB`). Set to empty string to disable.
Default: disabled in dev/test, `32MB` in production. | -| `PORT` | The port your Phoenix app runs on | -| `PROMEX_DATASOURCE_ID` | The datasource that PromEx will use if configured to push initial dashboards to Grafana. Defaults to an empty string. | -| `PROMEX_ENABLED` | Enables PromEx tracking and publishing of metrics if set to 'true' or 'yes'. Defaults to false. | -| `PROMEX_ENDPOINT_SCHEME` | The scheme needed when connecting to the Promex Endpoint. Defaults to https. | -| `PROMEX_EXPENSIVE_METRICS_ENABLED` | Certain metrics may be expensive to generate if Lightning is under load. If set to 'true', or 'yes' these metrics will be enabled. Defaults to 'false'. | -| `PROMEX_GRAFANA_HOST` | This is used when PromEx is required to push data to a Grafana instance, e.g. when PromEx sets up initial dashboards. | -| `PROMEX_GRAFANA_PASSWORD` | This is used when PromEx is required to push data to a Grafana instance, e.g. when PromEx sets up initial dashboards. | -| `PROMEX_GRAFANA_USER` | This is used when PromEx is required to push data to a Grafana instance, e.g. when PromEx sets up initial dashboards. | -| `PROMEX_METRICS_ENDPOINT_AUTHORIZATION_REQUIRED` | If set to 'true' or 'yes', the PromEx endpoint on Lightning will require consumers to provide credentials for authorization. Defaults to 'true'. | -| `PROMEX_METRICS_ENDPOINT_TOKEN` | A Bearer token that the consumer of the promEx endpoint must provide in the Authorization header. Defaults to a random series of bytes. | -| `PROMEX_UPLOAD_GRAFANA_DASHBOARDS_ON_START` | Instructs PromEx to upload iniital dashboards to a Grafana instance if set to 'true' or 'yes'. Defaults to false. | -| `PRIMARY_ENCRYPTION_KEY` | A base64 encoded 32 character long string.
See [Encryption](#encryption). | -| `QUEUE_RESULT_RETENTION_PERIOD_MINUTES` | The number of minutes to keep completed (successful) `ObanJobs` in the queue (not to be confused with runs and/or history) | -| `SECRET_KEY_BASE` | A secret key used as a base to generate secrets for encrypting and signing data. | -| `SENTRY_DSN` | If using Sentry for error monitoring, your DSN | -| `URL_HOST` | The host used for writing URLs (e.g., `demo.openfn.org`) | -| `URL_PORT` | The port, usually `443` for production | -| `URL_SCHEME` | The scheme for writing URLs (e.g., `https`) | -| `USAGE_TRACKER_HOST` | The host that receives usage tracking submissions
(defaults to https://impact.openfn.org) | -| `USAGE_TRACKING_DAILY_BATCH_SIZE` | The number of days that will be reported on with each run of `UsageTracking.DayWorker`. This will only have a noticeable effect in cases where there is a backlog or where reports are being generated retroactively (defaults to 10). | -| `USAGE_TRACKING_ENABLED` | Enables the submission of anonymized usage data to OpenFn (defaults to `true`) | -| `USAGE_TRACKING_RESUBMISSION_BATCH_SIZE` | The number of failed reports that will be submitted on each resubmission run (defaults to 10) | -| `USAGE_TRACKING_RUN_CHUNK_SIZE` | The size of each batch of runs that is streamed from the database when generating UsageTracking reports (default 100). Decreasing this may decrease memory consumption when generating reports. | -| `USAGE_TRACKING_UUIDS` | Indicates whether submissions should include cleartext UUIDs or not. Options are `cleartext` or `hashed_only`, with the default being `hashed_only`. | -| `REQUIRE_EMAIL_VERIFICATION` | Indicates whether user email addresses should be verified. Defaults to `false`. | +| **Variable** | Description | +| ------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ADAPTORS_ICONS_PATH` | Directory the on-disk adaptor icon cache is written to. The official image sets this to `/app/priv/adaptor_icons`; outside the image it defaults to a subdirectory of the system temp directory. Either way, mount a persistent volume at this path so cached icons survive a restart. | +| `ADAPTORS_LOCAL_REPO` | Path to an OpenFn adaptors checkout, used when `ADAPTORS_STRATEGY` is `local`. Comma-separate several. The first checkout that has a package wins. See [Adaptors](ADAPTORS.md). | +| `ADAPTORS_NPM_GITHUB_REF` | Git ref of `OpenFn/adaptors` that adaptor icons are read from. Defaults to `main`. | +| `ADAPTORS_NPM_GITHUB_URL` | Raw GitHub host that adaptor icons are fetched from. Defaults to `https://raw.githubusercontent.com`. | +| `ADAPTORS_NPM_HTTP_TIMEOUT` | Receive timeout in milliseconds for npm registry, schema and icon requests. Defaults to `30000`. | +| `ADAPTORS_NPM_JSDELIVR_URL` | CDN that adaptor credential schemas are fetched from. Defaults to `https://cdn.jsdelivr.net`. | +| `ADAPTORS_NPM_REGISTRY_URL` | npm registry the adaptor catalogue is read from. Defaults to `https://registry.npmjs.org`. Set this and the two URLs above to use an internal mirror. See [Adaptors](ADAPTORS.md). | +| `ADAPTORS_PATH` | Where you store your locally installed adaptors | +| `ADAPTORS_REFRESH_INTERVAL_SECONDS` | How often, in seconds, the adaptor catalogue refreshes from npm. Defaults to one hour. Set to `0` to disable scheduled refreshes. See [Adaptors](ADAPTORS.md). | +| `ADAPTORS_STRATEGY` | Where the adaptor catalogue comes from: `npm` (default) or `local`. See [Adaptors](ADAPTORS.md). | +| `ALLOW_SIGNUP` | Set to `true` to enable user access to the registration page. Set to `false` to disable new user registrations and block access to the registration page.
Default is `false`. | +| `CORS_ORIGIN` | A list of acceptable hosts for browser/cors requests (',' separated) | +| `DISABLE_DB_SSL` | In production, the use of an SSL connection to Postgres is required by default.
Setting this to `"true"` allows unencrypted connections to the database. This is strongly discouraged in a real production environment. | +| `DISABLE_DB_SSL_CERT_VERIFY` | When a SSL connection is used to connect to Postgres, the server's certificate will be verified by default.
Setting this to `"true"` disables certificate verification. This is strongly discouraged in a real production environment. | +| `EMAIL_ADMIN` | This is used as the sender email address for system emails. It is also displayed in the menu as the support email. | +| `EMAIL_SENDER_NAME` | This is displayed in the email client as the sender name for emails sent by the application. | +| `ERLANG_NODE_DISCOVERY_VIA_POSTGRES_CHANNEL_NAME` | The name of the Postgresql channel that is used when Erlang node discovery via Postgres is enabled. Defaults to `lightning-cluster` if not set. | +| `ERLANG_NODE_DISCOVERY_VIA_POSTGRES_ENABLED` | If set to `true`, Lightning will use Postgres to discover Erlang nodes. This strategy will be used in addition to other strategies that are in use. Default value is `false` | +| `IDLE_TIMEOUT` | The number of seconds that must pass without data being received before the Lightning web server kills the connection. | +| `IS_RESETTABLE_DEMO` | If set to `yes`, it allows this instance to be reset to the initial "Lightning Demo" state. Note that this will destroy _most_ of what you have in your database! | +| `K8S_HEADLESS_SERVICE` | This environment variable is automatically set if you're running on GKE and it is used to establish an Erlang node cluster. Note that if you're _not_ using Kubernetes, the "gossip" strategy is used to establish clusters. | +| `LISTEN_ADDRESS` | The address the web server should bind to. Defaults to `127.0.0.1` to block access from other machines. | +| `LOG_LEVEL` | How noisy you want the logs to be (e.g., `debug`, `info`) | +| `METRICS_RUN_PERFORMANCE_AGE_SECONDS` | The oldest a run can be to be included in Run performance metrics. | +| `METRICS_RUN_QUEUE_AGE_SECONDS` | The polling period for run queue metrics. | +| `METRICS_STALLED_RUN_THRESHOLD_SECONDS` | The length of time a Run must be in the `available` state before it is considered stalled. | +| `METRICS_UNCLAIMED_RUN_THRESHOLD_SECONDS` | The length of time a Run must be in the `available` state before it counts towards an impeded project. | +| `MIX_ENV` | Your mix env, likely `prod` for deployment | +| `NODE_ENV` | Node env, likely `production` for deployment | +| `ORIGINS` | The allowed origins for web traffic to the backend | +| `PER_WORKFLOW_CLAIM_LIMIT` | The maximum number of runs per workflow to consider during run claiming. This prevents any single workflow from dominating the processing queue while ensuring fairness across workflows.
Default is `50`. | +| `CLAIM_WORK_MEM` | PostgreSQL `work_mem` setting for the run claim query. Helps optimize complex sorting operations. Set to a valid PostgreSQL memory value (e.g., `32MB`, `64MB`, `1GB`). Set to empty string to disable.
Default: disabled in dev/test, `32MB` in production. | +| `PORT` | The port your Phoenix app runs on | +| `PROMEX_DATASOURCE_ID` | The datasource that PromEx will use if configured to push initial dashboards to Grafana. Defaults to an empty string. | +| `PROMEX_ENABLED` | Enables PromEx tracking and publishing of metrics if set to 'true' or 'yes'. Defaults to false. | +| `PROMEX_ENDPOINT_SCHEME` | The scheme needed when connecting to the Promex Endpoint. Defaults to https. | +| `PROMEX_EXPENSIVE_METRICS_ENABLED` | Certain metrics may be expensive to generate if Lightning is under load. If set to 'true', or 'yes' these metrics will be enabled. Defaults to 'false'. | +| `PROMEX_GRAFANA_HOST` | This is used when PromEx is required to push data to a Grafana instance, e.g. when PromEx sets up initial dashboards. | +| `PROMEX_GRAFANA_PASSWORD` | This is used when PromEx is required to push data to a Grafana instance, e.g. when PromEx sets up initial dashboards. | +| `PROMEX_GRAFANA_USER` | This is used when PromEx is required to push data to a Grafana instance, e.g. when PromEx sets up initial dashboards. | +| `PROMEX_METRICS_ENDPOINT_AUTHORIZATION_REQUIRED` | If set to 'true' or 'yes', the PromEx endpoint on Lightning will require consumers to provide credentials for authorization. Defaults to 'true'. | +| `PROMEX_METRICS_ENDPOINT_TOKEN` | A Bearer token that the consumer of the promEx endpoint must provide in the Authorization header. Defaults to a random series of bytes. | +| `PROMEX_UPLOAD_GRAFANA_DASHBOARDS_ON_START` | Instructs PromEx to upload iniital dashboards to a Grafana instance if set to 'true' or 'yes'. Defaults to false. | +| `PRIMARY_ENCRYPTION_KEY` | A base64 encoded 32 character long string.
See [Encryption](#encryption). | +| `QUEUE_RESULT_RETENTION_PERIOD_MINUTES` | The number of minutes to keep completed (successful) `ObanJobs` in the queue (not to be confused with runs and/or history) | +| `SECRET_KEY_BASE` | A secret key used as a base to generate secrets for encrypting and signing data. | +| `SENTRY_DSN` | If using Sentry for error monitoring, your DSN | +| `URL_HOST` | The host used for writing URLs (e.g., `demo.openfn.org`) | +| `URL_PORT` | The port, usually `443` for production | +| `URL_SCHEME` | The scheme for writing URLs (e.g., `https`) | +| `USAGE_TRACKER_HOST` | The host that receives usage tracking submissions
(defaults to https://impact.openfn.org) | +| `USAGE_TRACKING_DAILY_BATCH_SIZE` | The number of days that will be reported on with each run of `UsageTracking.DayWorker`. This will only have a noticeable effect in cases where there is a backlog or where reports are being generated retroactively (defaults to 10). | +| `USAGE_TRACKING_ENABLED` | Enables the submission of anonymized usage data to OpenFn (defaults to `true`) | +| `USAGE_TRACKING_RESUBMISSION_BATCH_SIZE` | The number of failed reports that will be submitted on each resubmission run (defaults to 10) | +| `USAGE_TRACKING_RUN_CHUNK_SIZE` | The size of each batch of runs that is streamed from the database when generating UsageTracking reports (default 100). Decreasing this may decrease memory consumption when generating reports. | +| `USAGE_TRACKING_UUIDS` | Indicates whether submissions should include cleartext UUIDs or not. Options are `cleartext` or `hashed_only`, with the default being `hashed_only`. | +| `REQUIRE_EMAIL_VERIFICATION` | Indicates whether user email addresses should be verified. Defaults to `false`. | ### AI Chat diff --git a/Dockerfile b/Dockerfile index 6ffcbb0a054..4319eeff1f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,12 +115,17 @@ RUN chown lightning /app ENV MIX_ENV="prod" ENV ERL_FLAGS=${ERL_FLAGS} ENV ADAPTORS_PATH=/app/priv/openfn +ENV ADAPTORS_ICONS_PATH=/app/priv/adaptor_icons # Only copy the final release and the adaptor directory from the build stage COPY --from=builder --chown=lightning:root /app/_build/${MIX_ENV}/rel/lightning ./ COPY --from=builder --chown=lightning:root /app/priv/openfn ./priv/openfn COPY --from=builder --chown=lightning:root /app/priv/github ./priv/github +# A new volume mounted here inherits this directory's ownership, so the +# non-root runtime user can write to it. +RUN mkdir -p ${ADAPTORS_ICONS_PATH} && chown lightning:root ${ADAPTORS_ICONS_PATH} + USER lightning ENV COMMIT=${COMMIT} diff --git a/docker-compose.yml b/docker-compose.yml index 476b65a47b5..29f9ae10140 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ x-lightning: &default-app tty: ${TTY:-false} volumes: - '${LIGHTNING_VOLUME:-./priv/static:/app/priv/static}' + - 'adaptor_icons:/app/priv/adaptor_icons' services: postgres: @@ -39,6 +40,7 @@ services: memory: '${DOCKER_WEB_MEMORY:-0}' environment: - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/lightning_dev + - ADAPTORS_ICONS_PATH=/app/priv/adaptor_icons depends_on: - postgres healthcheck: @@ -69,3 +71,4 @@ services: volumes: postgres: {} + adaptor_icons: {} diff --git a/lib/lightning/adaptors/supervisor.ex b/lib/lightning/adaptors/supervisor.ex index 077f619015c..25c335fb539 100644 --- a/lib/lightning/adaptors/supervisor.ex +++ b/lib/lightning/adaptors/supervisor.ex @@ -19,7 +19,8 @@ defmodule Lightning.Adaptors.Supervisor do Options: - * `:name` - required; every child name derives from it + * `:name` - every child name derives from it, defaulting to + `Lightning.Adaptors.Config.default_instance/0` * `:strategy` - `Lightning.Adaptors.Strategy` implementation, defaulting to `Lightning.Adaptors.Config.strategy/0` * `:lock_key` - `HighlanderPG` advisory-lock key, defaulting to @@ -34,8 +35,37 @@ defmodule Lightning.Adaptors.Supervisor do """ @spec start_link(keyword()) :: Supervisor.on_start() def start_link(opts) do - name = Keyword.fetch!(opts, :name) - Supervisor.start_link(__MODULE__, opts, name: name) + opts = Keyword.put_new(opts, :name, Config.default_instance()) + Supervisor.start_link(__MODULE__, opts, name: opts[:name]) + end + + @doc """ + Starts an instance, or returns the running one. For entry points that + may run either against a cold BEAM or inside the booted application. + """ + @spec ensure_started(keyword()) :: {:ok, pid()} | {:error, term()} + def ensure_started(opts \\ []) do + {:ok, _} = Application.ensure_all_started(:cachex) + ensure_finch() + + case start_link(opts) do + {:error, {:already_started, pid}} -> {:ok, pid} + other -> other + end + end + + # Tesla is configured against a named Finch pool that only the full + # application starts; a strategy fetching over HTTP without it fails with + # "unknown registry". + defp ensure_finch do + with {Tesla.Adapter.Finch, opts} <- Application.get_env(:tesla, :adapter), + name when is_atom(name) and not is_nil(name) <- opts[:name], + nil <- Process.whereis(name) do + {:ok, _} = Application.ensure_all_started(:finch) + Finch.start_link(name: name) + end + + :ok end @impl true @@ -109,16 +139,30 @@ defmodule Lightning.Adaptors.Supervisor do supervisor has started under that name. """ @spec strategy(atom()) :: module() - def strategy(name) do - :persistent_term.get(meta_key(name)).strategy - end + def strategy(name), do: meta(name).strategy @doc """ Returns the source (`:npm | :local`) of the supervisor named `name`. """ @spec source(atom()) :: :npm | :local - def source(name) do - :persistent_term.get(meta_key(name)).source + def source(name), do: meta(name).source + + defp meta(name) do + case :persistent_term.get(meta_key(name), nil) do + nil -> + raise """ + The adaptors subsystem is not running under the name #{inspect(name)}. + + It starts with the Lightning application. Entry points that run \ + without it - mix tasks, `mix run --no-start`, `bin/lightning eval` - \ + must start it themselves, after the repo is up: + + Lightning.Adaptors.Supervisor.ensure_started() + """ + + meta -> + meta + end end @doc """ diff --git a/lib/lightning/application.ex b/lib/lightning/application.ex index 1523c1fec9f..5c8f1d73702 100644 --- a/lib/lightning/application.ex +++ b/lib/lightning/application.ex @@ -175,7 +175,7 @@ defmodule Lightning.Application do Lightning.Workflows.Presence, LightningWeb.WorkerPresence, adaptor_service_childspec, - {Lightning.Adaptors.Supervisor, name: Lightning.Adaptors}, + Lightning.Adaptors.Supervisor, schema_reconciler_childspec, {Lightning.TaskWorker, name: :cli_task_worker}, {Lightning.Runtime.RuntimeManager, diff --git a/lib/lightning/config/bootstrap.ex b/lib/lightning/config/bootstrap.ex index ba114600ca7..b8fd9076054 100644 --- a/lib/lightning/config/bootstrap.ex +++ b/lib/lightning/config/bootstrap.ex @@ -1049,7 +1049,8 @@ defmodule Lightning.Config.Bootstrap do icon_path: env!("ADAPTORS_ICONS_PATH", :string, nil) |> expand_or_nil(), refresh_interval: - env!("ADAPTORS_REFRESH_INTERVAL_MS", :integer?, nil) + env!("ADAPTORS_REFRESH_INTERVAL_SECONDS", :integer?, nil) + |> seconds_to_ms() ] |> Enum.reject(fn {_key, value} -> is_nil(value) end) @@ -1100,6 +1101,9 @@ defmodule Lightning.Config.Bootstrap do |> Enum.map(&Path.expand/1) end + defp seconds_to_ms(nil), do: nil + defp seconds_to_ms(seconds) when is_integer(seconds), do: seconds * 1000 + defp expand_or_nil(nil), do: nil defp expand_or_nil(path) when is_binary(path), do: Path.expand(path) diff --git a/lib/lightning/demo.ex b/lib/lightning/demo.ex index 61112d6de7e..ebecf0c2096 100644 --- a/lib/lightning/demo.ex +++ b/lib/lightning/demo.ex @@ -12,10 +12,8 @@ defmodule Lightning.Demo do """ def reset_demo do if Application.get_env(:lightning, :is_resettable_demo) do - {:ok, _pid} = Lightning.Setup.ensure_minimum_setup() - {:ok, _, _} = - Ecto.Migrator.with_repo(Lightning.Repo, fn _repo -> + Lightning.Setup.with_minimum_setup(fn -> SetupUtils.tear_down(destroy_super: true) SetupUtils.setup_demo(create_super: true) end) diff --git a/lib/lightning/setup.ex b/lib/lightning/setup.ex index 13c17f216b8..33ccd2dc69d 100644 --- a/lib/lightning/setup.ex +++ b/lib/lightning/setup.ex @@ -19,17 +19,36 @@ defmodule Lightning.Setup do @spec setup_user(map(), String.t() | nil, list(map()) | nil) :: {:ok, any(), any()} | {:error, any()} def setup_user(user, token \\ nil, credentials \\ nil) do - {:ok, _pid} = Lightning.Setup.ensure_minimum_setup() - {:ok, _, _} = - Ecto.Migrator.with_repo(Lightning.Repo, fn _repo -> + with_minimum_setup(fn -> SetupUtils.setup_user(user, token, credentials) end) end @doc """ - Set up the bare minimum so that commands can be executed against the repo. + Runs `fun` with the bare minimum an out-of-band command needs: the vault + (credential bodies are encrypted), a stub PubSub, the repo, and the adaptors + subsystem (job and credential validation resolves adaptor names against it). + The endpoint stays down - these commands often run against an instance that + is already serving traffic, and booting it here would fight the running + server for the port. + + Idempotent, so it is equally safe on a cold BEAM or inside the booted + application. """ + @spec with_minimum_setup((-> result)) :: + {:ok, result, [atom()]} | {:error, term()} + when result: term() + def with_minimum_setup(fun) when is_function(fun, 0) do + {:ok, _pid} = ensure_minimum_setup() + + Ecto.Migrator.with_repo(Lightning.Repo, fn _repo -> + {:ok, _pid} = Lightning.Adaptors.Supervisor.ensure_started() + fun.() + end) + end + + @deprecated "Use with_minimum_setup/1 instead" def ensure_minimum_setup do Lightning.Release.load_app() @@ -39,7 +58,9 @@ defmodule Lightning.Setup do name: Lightning.PubSub, adapter: Lightning.Setup.FakePubSub}, {Lightning.Vault, Application.get_env(:lightning, Lightning.Vault, [])} ] - |> Enum.reject(fn {mod, _} -> Process.whereis(mod) end) + |> Enum.reject(fn {mod, opts} -> + Process.whereis(Keyword.get(opts, :name, mod)) + end) Supervisor.start_link(children, strategy: :one_for_one) end diff --git a/lib/mix/tasks/kickstart.ex b/lib/mix/tasks/kickstart.ex index 9c3ddb274e9..e59d00c319b 100644 --- a/lib/mix/tasks/kickstart.ex +++ b/lib/mix/tasks/kickstart.ex @@ -56,14 +56,8 @@ defmodule Mix.Tasks.Lightning.Kickstart do Mix.Task.run("app.config") - # Start the repo, the vault (credential bodies are encrypted) and a stub - # PubSub — but not the endpoint: seeding often runs against an instance - # that is already serving traffic, and booting it here would fight the - # running server for the port. - {:ok, _pid} = Lightning.Setup.ensure_minimum_setup() - {:ok, result, _apps} = - Ecto.Migrator.with_repo(Lightning.Repo, fn _repo -> + Lightning.Setup.with_minimum_setup(fn -> Lightning.Kickstart.run_file(path, opts) end) diff --git a/test/lightning/adaptors/supervisor_test.exs b/test/lightning/adaptors/supervisor_test.exs index a05304c8ea2..1caf2cb7a4b 100644 --- a/test/lightning/adaptors/supervisor_test.exs +++ b/test/lightning/adaptors/supervisor_test.exs @@ -4,15 +4,22 @@ defmodule Lightning.Adaptors.SupervisorTest do alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor describe "start_link/1" do - test "raises KeyError when :name is missing" do - assert_raise KeyError, ~r/key :name not found/, fn -> - AdaptorsSupervisor.start_link([]) - end + test ":name defaults to the configured instance" do + pid = Process.whereis(Lightning.Adaptors.Config.default_instance()) + + assert {:error, {:already_started, ^pid}} = + AdaptorsSupervisor.start_link([]) + + assert {:ok, ^pid} = AdaptorsSupervisor.ensure_started() end + end - test "raises KeyError when opts has no :name key" do - assert_raise KeyError, fn -> - AdaptorsSupervisor.start_link(strategy: :ignored) + describe "source/1 and strategy/1" do + test "name the unstarted instance and how to start it" do + for fun <- [&AdaptorsSupervisor.source/1, &AdaptorsSupervisor.strategy/1] do + assert_raise RuntimeError, + ~r/not running under the name :nope.*ensure_started\(\)/s, + fn -> fun.(:nope) end end end end diff --git a/test/lightning/config/bootstrap_test.exs b/test/lightning/config/bootstrap_test.exs index c04a86ff247..8a2394d981e 100644 --- a/test/lightning/config/bootstrap_test.exs +++ b/test/lightning/config/bootstrap_test.exs @@ -664,8 +664,8 @@ defmodule Lightning.Config.BootstrapTest do end describe "adaptors refresh interval" do - test "ADAPTORS_REFRESH_INTERVAL_MS sets refresh_interval when present" do - Dotenvy.source([%{"ADAPTORS_REFRESH_INTERVAL_MS" => "60000"}]) + test "ADAPTORS_REFRESH_INTERVAL_SECONDS sets refresh_interval in ms" do + Dotenvy.source([%{"ADAPTORS_REFRESH_INTERVAL_SECONDS" => "60"}]) Bootstrap.configure() @@ -673,8 +673,8 @@ defmodule Lightning.Config.BootstrapTest do 60_000 end - test "ADAPTORS_REFRESH_INTERVAL_MS accepts 0 to disable the scheduler" do - Dotenvy.source([%{"ADAPTORS_REFRESH_INTERVAL_MS" => "0"}]) + test "ADAPTORS_REFRESH_INTERVAL_SECONDS accepts 0 to disable the scheduler" do + Dotenvy.source([%{"ADAPTORS_REFRESH_INTERVAL_SECONDS" => "0"}]) Bootstrap.configure() @@ -693,7 +693,7 @@ defmodule Lightning.Config.BootstrapTest do end test "does not set refresh_interval when set but empty" do - Dotenvy.source([%{"ADAPTORS_REFRESH_INTERVAL_MS" => ""}]) + Dotenvy.source([%{"ADAPTORS_REFRESH_INTERVAL_SECONDS" => ""}]) Bootstrap.configure() From 6d14dfef4cf77090ef184e6697d5962f86524397 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Mon, 14 Sep 2026 08:36:56 +0200 Subject: [PATCH 30/37] Gate catalogue reads on the first load and report source failures - Gate every catalogue read on the first load, and give the facade tuples - Count the adaptors a refresh tick failed to write - Settle the first-load gate on a source that lists no adaptors - Stop reporting an unreachable catalogue as a refused adaptor --- lib/lightning/adaptor_service.ex | 57 ++++--- lib/lightning/adaptors.ex | 130 ++++++++-------- lib/lightning/adaptors/config.ex | 4 +- lib/lightning/adaptors/scheduler.ex | 46 +++++- lib/lightning/adaptors/store.ex | 95 +++++++++++- lib/lightning/credentials.ex | 22 ++- lib/lightning/credentials/credential.ex | 14 +- lib/lightning/metadata_service.ex | 3 + lib/lightning/setup.ex | 4 + lib/lightning/workflows/job.ex | 9 +- .../controllers/adaptor_icon_controller.ex | 4 +- test/lightning/adaptor_service_test.exs | 19 ++- test/lightning/adaptors/readiness_test.exs | 12 +- test/lightning/adaptors/store_test.exs | 143 ++++++++++++++++-- test/lightning/adaptors_test.exs | 129 +++++++++++----- test/support/adaptor_test_helpers.ex | 2 +- 16 files changed, 512 insertions(+), 181 deletions(-) diff --git a/lib/lightning/adaptor_service.ex b/lib/lightning/adaptor_service.ex index 86974a560b6..f672718fdd9 100644 --- a/lib/lightning/adaptor_service.ex +++ b/lib/lightning/adaptor_service.ex @@ -243,7 +243,10 @@ defmodule Lightning.AdaptorService do @spec find_adaptor(Agent.agent(), package :: String.t()) :: InstalledAdaptor.t() | nil def find_adaptor(agent, package) when is_binary(package) do - find_adaptor(agent, resolve_package_name(package)) + case Adaptors.parse_spec(package) do + {:ok, package_spec} -> find_adaptor(agent, package_spec) + {:error, :invalid_format} -> nil + end end @spec find_adaptor(Agent.agent(), package_spec()) :: InstalledAdaptor.t() | nil @@ -291,37 +294,48 @@ defmodule Lightning.AdaptorService do @spec install(Agent.agent(), binary()) :: {:ok, InstalledAdaptor.t()} - | {:error, :adaptor_not_permitted} + | {:error, :adaptor_not_permitted | :invalid_format} + | {:error, {:catalogue_unavailable, term()}} | {:error, {Collectable.t(), exit_status :: non_neg_integer}} def install(agent, package) when is_binary(package) do - install(agent, resolve_package_name(package)) + with {:ok, package_spec} <- Adaptors.parse_spec(package) do + install(agent, package_spec) + end end @spec install(Agent.agent(), package_spec()) :: {:ok, InstalledAdaptor.t()} | {:error, :adaptor_not_permitted} + | {:error, {:catalogue_unavailable, term()}} | {:error, {Collectable.t(), exit_status :: non_neg_integer}} def install(agent, {package_name, _version} = package_spec) do - if known?(package_name) do - agent - |> find_adaptor(package_spec) - |> case do - nil -> install!(agent, package_spec) - existing -> {:ok, existing} - end - else - Logger.warning( - "Refusing to install non-permitted adaptor: #{inspect(package_name)}" - ) - - {:error, :adaptor_not_permitted} + case Adaptors.fetch_adaptor(package_name) do + {:ok, _package} -> + case find_adaptor(agent, package_spec) do + nil -> install!(agent, package_spec) + existing -> {:ok, existing} + end + + {:error, :not_found} -> + Logger.warning( + "Refusing to install non-permitted adaptor: #{inspect(package_name)}" + ) + + {:error, :adaptor_not_permitted} + + # A catalogue that cannot answer has not said no. Reporting that as + # a policy refusal sends whoever debugs the failed install to the + # allowlist for what is a timeout or an unreachable source. + {:error, reason} -> + Logger.warning( + "Cannot check #{inspect(package_name)} against the adaptor " <> + "catalogue: #{inspect(reason)}" + ) + + {:error, {:catalogue_unavailable, reason}} end end - defp known?(nil), do: false - - defp known?(name), do: match?({:ok, _}, Adaptors.fetch_adaptor(name)) - @spec install!(Agent.agent(), package_spec()) :: {:ok, InstalledAdaptor.t()} | {:error, {Collectable.t(), exit_status :: non_neg_integer}} @@ -358,9 +372,6 @@ defmodule Lightning.AdaptorService do end end - def resolve_package_name(package_name) when is_binary(package_name), - do: Adaptors.parse_spec(package_name) - @doc """ Turns a package name and version into a string for NPM. diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index 76ab2883ec6..2b47ad86702 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -95,31 +95,53 @@ defmodule Lightning.Adaptors do Returns the credential schema of the adaptor named `pkg`, as a JSON binary. An adaptor with no schema yields `"{}"` and an unknown name is `{:error, :not_found}`. + + Against a catalogue that has never loaded, waits for the first load; + the other errors are then those of `fetch_adaptor/2`. """ - @spec schema(atom(), String.t()) :: {:ok, String.t()} | {:error, term()} - def schema(sup \\ Config.default_instance(), pkg), do: Store.schema(sup, pkg) + @spec schema(atom(), String.t()) :: + {:ok, String.t()} + | {:error, :not_found | :timeout | :unavailable | :not_ready} + def schema(sup \\ Config.default_instance(), pkg), + do: Store.schema(sup, pkg) @doc """ Resolves a possibly-legacy short adaptor name (e.g. `"http"`) to its full npm package name (`"@openfn/language-http"`), if the full name resolves in - the catalogue. Returns `name` unchanged if it already resolves, or if - neither form does. + the catalogue. Returns `{:ok, name}` unchanged if it already resolves, or + if neither form does: a loaded catalogue that knows neither is a real + answer, and the name is left as given. `"raw"` and `"oauth"` are sentinels, not adaptor names, and are returned unchanged without consulting the catalogue. + + Waits for the catalogue's first load if it has never loaded, and returns + the `fetch_adaptor/2` errors other than `:not_found` when it cannot get + an answer at all. """ - @spec resolve_name(atom(), String.t()) :: String.t() + @spec resolve_name(atom(), String.t()) :: + {:ok, String.t()} | {:error, :timeout | :unavailable | :not_ready} def resolve_name(sup \\ Config.default_instance(), name) - def resolve_name(_sup, name) when name in ["raw", "oauth"], do: name + def resolve_name(_sup, name) when name in ["raw", "oauth"], do: {:ok, name} def resolve_name(sup, name) do + case fetch_adaptor(sup, name) do + {:ok, _package} -> {:ok, name} + {:error, :not_found} -> resolve_short_name(sup, name) + {:error, _reason} = error -> error + end + end + + defp resolve_short_name(_sup, "@" <> _scoped = name), do: {:ok, name} + + defp resolve_short_name(sup, name) do full = PackageName.full_name(name) - cond do - get_adaptor(sup, name) -> name - not String.starts_with?(name, "@") and get_adaptor(sup, full) -> full - true -> name + case fetch_adaptor(sup, full) do + {:ok, _package} -> {:ok, full} + {:error, :not_found} -> {:ok, name} + {:error, _reason} = error -> error end end @@ -150,20 +172,12 @@ defmodule Lightning.Adaptors do Store.catalogue(sup) end - @doc """ - Returns the adaptor named `name`, or `nil`. - - Takes a bare package name, not a spec; see `parse_spec/1`. Never waits - for the catalogue to load; see `fetch_adaptor/2` for that. - """ - @spec get_adaptor(atom(), String.t()) :: Package.t() | nil - def get_adaptor(sup \\ Config.default_instance(), name) when is_binary(name), - do: lookup(sup, name) - @doc """ Returns `{:ok, adaptor}` for the adaptor named `name`, waiting for the catalogue's first load if it has never loaded. + Takes a bare package name, not a spec; see `parse_spec/1`. + Errors: * `{:error, :not_found}` - the loaded catalogue has no such adaptor @@ -177,40 +191,28 @@ defmodule Lightning.Adaptors do | {:error, :not_found | :timeout | :unavailable | :not_ready} def fetch_adaptor(sup \\ Config.default_instance(), name) when is_binary(name) do - case lookup(sup, name) do - %Package{} = package -> - {:ok, package} - - nil -> - if ready?(sup), - do: {:error, :not_found}, - else: load_then_fetch(sup, name) - end - end + case Store.packages(sup) do + {:ok, metas} -> + resolve_meta(sup, name, Enum.find(metas, &(&1.name == name))) + + {:error, reason} when reason in [:timeout, :unavailable, :not_ready] -> + {:error, reason} - defp load_then_fetch(sup, name) do - with :ok <- load(sup) do - case lookup(sup, name) do - %Package{} = package -> {:ok, package} - nil -> {:error, :not_found} - end + # Any other failure is the cache's, and says nothing about the row. + {:error, _cache} -> + resolve_meta(sup, name, nil) end end - # Cache first, then the row itself: the cached list can lag a Scheduler - # write until the Invalidator drops it. - defp lookup(sup, name) do + # The cached listing first, then the row itself: the listing can lag a + # Scheduler write until the Invalidator drops it, and it leaves out the + # excluded and deprecated names a job may still be using. + defp resolve_meta(sup, name, cached) do source = AdaptorsSupervisor.source(sup) - cached = - case Store.packages(sup) do - {:ok, metas} -> Enum.find(metas, &(&1.name == name)) - {:error, _} -> nil - end - case cached || Catalogue.get_package_meta(name, source) do - nil -> nil - meta -> to_package(meta, source) + nil -> {:error, :not_found} + meta -> {:ok, to_package(meta, source)} end end @@ -226,33 +228,21 @@ defmodule Lightning.Adaptors do """ @spec ensure_loaded(atom()) :: :ok | {:error, :timeout | :unavailable | :not_ready} - def ensure_loaded(sup \\ Config.default_instance()) do - if ready?(sup), do: :ok, else: load(sup) - end - - defp load(sup) do - case refresh(sup, await: true) do - {:error, :timeout} -> {:error, :timeout} - {:error, :unavailable} -> {:error, :unavailable} - # A successful cycle can still leave the source empty, and a failed - # one can land on rows a seed already wrote. - _ -> if ready?(sup), do: :ok, else: {:error, :not_ready} - end - end - - defp ready?(sup), - do: Catalogue.max_checked_at(AdaptorsSupervisor.source(sup)) != nil + def ensure_loaded(sup \\ Config.default_instance()), + do: Store.ensure_loaded(sup) @doc """ - Splits an adaptor spec into `{name, version}`, with `version` `nil` when - the spec carries none, and `{nil, nil}` for a malformed spec. + Splits an adaptor spec into `{:ok, {name, version}}`, with `version` `nil` + when the spec carries none, or `{:error, :invalid_format}` for a spec that + is not a package name plus an optional `@version`. """ - @spec parse_spec(String.t()) :: {String.t() | nil, String.t() | nil} + @spec parse_spec(String.t()) :: + {:ok, {String.t(), String.t() | nil}} | {:error, :invalid_format} def parse_spec(spec) when is_binary(spec) do case Regex.run(PackageName.strict_format(), spec) do - [_, name, version] -> {name, version} - [_, name] -> {name, nil} - _ -> {nil, nil} + [_, name, version] -> {:ok, {name, version}} + [_, name] -> {:ok, {name, nil}} + _ -> {:error, :invalid_format} end end @@ -280,7 +270,7 @@ defmodule Lightning.Adaptors do source = AdaptorsSupervisor.source(sup) case parse_spec(spec) do - {name, "latest"} when source != :local -> + {:ok, {name, "latest"}} when source != :local -> with {:ok, %Package{latest_version: latest}} <- fetch_adaptor(sup, name) do {:ok, PackageName.to_wire(spec, source: source, latest: latest)} end diff --git a/lib/lightning/adaptors/config.ex b/lib/lightning/adaptors/config.ex index 8bee241e145..337b3f62093 100644 --- a/lib/lightning/adaptors/config.ex +++ b/lib/lightning/adaptors/config.ex @@ -13,7 +13,7 @@ defmodule Lightning.Adaptors.Config do @default_refresh_interval :timer.hours(1) @default_cache_timeout_ms 15_000 @default_icon_path {:tmp, "lightning/adaptor_icons"} - @default_first_load_timeout :timer.seconds(60) + @default_first_load_timeout :timer.seconds(90) @doc """ The supervisor instance public `Lightning.Adaptors` functions read @@ -102,7 +102,7 @@ defmodule Lightning.Adaptors.Config do @doc """ Bound, in milliseconds, on how long `Lightning.Adaptors.ensure_loaded/1` and `Lightning.Adaptors.fetch_adaptor/2` block waiting for the - catalogue's first load. Defaults to 60 seconds. + catalogue's first load. Defaults to 90 seconds. """ @spec first_load_timeout() :: non_neg_integer() def first_load_timeout do diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 006a9e80d08..9c9dccaef03 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -77,7 +77,8 @@ defmodule Lightning.Adaptors.Scheduler do @typedoc """ One refresh cycle's tallies: adaptors the upstream listing returned, how many of those had a changed `latest_version`, how many were - fetched and persisted, and how many per-adaptor fetches failed. + fetched and persisted, and how many adaptors the cycle failed to fetch + or to write. """ @type refresh_counts :: %{ listed: non_neg_integer(), @@ -91,8 +92,9 @@ defmodule Lightning.Adaptors.Scheduler do complete. Returns `{:ok, counts}` when the listing succeeded (per-adaptor fetch - failures are counted in `counts.errors`), `{:error, reason}` when it - failed, or `{:error, {:refresh_failed, reason}}` when the cycle crashed. + and upsert failures are counted in `counts.errors`), `{:error, reason}` + when it failed, or `{:error, {:refresh_failed, reason}}` when the cycle + crashed. """ @spec await_refresh(GenServer.server(), timeout()) :: {:ok, refresh_counts()} @@ -116,6 +118,29 @@ defmodule Lightning.Adaptors.Scheduler do GenServer.call(scheduler_name, :refresh_icons, 120_000) end + @doc """ + Whether a cycle has completed against a source that listed no adaptors + at all, since this Scheduler started. + + That is the one outcome no row can record: an upstream answering with an + empty list has told us there are no adaptors, and the empty catalogue it + leaves behind is loaded rather than unloaded. Every other completed + cycle leaves rows, which answer for themselves and keep answering after + a restart — so this deliberately says nothing about them, and a source + whose rows are later deleted reloads as it did before. + + A cycle that failed to list, fetch or write is not a completed one: we + cannot tell a source with nothing in it from one we could not read. + + Answers `false` for a Scheduler that is unreachable. + """ + @spec completed?(GenServer.server()) :: boolean() + def completed?(scheduler_name) do + GenServer.call(scheduler_name, :completed?) + catch + :exit, _reason -> false + end + @impl true def init(opts) do sup = Keyword.fetch!(opts, :sup) @@ -137,6 +162,7 @@ defmodule Lightning.Adaptors.Scheduler do tasks: tasks, checked_at: checked_at, refresh: nil, + completed?: false, waiters: [], package_refreshes: %{}, icon_refreshes: %{} @@ -242,7 +268,10 @@ defmodule Lightning.Adaptors.Scheduler do Enum.each(state.waiters, &GenServer.reply(&1, result)) - {:noreply, %{state | refresh: nil, waiters: []}} + completed? = + state.completed? or match?({:ok, %{listed: 0, errors: 0}}, result) + + {:noreply, %{state | refresh: nil, completed?: completed?, waiters: []}} end def handle_info( @@ -310,6 +339,10 @@ defmodule Lightning.Adaptors.Scheduler do end @impl true + def handle_call(:completed?, _from, state) do + {:reply, state.completed?, state} + end + def handle_call(:refresh_now, _from, state) do Logger.info("Adaptors[#{state.source}]: refresh_now requested") {:reply, :ok, maybe_start_refresh(state)} @@ -401,7 +434,7 @@ defmodule Lightning.Adaptors.Scheduler do case strategy.list_adaptors() do {:ok, upstream} -> - {fetched, changed, errors} = + {fetched, changed, fetch_errors} = state.tasks |> Task.Supervisor.async_stream_nolink( upstream, @@ -440,7 +473,8 @@ defmodule Lightning.Adaptors.Scheduler do not_modified = count_not_modified(icons) listed = length(upstream) - touched = listed - changed - errors + touched = listed - changed - fetch_errors + errors = fetch_errors + (changed - persisted) duration_ms = System.monotonic_time(:millisecond) - started_at Logger.info( diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index 05c7864e0c7..415f75dad2a 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -10,12 +10,24 @@ defmodule Lightning.Adaptors.Store do fetching the bytes from the strategy on the first miss. `catalogue/1` caches the picker payload already rendered, together with the ETag stamp that describes it. + + ## The first load + + Every read is gated on the catalogue having loaded at least once. A read + that comes back with data answers immediately; one that comes back empty + or not-found asks whether the catalogue has ever loaded, and if it has + not, triggers the first load, waits for it and reads again. An empty + answer from a catalogue that has loaded is a real answer and is returned + as-is — including the empty catalogue a source with no adaptors leaves + behind, which the Scheduler reports as loaded despite there being no row + to find. """ alias Lightning.Adaptors.Catalogue alias Lightning.Adaptors.Config alias Lightning.Adaptors.IconCache alias Lightning.Adaptors.IconField + alias Lightning.Adaptors.Scheduler alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor alias LightningWeb.AdaptorIconURL @@ -54,7 +66,9 @@ defmodule Lightning.Adaptors.Store do `{:error, :not_found}`. """ @spec schema(sup(), String.t()) :: {:ok, String.t()} | {:error, term()} - def schema(sup, name) do + def schema(sup, name), do: gated(sup, fn -> read_schema(sup, name) end) + + defp read_schema(sup, name) do cache = AdaptorsSupervisor.cache_name(sup) source = AdaptorsSupervisor.source(sup) @@ -85,11 +99,15 @@ defmodule Lightning.Adaptors.Store do @spec icon(sup(), String.t(), :square | :rectangle) :: {:ok, Path.t()} | {:error, :not_found | term()} def icon(sup, name, shape) when shape in [:square, :rectangle] do + gated(sup, fn -> read_icon(sup, name, shape) end) + end + + defp read_icon(sup, name, shape) do cache = AdaptorsSupervisor.cache_name(sup) source = AdaptorsSupervisor.source(sup) strategy = AdaptorsSupervisor.strategy(sup) - with {:ok, meta} <- icon_meta(sup, name), + with {:ok, meta} <- read_icon_meta(sup, name), {:ok, ext} <- ext_for_shape(meta, shape), {:ok, expected_sha} <- sha256_for_shape(meta, shape) do if IconCache.cached?(source, name, shape, ext, expected_sha) do @@ -137,7 +155,9 @@ defmodule Lightning.Adaptors.Store do `dependencies` and `peer_dependencies` columns. """ @spec packages(sup()) :: {:ok, [package_meta()]} | {:error, term()} - def packages(sup) do + def packages(sup), do: gated(sup, fn -> read_packages(sup) end) + + defp read_packages(sup) do cache = AdaptorsSupervisor.cache_name(sup) source = AdaptorsSupervisor.source(sup) @@ -164,7 +184,9 @@ defmodule Lightning.Adaptors.Store do can never drift apart. """ @spec catalogue(sup()) :: {:ok, catalogue()} | {:error, term()} - def catalogue(sup) do + def catalogue(sup), do: gated(sup, fn -> read_catalogue(sup) end) + + defp read_catalogue(sup) do cache = AdaptorsSupervisor.cache_name(sup) source = AdaptorsSupervisor.source(sup) @@ -188,7 +210,9 @@ defmodule Lightning.Adaptors.Store do """ @spec icon_meta(sup(), String.t()) :: {:ok, icon_meta()} | {:error, :not_found} - def icon_meta(sup, name) do + def icon_meta(sup, name), do: gated(sup, fn -> read_icon_meta(sup, name) end) + + defp read_icon_meta(sup, name) do cache = AdaptorsSupervisor.cache_name(sup) source = AdaptorsSupervisor.source(sup) @@ -206,6 +230,67 @@ defmodule Lightning.Adaptors.Store do |> unwrap() end + @doc """ + Waits until the catalogue has loaded at least once, triggering the first + load if needed. + + Returns `:ok`, `{:error, :timeout}` if the load did not finish within + `Lightning.Adaptors.Config.first_load_timeout/0`, `{:error, :unavailable}` + if no Scheduler is reachable, or `{:error, :not_ready}` if the load ran + and could neither write a row nor report a complete cycle. + """ + @spec ensure_loaded(sup()) :: + :ok | {:error, :timeout | :unavailable | :not_ready} + def ensure_loaded(sup) do + if loaded?(sup), do: :ok, else: first_load(sup) + end + + # Only an empty answer pays for the `loaded?/1` query: a read that found + # data cannot be waiting on the first load, whatever the catalogue's + # state. + defp gated(sup, read) do + result = read.() + + if empty?(result) and not loaded?(sup) do + with :ok <- first_load(sup), do: read.() + else + result + end + end + + defp empty?({:error, :not_found}), do: true + defp empty?({:ok, []}), do: true + defp empty?({:ok, {_stamp, []}}), do: true + defp empty?(_result), do: false + + # Rows answer for a catalogue a previous boot or a seed filled; the + # Scheduler answers for the one case that leaves no row to find, a cycle + # that completed against a source listing nothing. + defp loaded?(sup) do + Catalogue.max_checked_at(AdaptorsSupervisor.source(sup)) != nil or + Scheduler.completed?(AdaptorsSupervisor.global_scheduler_name(sup)) + end + + # A failed cycle can land on rows a seed already wrote, so the outcome + # is re-read rather than inferred from the refresh result. + defp first_load(sup) do + case await_refresh(sup) do + {:error, :timeout} -> {:error, :timeout} + {:error, :unavailable} -> {:error, :unavailable} + _other -> if loaded?(sup), do: :ok, else: {:error, :not_ready} + end + end + + defp await_refresh(sup) do + Scheduler.await_refresh( + AdaptorsSupervisor.global_scheduler_name(sup), + Config.first_load_timeout() + ) + catch + :exit, {:timeout, _} -> {:error, :timeout} + :exit, _reason -> {:error, :unavailable} + end + @doc """ Overwrites the cached package list, icon metadata and catalogue from the database. An empty catalogue is left uncached, as `catalogue/1` does. diff --git a/lib/lightning/credentials.ex b/lib/lightning/credentials.ex index c4d6dfa7f47..3ae96cb2d0e 100644 --- a/lib/lightning/credentials.ex +++ b/lib/lightning/credentials.ex @@ -587,12 +587,10 @@ defmodule Lightning.Credentials do """ @spec get_schema(String.t()) :: Credentials.Schema.t() def get_schema(schema_name) do - resolved = Lightning.Adaptors.resolve_name(schema_name) - - case Lightning.Adaptors.schema(resolved) do - {:ok, schema_body} -> - Credentials.Schema.new(schema_body, resolved) - + with {:ok, resolved} <- Lightning.Adaptors.resolve_name(schema_name), + {:ok, schema_body} <- Lightning.Adaptors.schema(resolved) do + Credentials.Schema.new(schema_body, resolved) + else {:error, reason} -> raise "Error reading credential schema. Got: #{inspect(reason)}" end @@ -616,16 +614,24 @@ defmodule Lightning.Credentials do |> Repo.all() |> Enum.reduce(0, fn short, count -> case Lightning.Adaptors.resolve_name(sup, short) do - ^short -> + {:ok, ^short} -> count - full -> + {:ok, full} -> {n, _} = Repo.update_all(from(c in Credential, where: c.schema == ^short), set: [schema: full] ) count + n + + {:error, reason} -> + Logger.warning( + "Could not resolve credential schema #{inspect(short)}: " <> + "#{inspect(reason)}" + ) + + count end end) diff --git a/lib/lightning/credentials/credential.ex b/lib/lightning/credentials/credential.ex index bb95bef8524..dd34abddad6 100644 --- a/lib/lightning/credentials/credential.ex +++ b/lib/lightning/credentials/credential.ex @@ -109,10 +109,20 @@ defmodule Lightning.Credentials.Credential do end end + # Expanding a legacy short name needs a loaded catalogue. When it cannot + # answer the name is stored as typed rather than failing the save: the + # short form is a supported legacy shape that `get_schema/1` resolves on + # read and `Credentials.reconcile_legacy_schema_names/1` rewrites later. defp resolve_schema_name(changeset) do update_change(changeset, :schema, fn - schema when is_binary(schema) -> Lightning.Adaptors.resolve_name(schema) - schema -> schema + schema when is_binary(schema) -> + case Lightning.Adaptors.resolve_name(schema) do + {:ok, resolved} -> resolved + {:error, _catalogue_unavailable} -> schema + end + + schema -> + schema end) end end diff --git a/lib/lightning/metadata_service.ex b/lib/lightning/metadata_service.ex index 242c7a0b36b..54695835672 100644 --- a/lib/lightning/metadata_service.ex +++ b/lib/lightning/metadata_service.ex @@ -124,6 +124,9 @@ defmodule Lightning.MetadataService do defp get_adaptor_path(adaptor) do case AdaptorService.install(@adaptor_service, adaptor) do + {:error, {:catalogue_unavailable, _reason}} -> + {:error, Error.new("adaptor_catalogue_unavailable")} + {:error, _} -> {:error, Error.new("no_matching_adaptor")} diff --git a/lib/lightning/setup.ex b/lib/lightning/setup.ex index 33ccd2dc69d..0db7626a9d6 100644 --- a/lib/lightning/setup.ex +++ b/lib/lightning/setup.ex @@ -44,6 +44,10 @@ defmodule Lightning.Setup do Ecto.Migrator.with_repo(Lightning.Repo, fn _repo -> {:ok, _pid} = Lightning.Adaptors.Supervisor.ensure_started() + # Load the catalogue up front rather than let the first adaptor lookup + # block inside `fun`'s transaction for the length of a source fetch. + # A failure here surfaces at that lookup instead. + _ = Lightning.Adaptors.ensure_loaded() fun.() end) end diff --git a/lib/lightning/workflows/job.ex b/lib/lightning/workflows/job.ex index dfdce002e44..2fcc57f30a4 100644 --- a/lib/lightning/workflows/job.ex +++ b/lib/lightning/workflows/job.ex @@ -160,18 +160,15 @@ defmodule Lightning.Workflows.Job do # yet, so that case gets its own, retry-able message. defp validate_known_adaptor(changeset) do validate_change(changeset, :adaptor, fn :adaptor, adaptor -> - with {name, _version} when is_binary(name) <- Adaptors.parse_spec(adaptor), + with {:ok, {name, _version}} <- Adaptors.parse_spec(adaptor), {:ok, _package} <- Adaptors.fetch_adaptor(name) do [] else - {:error, :not_found} -> + {:error, reason} when reason in [:not_found, :invalid_format] -> [adaptor: "is not a recognised adaptor"] - {:error, _} -> + {:error, _reason} -> [adaptor: "adaptor catalogue is not ready yet, try again shortly"] - - _ -> - [adaptor: "is not a recognised adaptor"] end end) end diff --git a/lib/lightning_web/controllers/adaptor_icon_controller.ex b/lib/lightning_web/controllers/adaptor_icon_controller.ex index 10e525e2390..76bb8535e2a 100644 --- a/lib/lightning_web/controllers/adaptor_icon_controller.ex +++ b/lib/lightning_web/controllers/adaptor_icon_controller.ex @@ -83,7 +83,9 @@ defmodule LightningWeb.AdaptorIconController do shape = String.to_existing_atom(shape) case Adaptors.icon_meta(name) do - {:error, :not_found} -> + # A catalogue that will not load has no icon to serve either, so its + # errors are 404s here rather than a 500. + {:error, _reason} -> send_resp(conn, 404, "") {:ok, meta} -> diff --git a/test/lightning/adaptor_service_test.exs b/test/lightning/adaptor_service_test.exs index 41c51d35ae3..6e58064a6fe 100644 --- a/test/lightning/adaptor_service_test.exs +++ b/test/lightning/adaptor_service_test.exs @@ -5,20 +5,20 @@ defmodule Lightning.AdaptorServiceTest do load, and a name the loaded catalogue lacks refuses the install. """ - # set_mox_global: the load runs in a Task owned by the production - # Scheduler. + # set_mox_global: the load runs in a Task owned by the Scheduler. use Lightning.DataCase, async: false import Mox + import Lightning.AdaptorTestHelpers, only: [isolated_adaptors: 1] alias Lightning.Adaptors.Catalogue alias Lightning.AdaptorService setup :set_mox_global setup :verify_on_exit! + setup :isolated_adaptors setup do - Lightning.AdaptorTestHelpers.clear_global_adaptors_cache() stub(Lightning.AdaptorService.RepoMock, :list_local, fn _path -> [] end) {:ok, agent} = @@ -50,6 +50,19 @@ defmodule Lightning.AdaptorServiceTest do assert_received :listed end + test "a catalogue that cannot answer is not a refusal", %{agent: agent} do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:error, :econnrefused} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + assert {:error, {:catalogue_unavailable, :not_ready}} = + AdaptorService.install(agent, "@openfn/language-http") + end + test "populated catalogue without this package: refuses", %{agent: agent} do {:ok, _} = Catalogue.upsert_adaptor(%{ diff --git a/test/lightning/adaptors/readiness_test.exs b/test/lightning/adaptors/readiness_test.exs index c4d653fab5c..17597994ec8 100644 --- a/test/lightning/adaptors/readiness_test.exs +++ b/test/lightning/adaptors/readiness_test.exs @@ -118,8 +118,6 @@ defmodule Lightning.Adaptors.ReadinessTest do assert {:error, :not_found} = Adaptors.fetch_adaptor(sup, "@openfn/never-existed") - - assert Adaptors.get_adaptor(sup, "@openfn/never-existed") == nil end test "answers for the given supervisor's source, not the default one" do @@ -202,12 +200,12 @@ defmodule Lightning.Adaptors.ReadinessTest do Adaptors.fetch_adaptor(sup, "@openfn/never-existed") end - test "returns {:error, :not_ready} when the load leaves the catalogue empty", + test "returns {:error, :not_found} when the load lists nothing at all", %{sup: sup} do expect_one_load([]) start_scheduler(sup) - assert {:error, :not_ready} = + assert {:error, :not_found} = Adaptors.fetch_adaptor(sup, "@openfn/language-http") end @@ -277,11 +275,15 @@ defmodule Lightning.Adaptors.ReadinessTest do test "maps a failed wait to its error atom", %{sup: sup} do assert {:error, :unavailable} = Adaptors.ensure_loaded(sup) + end + test "a completed load against a source with no adaptors is :ok", %{ + sup: sup + } do expect_one_load([]) start_scheduler(sup) - assert {:error, :not_ready} = Adaptors.ensure_loaded(sup) + assert :ok = Adaptors.ensure_loaded(sup) end end end diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index e9ee54adadc..af0708b2b3a 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -6,6 +6,7 @@ defmodule Lightning.Adaptors.StoreTest do import Mox alias Lightning.Adaptors.Catalogue + alias Lightning.Adaptors.Scheduler alias Lightning.Adaptors.Store alias Lightning.Adaptors.Supervisor, as: AdaptorsSupervisor alias LightningWeb.AdaptorIconURL @@ -31,6 +32,121 @@ defmodule Lightning.Adaptors.StoreTest do {:ok, sup: sup, cache: cache} end + # Replaces the supervisor's own Highlander-wrapped Scheduler with one + # this test owns, so a gated read's await_refresh lands on a process + # whose sandbox connection and Mox stubs are ours. + defp start_scheduler(sup) do + :ok = + Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) + + pid = + start_supervised!({ + Scheduler, + name: AdaptorsSupervisor.global_scheduler_name(sup), + sup: sup, + lock_key: AdaptorsSupervisor.lock_key(sup), + cache: AdaptorsSupervisor.cache_name(sup), + tasks: AdaptorsSupervisor.tasks_name(sup), + source_topic: AdaptorsSupervisor.source_topic(sup), + refresh_interval: 0, + warn_when_empty: false, + checked_at: fn _source -> nil end + }) + + Ecto.Adapters.SQL.Sandbox.allow(Lightning.Repo, self(), pid) + Mox.allow(Lightning.Adaptors.StrategyMock, self(), pid) + pid + end + + defp expect_one_load(records) do + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> + {:ok, Enum.map(records, &Map.take(&1, [:name, :latest_version]))} + end) + + expect( + Lightning.Adaptors.StrategyMock, + :fetch_adaptor, + length(records), + fn name -> {:ok, Enum.find(records, &(&1.name == name))} end + ) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + end + + describe "the first-load gate" do + test "schema/2 on a never-loaded catalogue waits, then reads again", %{ + sup: sup + } do + expect_one_load([adaptor_record(schema_data: ~s({"type":"object"}))]) + start_scheduler(sup) + + assert {:ok, ~s({"type":"object"})} = + Store.schema(sup, "@openfn/language-http") + end + + test "packages/1 on a never-loaded catalogue waits, then reads again", %{ + sup: sup + } do + expect_one_load([adaptor_record()]) + start_scheduler(sup) + + assert {:ok, [%{name: "@openfn/language-http"}]} = Store.packages(sup) + end + + test "a loaded catalogue answers empty without a second load", %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + flunk("a loaded catalogue must not be reloaded") + end) + + start_scheduler(sup) + + assert {:error, :not_found} = Store.schema(sup, "@openfn/never-existed") + end + + test "a source with no adaptors settles on an empty catalogue", %{sup: sup} do + # expect/4 with a count of 1 fails the test on a second listing, which + # is the point: a completed cycle that wrote nothing must not send + # every later read back for another one. + expect_one_load([]) + start_scheduler(sup) + + assert {:ok, []} = Store.packages(sup) + assert {:ok, []} = Store.packages(sup) + assert {:error, :not_found} = Store.schema(sup, "@openfn/language-http") + end + + test "a load that fails to write anything it listed is :not_ready", %{ + sup: sup + } do + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} + end) + + expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _name -> + {:error, :boom} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + start_scheduler(sup) + + assert {:error, :not_ready} = Store.packages(sup) + end + + test "no reachable Scheduler is :unavailable", %{sup: sup} do + :ok = + Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) + + assert {:error, :unavailable} = Store.packages(sup) + end + end + describe "schema/2" do test "cache hit returns cached value without touching Strategy or DB", %{ sup: sup, @@ -93,6 +209,7 @@ defmodule Lightning.Adaptors.StoreTest do end) source = AdaptorsSupervisor.source(sup) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) assert {:error, :not_found} = Store.schema(sup, "@openfn/never-existed") assert Catalogue.get_adaptor("@openfn/never-existed", source) == nil @@ -117,14 +234,17 @@ defmodule Lightning.Adaptors.StoreTest do end describe "packages/1" do - test "empty DB returns {:ok, []} but does NOT cache the empty result", %{ - sup: sup, - cache: cache - } do + test "a loaded catalogue with nothing listable returns {:ok, []} but does NOT cache it", + %{sup: sup, cache: cache} do expect(Lightning.Adaptors.StrategyMock, :fetch_adaptor, 0, fn _ -> :unreachable end) + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record(name: "@openfn/language-collections") + ) + assert {:ok, []} = Store.packages(sup) source = AdaptorsSupervisor.source(sup) @@ -165,11 +285,14 @@ defmodule Lightning.Adaptors.StoreTest do end describe "catalogue/1" do - test "empty DB returns an empty payload but does NOT cache it", %{ - sup: sup, - cache: cache - } do - assert {:ok, {{nil, 0}, []}} = Store.catalogue(sup) + test "a loaded catalogue with nothing listable returns an empty payload but does NOT cache it", + %{sup: sup, cache: cache} do + {:ok, _} = + Catalogue.upsert_adaptor( + adaptor_record(name: "@openfn/language-collections") + ) + + assert {:ok, {_stamp, []}} = Store.catalogue(sup) source = AdaptorsSupervisor.source(sup) assert {:ok, nil} = Cachex.get(cache, {:catalogue, source}) @@ -562,6 +685,8 @@ defmodule Lightning.Adaptors.StoreTest do sup: sup, cache: cache } do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + assert {:error, :not_found} = Store.icon_meta(sup, "@openfn/never-existed") source = AdaptorsSupervisor.source(sup) diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index 13729831ad3..c159b70da1f 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -14,6 +14,22 @@ defmodule Lightning.AdaptorsTest do setup :verify_on_exit! setup :isolated_adaptors + # A refresh cycle lists the source, fetches each listed package, then + # fetches icons; stubbing fewer than all three crashes the cycle. + defp stub_refresh_cycle(record) do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + {:ok, [record]} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _name -> + {:ok, record} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + end + defp start_scheduler(sup) do # Stop the supervisor's auto-started HighlanderPG (and its wrapped # Scheduler) so we can start a replacement under the controlled @@ -50,7 +66,14 @@ defmodule Lightning.AdaptorsTest do assert pkg.source == :npm end - test "returns {:ok, []} when DB is empty", %{sup: sup} do + test "an empty catalogue that has never loaded waits, then reports it", + %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> {:ok, []} end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + assert {:ok, []} = Adaptors.packages(sup) end @@ -96,7 +119,7 @@ defmodule Lightning.AdaptorsTest do end describe "schema/2" do - test "delegates to Store.schema/2 and returns schema", %{sup: sup} do + test "returns the schema body", %{sup: sup} do stub(Lightning.Adaptors.StrategyMock, :fetch_adaptor, fn _ -> {:error, :unreachable} end) @@ -123,32 +146,45 @@ defmodule Lightning.AdaptorsTest do assert {:ok, ^ordered_body} = Adaptors.schema(sup, "@openfn/language-http") end + + test "waits for the first load and returns the schema", %{sup: sup} do + record = adaptor_record(schema_data: ~s({"type":"object"})) + stub_refresh_cycle(record) + + assert {:ok, ~s({"type":"object"})} = + Adaptors.schema(sup, "@openfn/language-http") + end + + test "is :not_found once the catalogue has loaded", %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + flunk("a loaded catalogue must not be reloaded") + end) + + assert {:error, :not_found} = + Adaptors.schema(sup, "@openfn/never-existed") + end end - describe "get_adaptor/1" do + describe "fetch_adaptor/2" do test "returns a Package for an adaptor in the active source" do {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(latest_version: "4.1.0")) - assert %Adaptors.Package{ - name: "@openfn/language-http", - source: :npm, - latest_version: "4.1.0" - } = Adaptors.get_adaptor("@openfn/language-http") + assert {:ok, + %Adaptors.Package{ + name: "@openfn/language-http", + source: :npm, + latest_version: "4.1.0" + }} = Adaptors.fetch_adaptor("@openfn/language-http") end - test "returns nil for an adaptor absent from the catalogue" do + test "is :not_found for an adaptor absent from the catalogue" do {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) - assert Adaptors.get_adaptor("@openfn/never-existed") == nil - end - - test "returns nil when the catalogue is empty, without triggering a refresh" do - stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> - flunk("get_adaptor/1 must not trigger a load") - end) - - assert Adaptors.get_adaptor("@openfn/language-http") == nil + assert {:error, :not_found} = + Adaptors.fetch_adaptor("@openfn/never-existed") end test "still resolves a name the catalogue listing excludes" do @@ -157,17 +193,16 @@ defmodule Lightning.AdaptorsTest do adaptor_record(name: "@openfn/language-collections") ) - assert %Adaptors.Package{name: "@openfn/language-collections"} = - Adaptors.get_adaptor("@openfn/language-collections") - assert {:ok, %Adaptors.Package{name: "@openfn/language-collections"}} = Adaptors.fetch_adaptor("@openfn/language-collections") end - test "returns nil for a row under a different source than the active one" do + test "is :not_found for a row under a different source than the active one" do {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(source: :local)) + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/other")) - assert Adaptors.get_adaptor("@openfn/language-http") == nil + assert {:error, :not_found} = + Adaptors.fetch_adaptor("@openfn/language-http") end test "computes has_schema on the DB-fallback path (excluded from the lean listing)" do @@ -179,8 +214,8 @@ defmodule Lightning.AdaptorsTest do ) ) - assert %Adaptors.Package{has_schema: true} = - Adaptors.get_adaptor("@openfn/language-collections") + assert {:ok, %Adaptors.Package{has_schema: true}} = + Adaptors.fetch_adaptor("@openfn/language-collections") end test "has_schema is false on the DB-fallback path when schema_data is nil" do @@ -192,8 +227,8 @@ defmodule Lightning.AdaptorsTest do ) ) - assert %Adaptors.Package{has_schema: false} = - Adaptors.get_adaptor("@openfn/language-collections") + assert {:ok, %Adaptors.Package{has_schema: false}} = + Adaptors.fetch_adaptor("@openfn/language-collections") end end @@ -202,7 +237,7 @@ defmodule Lightning.AdaptorsTest do %{sup: sup} do {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) - assert Adaptors.resolve_name(sup, "http") == "@openfn/language-http" + assert Adaptors.resolve_name(sup, "http") == {:ok, "@openfn/language-http"} end test "leaves a full name that is already in the catalogue unchanged", %{ @@ -211,11 +246,21 @@ defmodule Lightning.AdaptorsTest do {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) assert Adaptors.resolve_name(sup, "@openfn/language-http") == - "@openfn/language-http" + {:ok, "@openfn/language-http"} end test "leaves an unknown short name unchanged", %{sup: sup} do - assert Adaptors.resolve_name(sup, "unknownish") == "unknownish" + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + assert Adaptors.resolve_name(sup, "unknownish") == {:ok, "unknownish"} + end + + test "waits for the first load when the catalogue has never loaded", %{ + sup: sup + } do + stub_refresh_cycle(adaptor_record()) + + assert Adaptors.resolve_name(sup, "http") == {:ok, "@openfn/language-http"} end test "never resolves the raw and oauth sentinels, even if shadowed in the catalogue", @@ -226,8 +271,8 @@ defmodule Lightning.AdaptorsTest do {:ok, _} = Catalogue.upsert_adaptor(adaptor_record(name: "@openfn/language-oauth")) - assert Adaptors.resolve_name(sup, "raw") == "raw" - assert Adaptors.resolve_name(sup, "oauth") == "oauth" + assert Adaptors.resolve_name(sup, "raw") == {:ok, "raw"} + assert Adaptors.resolve_name(sup, "oauth") == {:ok, "oauth"} end end @@ -261,25 +306,27 @@ defmodule Lightning.AdaptorsTest do describe "parse_spec/1" do test "splits a spec carrying a version" do assert Adaptors.parse_spec("@openfn/language-http@1.2.3") == - {"@openfn/language-http", "1.2.3"} + {:ok, {"@openfn/language-http", "1.2.3"}} assert Adaptors.parse_spec("@openfn/language-http@latest") == - {"@openfn/language-http", "latest"} + {:ok, {"@openfn/language-http", "latest"}} - assert Adaptors.parse_spec("common@1.0.0") == {"common", "1.0.0"} + assert Adaptors.parse_spec("common@1.0.0") == {:ok, {"common", "1.0.0"}} end test "returns a nil version for a spec without one" do assert Adaptors.parse_spec("@openfn/language-http") == - {"@openfn/language-http", nil} + {:ok, {"@openfn/language-http", nil}} end - test "returns {nil, nil} for a string that isn't a well-formed spec" do + test "errors on a string that isn't a well-formed spec" do assert Adaptors.parse_spec("@openfn/language-http; rm -rf /") == - {nil, nil} + {:error, :invalid_format} + + assert Adaptors.parse_spec("@openfn/x\npwd\nb@1.0.0") == + {:error, :invalid_format} - assert Adaptors.parse_spec("@openfn/x\npwd\nb@1.0.0") == {nil, nil} - assert Adaptors.parse_spec("") == {nil, nil} + assert Adaptors.parse_spec("") == {:error, :invalid_format} end end @@ -461,6 +508,8 @@ defmodule Lightning.AdaptorsTest do test "icon_meta/2 returns {:error, :not_found} for unknown adaptor", %{ sup: sup } do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + assert {:error, :not_found} = Adaptors.icon_meta(sup, "@openfn/never-existed") end diff --git a/test/support/adaptor_test_helpers.ex b/test/support/adaptor_test_helpers.ex index 8d3f31ea69e..2487fcb7817 100644 --- a/test/support/adaptor_test_helpers.ex +++ b/test/support/adaptor_test_helpers.ex @@ -171,7 +171,7 @@ defmodule Lightning.AdaptorTestHelpers do ensure_isolated!() case Lightning.Adaptors.parse_spec(spec) do - {name, _version} when is_binary(name) -> + {:ok, {name, _version}} -> source = AdaptorsSupervisor.source(Config.default_instance()) if is_nil(Lightning.Adaptors.Catalogue.get_adaptor(name, source)), From 321b3fff7276332d93e454d045eccf2c54a83405 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Mon, 14 Sep 2026 10:42:04 +0200 Subject: [PATCH 31/37] Document the upgrade and drop the unused adaptors channel handler - Spell out what an upgrade has to act on for the adaptor catalogue - Delete the unused request_adaptors channel handler - Describe install/2's three outcomes in the AdaptorService test moduledoc --- CHANGELOG.md | 51 +++++++++- .../__helpers__/README.md | 1 - .../channels/workflow_channel.ex | 30 ------ test/lightning/adaptor_service_test.exs | 8 +- .../channels/workflow_channel_test.exs | 98 +------------------ 5 files changed, 59 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a8ea7a438c..6baa7d973b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,56 @@ and this project adheres to - Lightning now keeps its own adaptor registry instead of fetching the list from npm at startup, so new adaptors and versions show up without a rebuild or - redeploy. See [ADAPTORS.md](ADAPTORS.md). + redeploy. The entries below cover what an upgrade has to act on. See + [ADAPTORS.md](ADAPTORS.md). + [#4801](https://github.com/OpenFn/lightning/pull/4801) + +- `SCHEMAS_PATH` and `ADAPTORS_REGISTRY_JSON_PATH` are gone. Nothing reads + either of them any more, so delete them from your deployment config. + [#4801](https://github.com/OpenFn/lightning/pull/4801) + +- `LOCAL_ADAPTORS` and `OPENFN_ADAPTORS_REPO` are deprecated in favour of + `ADAPTORS_STRATEGY=local` and `ADAPTORS_LOCAL_REPO`. The old names still work + and log a warning on boot. + [#4801](https://github.com/OpenFn/lightning/pull/4801) + +- Nine new variables configure the registry, all optional and all with a working + default: `ADAPTORS_STRATEGY`, `ADAPTORS_LOCAL_REPO`, `ADAPTORS_ICONS_PATH`, + `ADAPTORS_REFRESH_INTERVAL_SECONDS`, `ADAPTORS_NPM_REGISTRY_URL`, + `ADAPTORS_NPM_JSDELIVR_URL`, `ADAPTORS_NPM_GITHUB_URL`, + `ADAPTORS_NPM_GITHUB_REF` and `ADAPTORS_NPM_HTTP_TIMEOUT`. See + [ADAPTORS.md](ADAPTORS.md) for what each one does. + [#4801](https://github.com/OpenFn/lightning/pull/4801) + +- `mix lightning.install_schemas`, `mix lightning.install_adaptor_icons` and + `mix lightning.download_adaptor_registry_cache` are deleted. Lightning fetches + schemas, icons and the package list itself while it runs. A custom build + script that calls any of them will fail. + [#4801](https://github.com/OpenFn/lightning/pull/4801) + +- Three migrations come with this release. All are forward-only and none + backfills anything. Two create and index the adaptor catalogue tables; the + third widens `credentials.schema` from 40 to 100 characters so it can hold + full package names such as `@openfn/language-http`. + [#4801](https://github.com/OpenFn/lightning/pull/4801) + +- The adaptor icon cache needs storage that survives a restart, at + `ADAPTORS_ICONS_PATH`. The official image and `docker-compose.yml` mount a + volume for it. Any other deployment has to provide one, or every restart + downloads the icons again. + [#4801](https://github.com/OpenFn/lightning/pull/4801) + +- A running instance now needs outbound access to npm, jsDelivr and + raw.githubusercontent.com, for the package list, credential schemas and icons + respectively. All three are public, unauthenticated requests, and an internal + mirror can be used by setting the `ADAPTORS_NPM_*` URL variables. An instance + with no internet access at all can import a snapshot instead; see "Running + without internet access" in [ADAPTORS.md](ADAPTORS.md). + [#4801](https://github.com/OpenFn/lightning/pull/4801) + +- Adaptors that npm marks as deprecated no longer appear in the adaptor picker + or the credential type list. Jobs and credentials already using one still + resolve, validate and run. [#4801](https://github.com/OpenFn/lightning/pull/4801) ### Removed diff --git a/assets/test/collaborative-editor/__helpers__/README.md b/assets/test/collaborative-editor/__helpers__/README.md index 71dd346b63f..6bafaad964f 100644 --- a/assets/test/collaborative-editor/__helpers__/README.md +++ b/assets/test/collaborative-editor/__helpers__/README.md @@ -26,7 +26,6 @@ patterns. import { createMockChannelWithResponses } from './__helpers__'; const channel = createMockChannelWithResponses({ - request_adaptors: { adaptors: mockAdaptorsList }, get_context: { user: mockUser, project: mockProject }, }); ``` diff --git a/lib/lightning_web/channels/workflow_channel.ex b/lib/lightning_web/channels/workflow_channel.ex index 3fb230867df..355fe03bb46 100644 --- a/lib/lightning_web/channels/workflow_channel.ex +++ b/lib/lightning_web/channels/workflow_channel.ex @@ -114,17 +114,6 @@ defmodule LightningWeb.WorkflowChannel do {:error, %{reason: "invalid parameters. project_id and action are required"}} end - @impl true - def handle_in("request_adaptors", _payload, socket) do - async_task(socket, "request_adaptors", fn -> - adaptors = - list_all_packages() - |> Enum.map(&with_icon_urls/1) - - %{adaptors: adaptors} - end) - end - @impl true def handle_in("request_credentials", _payload, socket) do project = socket.assigns.project @@ -1344,24 +1333,6 @@ defmodule LightningWeb.WorkflowChannel do defp unhandled_message_type(%struct{}), do: inspect(struct) defp unhandled_message_type(_msg), do: "unrecognised" - defp list_all_packages do - case Lightning.Adaptors.packages() do - {:ok, pkgs} -> pkgs - {:error, _} -> [] - end - end - - defp with_icon_urls(%Lightning.Adaptors.Package{name: name} = pkg) do - %{ - name: name, - latest_version: pkg.latest_version, - icon_urls: %{ - square: LightningWeb.AdaptorIconURL.build(name, pkg, :square), - rectangle: LightningWeb.AdaptorIconURL.build(name, pkg, :rectangle) - } - } - end - defp handle_async_event("request_run_steps", socket_ref, reply) do unwrapped_reply = unwrap_run_steps_reply(reply) reply(socket_ref, unwrapped_reply) @@ -1369,7 +1340,6 @@ defmodule LightningWeb.WorkflowChannel do defp handle_async_event(event, socket_ref, reply) when event in [ - "request_adaptors", "request_credentials", "request_metadata", "request_current_user", diff --git a/test/lightning/adaptor_service_test.exs b/test/lightning/adaptor_service_test.exs index 6e58064a6fe..83f4e83bdd0 100644 --- a/test/lightning/adaptor_service_test.exs +++ b/test/lightning/adaptor_service_test.exs @@ -1,8 +1,10 @@ defmodule Lightning.AdaptorServiceTest do @moduledoc """ - Covers `AdaptorService.known?/1` gating `install/2` on - `Lightning.Adaptors.fetch_adaptor/1`: an empty catalogue waits for one - load, and a name the loaded catalogue lacks refuses the install. + Covers `AdaptorService.install/2` checking the package against + `Lightning.Adaptors.fetch_adaptor/1` first: an empty catalogue waits for + one load, a name the loaded catalogue lacks is refused as + `:adaptor_not_permitted`, and a catalogue that cannot answer at all comes + back as `{:catalogue_unavailable, reason}` rather than a refusal. """ # set_mox_global: the load runs in a Task owned by the Scheduler. diff --git a/test/lightning_web/channels/workflow_channel_test.exs b/test/lightning_web/channels/workflow_channel_test.exs index df0d2dafb5b..238c9f9856c 100644 --- a/test/lightning_web/channels/workflow_channel_test.exs +++ b/test/lightning_web/channels/workflow_channel_test.exs @@ -2703,29 +2703,13 @@ defmodule LightningWeb.WorkflowChannelTest do end end - describe "request_adaptors and request_credentials" do - setup do - insert(:adaptor, name: "@openfn/language-salesforce", source: :npm) - insert(:adaptor, name: "@openfn/language-http", source: :npm) - :ok - end - - test "handles multiple concurrent requests independently", %{ + describe "request_credentials" do + test "replies with project and keychain credential lists", %{ socket: socket } do - ref_adaptors = push(socket, "request_adaptors", %{}) - ref_credentials = push(socket, "request_credentials", %{}) - - assert_reply ref_adaptors, :ok, %{adaptors: adaptors} - assert_reply ref_credentials, :ok, %{credentials: credentials} - - assert is_list(adaptors) - assert adaptors != [] - assert Enum.all?(adaptors, &Map.has_key?(&1, :icon_urls)) + ref = push(socket, "request_credentials", %{}) - assert Enum.all?(adaptors, fn a -> - Enum.sort(Map.keys(a.icon_urls)) == [:rectangle, :square] - end) + assert_reply ref, :ok, %{credentials: credentials} assert Map.has_key?(credentials, :project_credentials) assert Map.has_key?(credentials, :keychain_credentials) @@ -2733,80 +2717,6 @@ defmodule LightningWeb.WorkflowChannelTest do assert is_list(credentials.keychain_credentials) end - test "request_adaptors enriches records with icon_urls when meta present", - %{socket: socket} do - name = "@openfn/language-common" - square_sha = :crypto.strong_rand_bytes(32) - rectangle_sha = :crypto.strong_rand_bytes(32) - - insert(:adaptor, - name: name, - icon_square_ext: "png", - icon_square_sha256: square_sha, - icon_rectangle_ext: "svg", - icon_rectangle_sha256: rectangle_sha - ) - - ref = push(socket, "request_adaptors", %{}) - assert_reply ref, :ok, %{adaptors: adaptors} - - record = Enum.find(adaptors, &(&1.name == name)) - assert record, "expected legacy registry to include #{name}" - - {:ok, meta} = Lightning.Adaptors.icon_meta(name) - - assert record.icon_urls.square == - LightningWeb.AdaptorIconURL.build(name, meta, :square) - - assert record.icon_urls.rectangle == - LightningWeb.AdaptorIconURL.build(name, meta, :rectangle) - - assert is_binary(record.icon_urls.square) - assert is_binary(record.icon_urls.rectangle) - end - - test "request_adaptors emits nil icon_urls when row has no icon meta", - %{socket: socket} do - name = "@openfn/language-dhis2" - - insert(:adaptor, - name: name, - source: :npm, - icon_square_ext: nil, - icon_square_sha256: nil, - icon_rectangle_ext: nil, - icon_rectangle_sha256: nil - ) - - ref = push(socket, "request_adaptors", %{}) - assert_reply ref, :ok, %{adaptors: adaptors} - - record = Enum.find(adaptors, &(&1.name == name)) - assert record, "expected packages/0 to include #{name}" - assert record.icon_urls == %{square: nil, rectangle: nil} - end - - test "request_adaptors handles half-populated icon meta", %{socket: socket} do - name = "@openfn/language-commcare" - square_sha = :crypto.strong_rand_bytes(32) - - insert(:adaptor, - name: name, - icon_square_ext: "png", - icon_square_sha256: square_sha, - icon_rectangle_ext: nil, - icon_rectangle_sha256: nil - ) - - ref = push(socket, "request_adaptors", %{}) - assert_reply ref, :ok, %{adaptors: adaptors} - - record = Enum.find(adaptors, &(&1.name == name)) - assert record, "expected legacy registry to include #{name}" - assert is_binary(record.icon_urls.square) - assert record.icon_urls.rectangle == nil - end - test "returns correctly structured project credentials", %{ socket: socket, project: project From 71d56c2e0516017bf90399d3c9c65b14f41b4373 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Mon, 14 Sep 2026 10:48:58 +0200 Subject: [PATCH 32/37] Answer catalogue reads immediately, make waiting for the first load opt-in - Treat an empty npm org listing as an error, not an empty catalogue - Answer catalogue reads immediately and make waiting for the first load opt-in - Drop refresh waiters whose call has already timed out --- lib/lightning/adaptors.ex | 53 ++++++++-------- lib/lightning/adaptors/local.ex | 4 ++ lib/lightning/adaptors/npm.ex | 3 +- lib/lightning/adaptors/npm/registry.ex | 21 +++++-- lib/lightning/adaptors/scheduler.ex | 32 +++++++--- lib/lightning/adaptors/store.ex | 63 ++++++++++--------- lib/lightning/adaptors/strategy.ex | 7 +++ .../controllers/adaptor_controller.ex | 4 ++ .../controllers/adaptor_icon_controller.ex | 9 ++- lib/mix/tasks/lightning.adaptors.refresh.ex | 6 +- test/lightning/adaptors/npm/registry_test.exs | 18 +++--- test/lightning/adaptors/readiness_test.exs | 20 +++++- test/lightning/adaptors/scheduler_test.exs | 22 +++++++ test/lightning/adaptors/store_test.exs | 51 +++++++++++---- .../controllers/adaptor_controller_test.exs | 15 +++++ .../adaptor_icon_controller_test.exs | 13 ++++ .../tasks/lightning.adaptors.refresh_test.exs | 14 ++++- .../lightning.adaptors.snapshot_test.exs | 2 +- 18 files changed, 256 insertions(+), 101 deletions(-) diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index 2b47ad86702..d487dda3e9e 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -5,9 +5,9 @@ defmodule Lightning.Adaptors do A scheduler fetches the catalogue from the configured source and persists it. Reads check an in-memory cache first and the database - second. The first read against an empty catalogue triggers the initial - load and waits for it, bounded by the first-load timeout, and returns an - error if the load does not complete in time. + second, and answer immediately: a read that finds nothing in a catalogue + which has never loaded is `{:error, :not_ready}`. Waiting for that first + load is opt-in, through `ensure_loaded/2` and `fetch_adaptor/3`. ## Adaptor specs @@ -81,9 +81,10 @@ defmodule Lightning.Adaptors do end @doc """ - Returns every adaptor in the catalogue as `Package` structs. + Returns every adaptor in the catalogue as `Package` structs, or + `{:error, :not_ready}` against a catalogue that has never loaded. """ - @spec packages(atom()) :: {:ok, [Package.t()]} | {:error, :timeout | term()} + @spec packages(atom()) :: {:ok, [Package.t()]} | {:error, term()} def packages(sup \\ Config.default_instance()) do with {:ok, metas} <- Store.packages(sup) do source = AdaptorsSupervisor.source(sup) @@ -94,14 +95,11 @@ defmodule Lightning.Adaptors do @doc """ Returns the credential schema of the adaptor named `pkg`, as a JSON binary. An adaptor with no schema yields `"{}"` and an unknown name - is `{:error, :not_found}`. - - Against a catalogue that has never loaded, waits for the first load; - the other errors are then those of `fetch_adaptor/2`. + is `{:error, :not_found}`, or `{:error, :not_ready}` against a catalogue + that has never loaded. """ @spec schema(atom(), String.t()) :: - {:ok, String.t()} - | {:error, :not_found | :timeout | :unavailable | :not_ready} + {:ok, String.t()} | {:error, :not_found | :not_ready} def schema(sup \\ Config.default_instance(), pkg), do: Store.schema(sup, pkg) @@ -116,7 +114,7 @@ defmodule Lightning.Adaptors do unchanged without consulting the catalogue. Waits for the catalogue's first load if it has never loaded, and returns - the `fetch_adaptor/2` errors other than `:not_found` when it cannot get + the `fetch_adaptor/3` errors other than `:not_found` when it cannot get an answer at all. """ @spec resolve_name(atom(), String.t()) :: @@ -147,7 +145,8 @@ defmodule Lightning.Adaptors do @doc """ Returns the on-disk path of the adaptor's `:square` or `:rectangle` - icon, fetching it on the first request. + icon, fetching it on the first request. `{:error, :not_ready}` against a + catalogue that has never loaded. """ @spec icon(atom(), String.t(), :square | :rectangle) :: {:ok, Path.t()} | {:error, term()} @@ -174,29 +173,30 @@ defmodule Lightning.Adaptors do @doc """ Returns `{:ok, adaptor}` for the adaptor named `name`, waiting for the - catalogue's first load if it has never loaded. + catalogue's first load if it has never loaded, bounded by `:timeout` + (default `Lightning.Adaptors.Config.first_load_timeout/0`). Takes a bare package name, not a spec; see `parse_spec/1`. Errors: * `{:error, :not_found}` - the loaded catalogue has no such adaptor - * `{:error, :timeout}` - the first load did not finish within - `Lightning.Adaptors.Config.first_load_timeout/0` + * `{:error, :timeout}` - the first load did not finish in time * `{:error, :unavailable}` - no Scheduler process is reachable * `{:error, :not_ready}` - the load ran but left the catalogue empty """ - @spec fetch_adaptor(atom(), String.t()) :: + @spec fetch_adaptor(atom(), String.t(), keyword()) :: {:ok, Package.t()} | {:error, :not_found | :timeout | :unavailable | :not_ready} - def fetch_adaptor(sup \\ Config.default_instance(), name) + def fetch_adaptor(sup \\ Config.default_instance(), name, opts \\ []) when is_binary(name) do case Store.packages(sup) do {:ok, metas} -> resolve_meta(sup, name, Enum.find(metas, &(&1.name == name))) - {:error, reason} when reason in [:timeout, :unavailable, :not_ready] -> - {:error, reason} + {:error, :not_ready} -> + with :ok <- Store.ensure_loaded(sup, opts), + do: fetch_adaptor(sup, name, opts) # Any other failure is the cache's, and says nothing about the row. {:error, _cache} -> @@ -221,15 +221,16 @@ defmodule Lightning.Adaptors do @doc """ Waits until the catalogue has loaded at least once, triggering the - first load if needed. + first load if needed, bounded by `:timeout` (default + `Lightning.Adaptors.Config.first_load_timeout/0`). - Returns `:ok`, or one of the `fetch_adaptor/2` errors other than + Returns `:ok`, or one of the `fetch_adaptor/3` errors other than `:not_found`. """ - @spec ensure_loaded(atom()) :: + @spec ensure_loaded(atom(), keyword()) :: :ok | {:error, :timeout | :unavailable | :not_ready} - def ensure_loaded(sup \\ Config.default_instance()), - do: Store.ensure_loaded(sup) + def ensure_loaded(sup \\ Config.default_instance(), opts \\ []), + do: Store.ensure_loaded(sup, opts) @doc """ Splits an adaptor spec into `{:ok, {name, version}}`, with `version` `nil` @@ -345,7 +346,7 @@ defmodule Lightning.Adaptors do `t:Lightning.Adaptors.Store.icon_meta/0`. """ @spec icon_meta(atom(), String.t()) :: - {:ok, Store.icon_meta()} | {:error, :not_found} + {:ok, Store.icon_meta()} | {:error, :not_found | :not_ready} def icon_meta(sup \\ Config.default_instance(), name), do: Store.icon_meta(sup, name) diff --git a/lib/lightning/adaptors/local.ex b/lib/lightning/adaptors/local.ex index 2081288ed49..dfb35d26d3f 100644 --- a/lib/lightning/adaptors/local.ex +++ b/lib/lightning/adaptors/local.ex @@ -28,6 +28,10 @@ defmodule Lightning.Adaptors.Local do and a single `Logger.warning` names every shadowed package once per scan. + A checkout with no packages in it lists `{:ok, []}`: the directory is + readable and really is empty, which is a different thing from an npm org + listing that comes back with nothing in it. + `source: :local` is **not** set here — the Store stamps it before upsert. No network calls anywhere in this module. """ diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index ede808b1a27..fb0eed0356d 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -10,7 +10,8 @@ defmodule Lightning.Adaptors.NPM do cheap version lookup, returning `name + latest_version` for every `@openfn/language-*` package. See `Lightning.Adaptors.NPM.Registry` for why this is two calls, not - one. + one. A listing with no `@openfn/language-*` names is an error, not + an empty catalogue. * `c:Lightning.Adaptors.Strategy.fetch_adaptor/1` — packument fetch + per-version decode and latest-version schema retrieval via jsDelivr. Icon fields are **not** stamped here; the Scheduler diff --git a/lib/lightning/adaptors/npm/registry.ex b/lib/lightning/adaptors/npm/registry.ex index 7b30e1461e9..c355f0856b9 100644 --- a/lib/lightning/adaptors/npm/registry.ex +++ b/lib/lightning/adaptors/npm/registry.ex @@ -48,6 +48,11 @@ defmodule Lightning.Adaptors.NPM.Registry do `/-/v1/search` (cheap version lookup), falling back to a per-name packument fetch for any name search doesn't cover. See the moduledoc for why this isn't a single call. + + A listing holding no `@openfn/language-*` names is `{:error, :empty_listing}` + and a 200 whose body is not a map is `{:error, :malformed_listing}`: an org + with hundreds of packages does not empty out, so an empty answer is a broken + registry rather than knowledge that no adaptors exist. """ @spec list_adaptors() :: {:ok, [%{name: String.t(), latest_version: String.t()}]} @@ -145,12 +150,16 @@ defmodule Lightning.Adaptors.NPM.Registry do defp scoped_package_names do case Tesla.get(json_client(), "/-/user/openfn/package") do {:ok, %Tesla.Env{status: 200, body: body}} when is_map(body) -> - names = - body - |> Map.keys() - |> Enum.filter(&String.starts_with?(&1, @language_prefix)) - - {:ok, names} + case Enum.filter( + Map.keys(body), + &String.starts_with?(&1, @language_prefix) + ) do + [] -> {:error, :empty_listing} + names -> {:ok, names} + end + + {:ok, %Tesla.Env{status: 200}} -> + {:error, :malformed_listing} {:ok, %Tesla.Env{status: status}} -> {:error, {:http_status, status}} diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index 9c9dccaef03..b82635814d2 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -95,12 +95,15 @@ defmodule Lightning.Adaptors.Scheduler do and upsert failures are counted in `counts.errors`), `{:error, reason}` when it failed, or `{:error, {:refresh_failed, reason}}` when the cycle crashed. + + A caller whose `timeout` expires before the cycle finishes is dropped + rather than replied to, so a late result never lands in its mailbox. """ @spec await_refresh(GenServer.server(), timeout()) :: {:ok, refresh_counts()} | {:error, {:refresh_failed, term()} | term()} def await_refresh(scheduler_name, timeout) do - GenServer.call(scheduler_name, :await_refresh, timeout) + GenServer.call(scheduler_name, {:await_refresh, timeout}, timeout) end @doc """ @@ -266,7 +269,7 @@ defmodule Lightning.Adaptors.Scheduler do "#{length(state.waiters)} waiter(s)" ) - Enum.each(state.waiters, &GenServer.reply(&1, result)) + reply_waiters(state.waiters, result) completed? = state.completed? or match?({:ok, %{listed: 0, errors: 0}}, result) @@ -283,10 +286,7 @@ defmodule Lightning.Adaptors.Scheduler do "replying error to #{length(state.waiters)} waiter(s)" ) - Enum.each( - state.waiters, - &GenServer.reply(&1, {:error, {:refresh_failed, reason}}) - ) + reply_waiters(state.waiters, {:error, {:refresh_failed, reason}}) {:noreply, %{state | refresh: nil, waiters: []}} end @@ -338,6 +338,22 @@ defmodule Lightning.Adaptors.Scheduler do {:noreply, state} end + defp deadline(:infinity), do: :infinity + + defp deadline(timeout) when is_integer(timeout), + do: System.monotonic_time(:millisecond) + timeout + + # A caller that outlived its own timeout is no longer expecting a reply. + defp reply_waiters(waiters, result) do + now = System.monotonic_time(:millisecond) + + Enum.each(waiters, fn {from, deadline} -> + if deadline == :infinity or deadline > now do + GenServer.reply(from, result) + end + end) + end + @impl true def handle_call(:completed?, _from, state) do {:reply, state.completed?, state} @@ -348,12 +364,12 @@ defmodule Lightning.Adaptors.Scheduler do {:reply, :ok, maybe_start_refresh(state)} end - def handle_call(:await_refresh, from, state) do + def handle_call({:await_refresh, timeout}, from, state) do Logger.debug( "Adaptors[#{state.source}]: await_refresh attached (#{length(state.waiters) + 1} waiters)" ) - state = %{state | waiters: [from | state.waiters]} + state = %{state | waiters: [{from, deadline(timeout)} | state.waiters]} {:noreply, maybe_start_refresh(state)} end diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index 415f75dad2a..5d92d434a9c 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -13,14 +13,14 @@ defmodule Lightning.Adaptors.Store do ## The first load - Every read is gated on the catalogue having loaded at least once. A read - that comes back with data answers immediately; one that comes back empty - or not-found asks whether the catalogue has ever loaded, and if it has - not, triggers the first load, waits for it and reads again. An empty - answer from a catalogue that has loaded is a real answer and is returned - as-is — including the empty catalogue a source with no adaptors leaves - behind, which the Scheduler reports as loaded despite there being no row - to find. + Reads never block. A read that comes back empty or not-found asks whether + the catalogue has ever loaded, and answers `{:error, :not_ready}` when it + has not. An empty answer from a catalogue that has loaded is a real answer + and is returned as-is — including the empty catalogue a source with no + adaptors leaves behind, which the Scheduler reports as loaded despite there + being no row to find. + + Waiting for the first load is opt-in, through `ensure_loaded/2`. """ alias Lightning.Adaptors.Catalogue @@ -66,7 +66,7 @@ defmodule Lightning.Adaptors.Store do `{:error, :not_found}`. """ @spec schema(sup(), String.t()) :: {:ok, String.t()} | {:error, term()} - def schema(sup, name), do: gated(sup, fn -> read_schema(sup, name) end) + def schema(sup, name), do: settled(sup, read_schema(sup, name)) defp read_schema(sup, name) do cache = AdaptorsSupervisor.cache_name(sup) @@ -99,7 +99,7 @@ defmodule Lightning.Adaptors.Store do @spec icon(sup(), String.t(), :square | :rectangle) :: {:ok, Path.t()} | {:error, :not_found | term()} def icon(sup, name, shape) when shape in [:square, :rectangle] do - gated(sup, fn -> read_icon(sup, name, shape) end) + settled(sup, read_icon(sup, name, shape)) end defp read_icon(sup, name, shape) do @@ -155,7 +155,7 @@ defmodule Lightning.Adaptors.Store do `dependencies` and `peer_dependencies` columns. """ @spec packages(sup()) :: {:ok, [package_meta()]} | {:error, term()} - def packages(sup), do: gated(sup, fn -> read_packages(sup) end) + def packages(sup), do: settled(sup, read_packages(sup)) defp read_packages(sup) do cache = AdaptorsSupervisor.cache_name(sup) @@ -184,7 +184,7 @@ defmodule Lightning.Adaptors.Store do can never drift apart. """ @spec catalogue(sup()) :: {:ok, catalogue()} | {:error, term()} - def catalogue(sup), do: gated(sup, fn -> read_catalogue(sup) end) + def catalogue(sup), do: settled(sup, read_catalogue(sup)) defp read_catalogue(sup) do cache = AdaptorsSupervisor.cache_name(sup) @@ -209,8 +209,8 @@ defmodule Lightning.Adaptors.Store do without touching disk, or `{:error, :not_found}` for an unknown name. """ @spec icon_meta(sup(), String.t()) :: - {:ok, icon_meta()} | {:error, :not_found} - def icon_meta(sup, name), do: gated(sup, fn -> read_icon_meta(sup, name) end) + {:ok, icon_meta()} | {:error, :not_found | :not_ready} + def icon_meta(sup, name), do: settled(sup, read_icon_meta(sup, name)) defp read_icon_meta(sup, name) do cache = AdaptorsSupervisor.cache_name(sup) @@ -232,27 +232,30 @@ defmodule Lightning.Adaptors.Store do @doc """ Waits until the catalogue has loaded at least once, triggering the first - load if needed. + load if needed. `:timeout` bounds the wait, defaulting to + `Lightning.Adaptors.Config.first_load_timeout/0`. - Returns `:ok`, `{:error, :timeout}` if the load did not finish within - `Lightning.Adaptors.Config.first_load_timeout/0`, `{:error, :unavailable}` - if no Scheduler is reachable, or `{:error, :not_ready}` if the load ran - and could neither write a row nor report a complete cycle. + Returns `:ok`, `{:error, :timeout}` if the load did not finish in time, + `{:error, :unavailable}` if no Scheduler is reachable, or + `{:error, :not_ready}` if the load ran and could neither write a row nor + report a complete cycle. """ - @spec ensure_loaded(sup()) :: + @spec ensure_loaded(sup(), keyword()) :: :ok | {:error, :timeout | :unavailable | :not_ready} - def ensure_loaded(sup) do - if loaded?(sup), do: :ok, else: first_load(sup) + def ensure_loaded(sup, opts \\ []) do + if loaded?(sup) do + :ok + else + first_load(sup, opts[:timeout] || Config.first_load_timeout()) + end end # Only an empty answer pays for the `loaded?/1` query: a read that found # data cannot be waiting on the first load, whatever the catalogue's # state. - defp gated(sup, read) do - result = read.() - + defp settled(sup, result) do if empty?(result) and not loaded?(sup) do - with :ok <- first_load(sup), do: read.() + {:error, :not_ready} else result end @@ -273,18 +276,18 @@ defmodule Lightning.Adaptors.Store do # A failed cycle can land on rows a seed already wrote, so the outcome # is re-read rather than inferred from the refresh result. - defp first_load(sup) do - case await_refresh(sup) do + defp first_load(sup, timeout) do + case await_refresh(sup, timeout) do {:error, :timeout} -> {:error, :timeout} {:error, :unavailable} -> {:error, :unavailable} _other -> if loaded?(sup), do: :ok, else: {:error, :not_ready} end end - defp await_refresh(sup) do + defp await_refresh(sup, timeout) do Scheduler.await_refresh( AdaptorsSupervisor.global_scheduler_name(sup), - Config.first_load_timeout() + timeout ) catch :exit, {:timeout, _} -> {:error, :timeout} diff --git a/lib/lightning/adaptors/strategy.ex b/lib/lightning/adaptors/strategy.ex index cec1bc9c71b..a1683250790 100644 --- a/lib/lightning/adaptors/strategy.ex +++ b/lib/lightning/adaptors/strategy.ex @@ -145,6 +145,13 @@ defmodule Lightning.Adaptors.Strategy do Cheap change-signal listing: `name + latest_version` for every `@openfn/*` package known to the strategy. The scheduler diffs this against the `adaptors` table to compute its work list. + + `{:ok, []}` means the strategy looked and there is genuinely nothing + there, which settles the Store's first-load gate. A strategy that cannot + tell an empty source from an unreadable one must return `{:error, _}`: + `Lightning.Adaptors.Local` reports an empty checkout as `{:ok, []}`, + `Lightning.Adaptors.NPM` reports an empty org listing as + `{:error, :empty_listing}`. """ @callback list_adaptors() :: {:ok, [%{name: String.t(), latest_version: String.t()}]} diff --git a/lib/lightning_web/controllers/adaptor_controller.ex b/lib/lightning_web/controllers/adaptor_controller.ex index 0f14b907684..f1c747eaab3 100644 --- a/lib/lightning_web/controllers/adaptor_controller.ex +++ b/lib/lightning_web/controllers/adaptor_controller.ex @@ -6,6 +6,9 @@ defmodule LightningWeb.AdaptorController do are cached together by `Lightning.Adaptors.Store.catalogue/1`, so a matching `If-None-Match` answers 304 without touching Postgres, and a miss on the ETag still serves an already-rendered payload. + + A catalogue that has never loaded answers 503 with a `retry-after`, not + an empty list: the picker shows its retry state instead of no adaptors. """ use LightningWeb, :controller @@ -38,6 +41,7 @@ defmodule LightningWeb.AdaptorController do ) conn + |> put_resp_header("retry-after", "5") |> put_status(:service_unavailable) |> json(%{"error" => "adaptor catalogue unavailable"}) end diff --git a/lib/lightning_web/controllers/adaptor_icon_controller.ex b/lib/lightning_web/controllers/adaptor_icon_controller.ex index 76bb8535e2a..ab44ce5b029 100644 --- a/lib/lightning_web/controllers/adaptor_icon_controller.ex +++ b/lib/lightning_web/controllers/adaptor_icon_controller.ex @@ -83,8 +83,13 @@ defmodule LightningWeb.AdaptorIconController do shape = String.to_existing_atom(shape) case Adaptors.icon_meta(name) do - # A catalogue that will not load has no icon to serve either, so its - # errors are 404s here rather than a 500. + # A catalogue that has never loaded may yet have this icon; anything + # else is a 404 here rather than a 500. + {:error, :not_ready} -> + conn + |> put_resp_header("retry-after", "5") + |> send_resp(503, "") + {:error, _reason} -> send_resp(conn, 404, "") diff --git a/lib/mix/tasks/lightning.adaptors.refresh.ex b/lib/mix/tasks/lightning.adaptors.refresh.ex index b1f2a13faff..70d90af140c 100644 --- a/lib/mix/tasks/lightning.adaptors.refresh.ex +++ b/lib/mix/tasks/lightning.adaptors.refresh.ex @@ -15,7 +15,7 @@ defmodule Mix.Tasks.Lightning.Adaptors.Refresh do * `0` - success * `1` - package name not found - * `2` - any other error, including a listing that returned no adaptors + * `2` - any other error, including a source that could not be listed or a refresh that took longer than 10 minutes """ @@ -43,10 +43,6 @@ defmodule Mix.Tasks.Lightning.Adaptors.Refresh do await: true, timeout: @await_timeout ) do - {:ok, %{listed: 0}} -> - Mix.shell().error("Refresh completed but the source listed no adaptors.") - exit({:shutdown, 2}) - {:ok, counts} -> duration_s = div(System.monotonic_time(:millisecond) - started, 1000) diff --git a/test/lightning/adaptors/npm/registry_test.exs b/test/lightning/adaptors/npm/registry_test.exs index e8b701955f8..2ff8198a146 100644 --- a/test/lightning/adaptors/npm/registry_test.exs +++ b/test/lightning/adaptors/npm/registry_test.exs @@ -35,22 +35,26 @@ defmodule Lightning.Adaptors.NPM.RegistryTest do end describe "list_adaptors/0" do - test "returns an empty list when the org has no packages", %{ + test "an org listing with no language packages is an error", %{ bypass: bypass } do Bypass.expect(bypass, "GET", "/-/user/openfn/package", fn conn -> json_resp(conn, 200, %{}) end) - Bypass.expect(bypass, "GET", "/-/v1/search", fn conn -> - conn = Plug.Conn.fetch_query_params(conn) - assert conn.query_params["text"] == "@openfn" - assert conn.query_params["size"] == "250" - + Bypass.stub(bypass, "GET", "/-/v1/search", fn conn -> json_resp(conn, 200, %{"objects" => []}) end) - assert {:ok, []} = Registry.list_adaptors() + assert {:error, :empty_listing} = Registry.list_adaptors() + end + + test "an org listing that is not a map is an error", %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/-/user/openfn/package", fn conn -> + json_resp(conn, 200, ["@openfn/language-http"]) + end) + + assert {:error, :malformed_listing} = Registry.list_adaptors() end test "returns name + latest_version for each authoritative name", %{ diff --git a/test/lightning/adaptors/readiness_test.exs b/test/lightning/adaptors/readiness_test.exs index 17597994ec8..1dc1c483d12 100644 --- a/test/lightning/adaptors/readiness_test.exs +++ b/test/lightning/adaptors/readiness_test.exs @@ -1,6 +1,6 @@ defmodule Lightning.Adaptors.ReadinessTest do @moduledoc """ - `fetch_adaptor/2` and `ensure_loaded/1` against an isolated supervisor: + `fetch_adaptor/3` and `ensure_loaded/2` against an isolated supervisor: a populated catalogue never contacts the Scheduler, an empty one waits for exactly one coalesced load, and every failure mode of that wait maps to its error atom. @@ -251,9 +251,25 @@ defmodule Lightning.Adaptors.ReadinessTest do assert Process.alive?(pid) end + + test "bounds the wait with its own :timeout option", %{sup: sup} do + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> + Process.sleep(300) + {:ok, []} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + start_scheduler(sup) + + assert {:error, :timeout} = + Adaptors.fetch_adaptor(sup, "@openfn/language-http", timeout: 50) + end end - describe "ensure_loaded/1" do + describe "ensure_loaded/2" do test "returns :ok immediately when rows exist, without contacting the Scheduler", %{sup: sup} do {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index f6ee027692c..7fdbc4876fc 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -773,6 +773,28 @@ defmodule Lightning.Adaptors.SchedulerTest do Scheduler.await_refresh(sched_name, 5_000) end + test "does not reply to a waiter whose timeout expired", %{sup: sup} do + {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) + + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, 1, fn -> + Process.sleep(300) + {:ok, []} + end) + + pid = start_scheduler(sup) + + # The call is assembled by hand because `GenServer.call/3` replies to + # an alias the VM drops once the caller times out, which would hide a + # reply the Scheduler should never have sent. + expired = make_ref() + live = make_ref() + send(pid, {:"$gen_call", {self(), expired}, {:await_refresh, 50}}) + send(pid, {:"$gen_call", {self(), live}, {:await_refresh, 30_000}}) + + assert_receive {^live, {:ok, %{listed: 0}}}, 2_000 + refute_received {^expired, _result} + end + test "returns the upstream listing failure to waiters", %{sup: sup} do {:ok, _} = Catalogue.upsert_adaptor(adaptor_record()) diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index af0708b2b3a..af8380fa74e 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -33,7 +33,7 @@ defmodule Lightning.Adaptors.StoreTest do end # Replaces the supervisor's own Highlander-wrapped Scheduler with one - # this test owns, so a gated read's await_refresh lands on a process + # this test owns, so `ensure_loaded`'s await_refresh lands on a process # whose sandbox connection and Mox stubs are ours. defp start_scheduler(sup) do :ok = @@ -76,23 +76,34 @@ defmodule Lightning.Adaptors.StoreTest do end describe "the first-load gate" do - test "schema/2 on a never-loaded catalogue waits, then reads again", %{ + test "a never-loaded catalogue answers :not_ready without waiting", %{ sup: sup } do - expect_one_load([adaptor_record(schema_data: ~s({"type":"object"}))]) + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + flunk("a read must not trigger a load") + end) + start_scheduler(sup) - assert {:ok, ~s({"type":"object"})} = - Store.schema(sup, "@openfn/language-http") + assert {:error, :not_ready} = Store.packages(sup) + assert {:error, :not_ready} = Store.schema(sup, "@openfn/language-http") + assert {:error, :not_ready} = Store.catalogue(sup) + assert {:error, :not_ready} = Store.icon_meta(sup, "@openfn/language-http") + + assert {:error, :not_ready} = + Store.icon(sup, "@openfn/language-http", :square) end - test "packages/1 on a never-loaded catalogue waits, then reads again", %{ - sup: sup - } do - expect_one_load([adaptor_record()]) + test "ensure_loaded/2 waits for the first load, after which reads answer", + %{sup: sup} do + expect_one_load([adaptor_record(schema_data: ~s({"type":"object"}))]) start_scheduler(sup) + assert :ok = Store.ensure_loaded(sup) assert {:ok, [%{name: "@openfn/language-http"}]} = Store.packages(sup) + + assert {:ok, ~s({"type":"object"})} = + Store.schema(sup, "@openfn/language-http") end test "a loaded catalogue answers empty without a second load", %{sup: sup} do @@ -114,7 +125,8 @@ defmodule Lightning.Adaptors.StoreTest do expect_one_load([]) start_scheduler(sup) - assert {:ok, []} = Store.packages(sup) + assert :ok = Store.ensure_loaded(sup) + assert :ok = Store.ensure_loaded(sup) assert {:ok, []} = Store.packages(sup) assert {:error, :not_found} = Store.schema(sup, "@openfn/language-http") end @@ -136,14 +148,29 @@ defmodule Lightning.Adaptors.StoreTest do start_scheduler(sup) - assert {:error, :not_ready} = Store.packages(sup) + assert {:error, :not_ready} = Store.ensure_loaded(sup) end test "no reachable Scheduler is :unavailable", %{sup: sup} do :ok = Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) - assert {:error, :unavailable} = Store.packages(sup) + assert {:error, :unavailable} = Store.ensure_loaded(sup) + end + + test "ensure_loaded/2 gives up at its own :timeout", %{sup: sup} do + expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + Process.sleep(500) + {:ok, []} + end) + + stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> + {:ok, %{}} + end) + + start_scheduler(sup) + + assert {:error, :timeout} = Store.ensure_loaded(sup, timeout: 50) end end diff --git a/test/lightning_web/controllers/adaptor_controller_test.exs b/test/lightning_web/controllers/adaptor_controller_test.exs index 554b03b9b5c..cbad90a0356 100644 --- a/test/lightning_web/controllers/adaptor_controller_test.exs +++ b/test/lightning_web/controllers/adaptor_controller_test.exs @@ -144,6 +144,21 @@ defmodule LightningWeb.AdaptorControllerTest do assert json_response(conn, 503) == %{ "error" => "adaptor catalogue unavailable" } + + assert get_resp_header(conn, "retry-after") == ["5"] + end + + test "a catalogue that has never loaded is a 503, not an empty list", %{ + conn: conn + } do + conn = log_in_user(conn, insert(:user)) + + stub(Adaptors, :catalogue, fn -> {:error, :not_ready} end) + + conn = get(conn, ~p"/adaptors/catalogue") + + assert json_response(conn, 503) + assert get_resp_header(conn, "retry-after") == ["5"] end defp version_record(version) do diff --git a/test/lightning_web/controllers/adaptor_icon_controller_test.exs b/test/lightning_web/controllers/adaptor_icon_controller_test.exs index f1cdcbc9808..04f3964f961 100644 --- a/test/lightning_web/controllers/adaptor_icon_controller_test.exs +++ b/test/lightning_web/controllers/adaptor_icon_controller_test.exs @@ -364,6 +364,8 @@ defmodule LightningWeb.AdaptorIconControllerTest do describe "show/2 — 404" do test "adaptor not in DB", %{conn: conn} do + insert_adaptor(unique_adaptor_name()) + params = %{ "name" => "nonexistent-adaptor-#{System.unique_integer([:positive])}", "shape" => "square", @@ -502,10 +504,21 @@ defmodule LightningWeb.AdaptorIconControllerTest do end test "404 on unknown adaptor", %{conn: conn} do + insert_adaptor(unique_adaptor_name()) + conn = get(conn, "/adaptors/icons/nope/square-aabbccdd.png") assert conn.status == 404 end + + test "503 while the catalogue has never loaded", %{conn: conn} do + Lightning.Adaptors.Catalogue.delete_all_for_source(source()) + + conn = get(conn, "/adaptors/icons/nope/square-aabbccdd.png") + + assert conn.status == 503 + assert get_resp_header(conn, "retry-after") == ["5"] + end end describe "AdaptorIconURL.build/3" do diff --git a/test/mix/tasks/lightning.adaptors.refresh_test.exs b/test/mix/tasks/lightning.adaptors.refresh_test.exs index 1800d9711d7..d8edbb0247d 100644 --- a/test/mix/tasks/lightning.adaptors.refresh_test.exs +++ b/test/mix/tasks/lightning.adaptors.refresh_test.exs @@ -29,11 +29,23 @@ defmodule Mix.Tasks.Lightning.Adaptors.RefreshTest do assert msg =~ "errors 1" end - test "exits 2 when the cycle succeeds but the source listed no adaptors" do + test "a cycle that listed no adaptors is a success" do stub(Lightning.Adaptors, :refresh, fn _sup, _opts -> {:ok, %{listed: 0, changed: 0, fetched: 0, errors: 0}} end) + Mix.Tasks.Lightning.Adaptors.Refresh.run([]) + + assert_received {:mix_shell, :info, [_]} + assert_received {:mix_shell, :info, [msg]} + assert msg =~ "listed 0" + end + + test "exits 2 when the source could not be listed" do + stub(Lightning.Adaptors, :refresh, fn _sup, _opts -> + {:error, :empty_listing} + end) + assert catch_exit(Mix.Tasks.Lightning.Adaptors.Refresh.run([])) == {:shutdown, 2} diff --git a/test/mix/tasks/lightning.adaptors.snapshot_test.exs b/test/mix/tasks/lightning.adaptors.snapshot_test.exs index 31593294d1a..e19cc6eed05 100644 --- a/test/mix/tasks/lightning.adaptors.snapshot_test.exs +++ b/test/mix/tasks/lightning.adaptors.snapshot_test.exs @@ -52,7 +52,7 @@ defmodule Mix.Tasks.Lightning.Adaptors.SnapshotTest do json_resp(conn, 200, %{}) end) - Bypass.expect(registry, "GET", "/-/v1/search", fn conn -> + Bypass.stub(registry, "GET", "/-/v1/search", fn conn -> json_resp(conn, 200, %{"objects" => []}) end) From 158d0d69921964398ae56cde36d6f673dd17542d Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Mon, 14 Sep 2026 11:06:21 +0200 Subject: [PATCH 33/37] Fail readably when the catalogue cannot answer - Return an error from get_schema/1 instead of raising - Show a retry state in the credential form when the catalogue cannot answer - Log why setup is waiting on the adaptor catalogue - Document what a cold catalogue does while it loads - Resolve adaptor names without waiting for the first load --- ADAPTORS.md | 38 +++++- CHANGELOG.md | 10 ++ lib/lightning/adaptors.ex | 27 ++-- lib/lightning/credentials.ex | 24 ++-- lib/lightning/setup.ex | 27 +++- .../live/components/credentials.ex | 4 + .../credential_form_component.ex | 122 ++++++++++++------ .../json_schema_body_component.ex | 54 +++++++- test/lightning/adaptors_test.exs | 32 ++++- test/lightning/credentials/schema_test.exs | 14 +- test/lightning/credentials_test.exs | 47 ++++++- test/lightning/setup_test.exs | 31 +++++ .../live/credential_live_test.exs | 71 ++++++++++ 13 files changed, 426 insertions(+), 75 deletions(-) create mode 100644 test/lightning/setup_test.exs diff --git a/ADAPTORS.md b/ADAPTORS.md index 9d6d6bbc8b7..087dc7370bd 100644 --- a/ADAPTORS.md +++ b/ADAPTORS.md @@ -82,6 +82,12 @@ Internal mirrors: any npm-compatible registry works. Set `ADAPTORS_NPM_GITHUB_URL`, leave the strategy as npm, and set `ADAPTORS_NPM_GITHUB_REF` if the mirror serves a branch other than `main`. +A registry that answers but lists no `@openfn/language-*` packages is treated as +a failed listing, not as a catalogue with nothing in it. That is nearly always a +mistyped mirror URL or a mirror that has not synced the `@openfn` scope. The +rows already in Postgres stay as they are and the next refresh tries again. A +local checkout with no packages in it is genuinely empty, and is read as such. + An imported catalogue survives the hourly refresh; a failed one logs a warning and leaves rows alone. @@ -100,7 +106,9 @@ mix lightning.adaptors.refresh --name @openfn/language-http ``` Without `--name` it runs a full refresh and waits; with `--name` it refetches -that adaptor regardless of version change. Exit codes are in +that adaptor regardless of version change. A cycle that ran but wrote no rows +exits `0`, since an empty result from a readable source is not a failure; a +source that could not be listed at all exits `2`. The full list is in `mix help lightning.adaptors.refresh`. A release image has no Mix; run the same call against the node: @@ -110,6 +118,24 @@ bin/lightning rpc 'Lightning.Adaptors.refresh(await: true)' bin/lightning rpc 'Lightning.Adaptors.refresh_package("@openfn/language-http")' ``` +## While the catalogue is still loading + +A fresh instance has an empty catalogue until the first refresh lands. Reads do +not wait for it. The adaptor picker and the credential form show "Couldn't load +adaptors" with a Retry button, and `GET /adaptors/catalogue` replies 503 with a +`retry-after` header. An open editor picks the catalogue up on its own once the +load lands, without a page reload; the credential form recovers when you press +Retry, and the endpoint answers normally on the next request. + +Saving a workflow is the exception. It has to check the job's adaptor against +the catalogue before it can store it, so it waits for the first load, up to 90 +seconds, and rejects the save with "adaptor catalogue is not ready yet" if +nothing has arrived by then. + +So an instance that cannot reach npm at all comes up, serves every page and +retries in the background, but cannot save a workflow until the catalogue has +loaded once. Import a snapshot to give it one. + ## Troubleshooting - Adaptor missing from the picker: find its `fetch_adaptor` warning in the log, @@ -122,5 +148,11 @@ bin/lightning rpc 'Lightning.Adaptors.refresh_package("@openfn/language-http")' the same name; the log names each shadowed package. - Deprecated-variable boot warning: rename `LOCAL_ADAPTORS=true` to `ADAPTORS_STRATEGY=local` and `OPENFN_ADAPTORS_REPO` to `ADAPTORS_LOCAL_REPO`. -- Workflow save rejected with "adaptor catalogue is not ready yet": see - [Running without internet access](#running-without-internet-access). +- Picker stuck on "Couldn't load adaptors", or the catalogue endpoint answering + 503: the first load has not finished. See + [While the catalogue is still loading](#while-the-catalogue-is-still-loading). +- Workflow save rejected with "adaptor catalogue is not ready yet": the same + thing, ninety seconds in. If the instance cannot reach npm, import a snapshot; + see [Running without internet access](#running-without-internet-access). +- Refresh exiting `2` with `:empty_listing`: the registry answered but served no + `@openfn` packages. Check `ADAPTORS_NPM_REGISTRY_URL`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6baa7d973b4..9ddbfd5e18a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,6 +121,16 @@ and this project adheres to resolve, validate and run. [#4801](https://github.com/OpenFn/lightning/pull/4801) +- While the catalogue is loading for the first time, reads answer straight away + with a retry state rather than holding the caller for up to 90 seconds. The + adaptor picker and the credential form show "Couldn't load adaptors" with a + Retry button, and `GET /adaptors/catalogue` replies 503 with a `retry-after` + header. Saving a workflow still waits for the first load, because it has to + check the job's adaptor before it can store it. An npm registry that lists no + `@openfn/language-*` packages is now treated as a failed listing and retried + on the next refresh, rather than as a catalogue with nothing in it. + [#4801](https://github.com/OpenFn/lightning/pull/4801) + ### Removed - The AI assistant's "Send code" tickbox. The assistant reads your workflow to diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index d487dda3e9e..4547e856461 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -113,18 +113,17 @@ defmodule Lightning.Adaptors do `"raw"` and `"oauth"` are sentinels, not adaptor names, and are returned unchanged without consulting the catalogue. - Waits for the catalogue's first load if it has never loaded, and returns - the `fetch_adaptor/3` errors other than `:not_found` when it cannot get - an answer at all. + Answers from the catalogue as it stands, without waiting for a first + load: a catalogue that has never loaded is `{:error, :not_ready}`. """ @spec resolve_name(atom(), String.t()) :: - {:ok, String.t()} | {:error, :timeout | :unavailable | :not_ready} + {:ok, String.t()} | {:error, :not_ready} def resolve_name(sup \\ Config.default_instance(), name) def resolve_name(_sup, name) when name in ["raw", "oauth"], do: {:ok, name} def resolve_name(sup, name) do - case fetch_adaptor(sup, name) do + case lookup_adaptor(sup, name) do {:ok, _package} -> {:ok, name} {:error, :not_found} -> resolve_short_name(sup, name) {:error, _reason} = error -> error @@ -136,7 +135,7 @@ defmodule Lightning.Adaptors do defp resolve_short_name(sup, name) do full = PackageName.full_name(name) - case fetch_adaptor(sup, full) do + case lookup_adaptor(sup, full) do {:ok, _package} -> {:ok, full} {:error, :not_found} -> {:ok, name} {:error, _reason} = error -> error @@ -190,13 +189,23 @@ defmodule Lightning.Adaptors do | {:error, :not_found | :timeout | :unavailable | :not_ready} def fetch_adaptor(sup \\ Config.default_instance(), name, opts \\ []) when is_binary(name) do + case lookup_adaptor(sup, name) do + {:error, :not_ready} -> + with :ok <- Store.ensure_loaded(sup, opts), + do: lookup_adaptor(sup, name) + + result -> + result + end + end + + defp lookup_adaptor(sup, name) do case Store.packages(sup) do {:ok, metas} -> resolve_meta(sup, name, Enum.find(metas, &(&1.name == name))) - {:error, :not_ready} -> - with :ok <- Store.ensure_loaded(sup, opts), - do: fetch_adaptor(sup, name, opts) + {:error, :not_ready} = not_ready -> + not_ready # Any other failure is the cache's, and says nothing about the row. {:error, _cache} -> diff --git a/lib/lightning/credentials.ex b/lib/lightning/credentials.ex index 3ae96cb2d0e..4129c6a9d4a 100644 --- a/lib/lightning/credentials.ex +++ b/lib/lightning/credentials.ex @@ -584,15 +584,17 @@ defmodule Lightning.Credentials do @doc """ Creates a credential schema from credential json schema. + + `{:error, :not_found}` means the catalogue has no such adaptor; + `{:error, :not_ready}` means it has never loaded. """ - @spec get_schema(String.t()) :: Credentials.Schema.t() + @spec get_schema(String.t()) :: + {:ok, Credentials.Schema.t()} + | {:error, :not_found | :not_ready | term()} def get_schema(schema_name) do with {:ok, resolved} <- Lightning.Adaptors.resolve_name(schema_name), {:ok, schema_body} <- Lightning.Adaptors.schema(resolved) do - Credentials.Schema.new(schema_body, resolved) - else - {:error, reason} -> - raise "Error reading credential schema. Got: #{inspect(reason)}" + {:ok, Credentials.Schema.new(schema_body, resolved)} end end @@ -650,6 +652,13 @@ defmodule Lightning.Credentials do {:ok, updated_body} -> Ecto.Changeset.put_change(changeset, :body, updated_body) + {:error, reason} when reason in [:not_ready, :timeout, :unavailable] -> + Ecto.Changeset.add_error( + changeset, + :body, + "adaptor catalogue is not ready yet, try again shortly" + ) + {:error, _reason} -> Ecto.Changeset.add_error(changeset, :body, "Invalid body types") end @@ -662,9 +671,8 @@ defmodule Lightning.Credentials do do: {:ok, body} defp put_typed_body(body, schema_name) do - schema = get_schema(schema_name) - - with changeset <- SchemaDocument.changeset(body, schema: schema), + with {:ok, schema} <- get_schema(schema_name), + changeset <- SchemaDocument.changeset(body, schema: schema), {:ok, typed_body} <- Ecto.Changeset.apply_action(changeset, :insert) do updated_body = Enum.into(typed_body, body, fn {field, typed_value} -> diff --git a/lib/lightning/setup.ex b/lib/lightning/setup.ex index 0db7626a9d6..33e48272073 100644 --- a/lib/lightning/setup.ex +++ b/lib/lightning/setup.ex @@ -5,6 +5,8 @@ defmodule Lightning.Setup do alias Lightning.SetupUtils + require Logger + @doc """ This makes it possible to run setup_user as an external command @@ -47,11 +49,34 @@ defmodule Lightning.Setup do # Load the catalogue up front rather than let the first adaptor lookup # block inside `fun`'s transaction for the length of a source fetch. # A failure here surfaces at that lookup instead. - _ = Lightning.Adaptors.ensure_loaded() + load_adaptor_catalogue() fun.() end) end + defp load_adaptor_catalogue do + timeout = Lightning.Adaptors.Config.first_load_timeout() + + Logger.info( + "Loading the adaptor catalogue, which may take up to " <> + "#{div(timeout, 1000)}s on a first load." + ) + + case Lightning.Adaptors.ensure_loaded() do + :ok -> + :ok + + {:error, reason} -> + Logger.warning( + "The adaptor catalogue did not load (#{inspect(reason)}); " <> + "continuing without it. See the offline section of ADAPTORS.md " <> + "if this instance has no access to npm." + ) + + :ok + end + end + @deprecated "Use with_minimum_setup/1 instead" def ensure_minimum_setup do Lightning.Release.load_app() diff --git a/lib/lightning_web/live/components/credentials.ex b/lib/lightning_web/live/components/credentials.ex index ec1f0c2a226..4c5311f6c3b 100644 --- a/lib/lightning_web/live/components/credentials.ex +++ b/lib/lightning_web/live/components/credentials.ex @@ -87,6 +87,8 @@ defmodule LightningWeb.Components.Credentials do attr :current_body, :map, default: %{} attr :schema_changeset, :any, default: nil attr :raw_body_touched, :boolean, default: false + attr :target, :any, default: nil + attr :attempt, :integer, default: 0 slot :inner_block def form_component(%{type: "raw"} = assigns) do @@ -109,6 +111,8 @@ defmodule LightningWeb.Components.Credentials do form={@form} current_body={@current_body} schema_changeset={@schema_changeset} + target={@target} + attempt={@attempt} > {render_slot(@inner_block, l)} diff --git a/lib/lightning_web/live/credential_live/credential_form_component.ex b/lib/lightning_web/live/credential_live/credential_form_component.ex index 732860d0d5f..4e0fe2702cb 100644 --- a/lib/lightning_web/live/credential_live/credential_form_component.ex +++ b/lib/lightning_web/live/credential_live/credential_form_component.ex @@ -47,6 +47,8 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do show_modal: true, body_valid?: true, schema_changeset: nil, + schema_attempt: 0, + adaptors_error: nil, touched_body_fields: MapSet.new(), touched_raw_bodies: MapSet.new() } @@ -175,6 +177,17 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do {:noreply, socket} end + def handle_event("retry_schema", _, socket) do + {:noreply, + socket + |> assign(:schema_changeset, nil) + |> update(:schema_attempt, &(&1 + 1))} + end + + def handle_event("retry_adaptors", _, socket) do + {:noreply, assign_oauth_clients_and_type_options(socket)} + end + def handle_event("change_page", _, socket) do {:noreply, socket |> assign(page: :second)} end @@ -650,11 +663,19 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do touched_fields, _raw_touched ) do - schema = Credentials.get_schema(schema_name) - full_changeset = Credentials.SchemaDocument.changeset(body, schema: schema) - display_changeset = filter_errors_to_touched(full_changeset, touched_fields) + case Credentials.get_schema(schema_name) do + {:ok, schema} -> + full_changeset = + Credentials.SchemaDocument.changeset(body, schema: schema) + + display_changeset = + filter_errors_to_touched(full_changeset, touched_fields) + + {full_changeset.valid?, display_changeset} - {full_changeset.valid?, display_changeset} + {:error, _reason} -> + {false, nil} + end end defp filter_errors_to_touched(changeset, touched_fields) do @@ -706,6 +727,21 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do phx-target={@myself} phx-change="schema_selected" > +

+

Couldn't load adaptors. Please try again.

+ +
{fieldset} @@ -1174,29 +1212,25 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do end defp get_type_options do - adaptor_options = - case Adaptors.packages() do - {:ok, packages} -> - packages - |> Enum.filter(& &1.has_schema) - |> Enum.map(&adaptor_type_option/1) - - {:error, _} -> - [] - end + with {:ok, packages} <- Adaptors.packages() do + options = + packages + |> Enum.filter(& &1.has_schema) + |> Enum.map(&adaptor_type_option/1) + |> Enum.reject(fn {_, name, _, _} -> + name in ["@openfn/language-googlesheets", "@openfn/language-gmail"] + end) - adaptor_options - |> Enum.reject(fn {_, name, _, _} -> - name in ["@openfn/language-googlesheets", "@openfn/language-gmail"] - end) - |> Enum.concat([ - {"Raw JSON", "raw", - Routes.static_path( - LightningWeb.Endpoint, - "/images/raw.png" - ), nil} - ]) - |> Enum.sort_by(&String.downcase(elem(&1, 0)), :asc) + {:ok, options} + end + end + + defp raw_type_option do + {"Raw JSON", "raw", + Routes.static_path( + LightningWeb.Endpoint, + "/images/raw.png" + ), nil} end defp adaptor_type_option(%Adaptors.Package{name: name} = pkg) do @@ -1327,7 +1361,7 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do do: OauthClients.list_clients(project), else: OauthClients.list_clients(current_user) - type_options = + {type_options, adaptors_error} = if action == :new do keychain_option = if socket.assigns[:from_collab_editor] do @@ -1342,19 +1376,33 @@ defmodule LightningWeb.CredentialLive.CredentialFormComponent do [] end - get_type_options() - |> Enum.concat( - Enum.map(oauth_clients, fn client -> - {client.name, client.id, "/images/oauth-2.png", "oauth"} - end) - ) - |> Enum.concat(keychain_option) - |> Enum.sort_by(&String.downcase(elem(&1, 0)), :asc) + {adaptor_options, error} = + case get_type_options() do + {:ok, options} -> {options, nil} + {:error, reason} -> {[], reason} + end + + options = + adaptor_options + |> Enum.concat([raw_type_option()]) + |> Enum.concat( + Enum.map(oauth_clients, fn client -> + {client.name, client.id, "/images/oauth-2.png", "oauth"} + end) + ) + |> Enum.concat(keychain_option) + |> Enum.sort_by(&String.downcase(elem(&1, 0)), :asc) + + {options, error} else - [] + {[], nil} end - assign(socket, oauth_clients: oauth_clients, type_options: type_options) + assign(socket, + oauth_clients: oauth_clients, + type_options: type_options, + adaptors_error: adaptors_error + ) end defp format_schema_name(schema) when is_binary(schema) do diff --git a/lib/lightning_web/live/credential_live/json_schema_body_component.ex b/lib/lightning_web/live/credential_live/json_schema_body_component.ex index 036dedda146..be592bf8657 100644 --- a/lib/lightning_web/live/credential_live/json_schema_body_component.ex +++ b/lib/lightning_web/live/credential_live/json_schema_body_component.ex @@ -12,14 +12,23 @@ defmodule LightningWeb.CredentialLive.JsonSchemaBodyComponent do attr :form, :map, required: true attr :current_body, :map, default: %{} attr :schema_changeset, :any, default: nil + attr :target, :any, default: nil + # Bumped on each retry so this component re-renders and re-reads the schema. + attr :attempt, :integer, default: 0 slot :inner_block def fieldset(assigns) do changeset = assigns.form.source - schema = - changeset |> Ecto.Changeset.get_field(:schema) |> Credentials.get_schema() + case changeset + |> Ecto.Changeset.get_field(:schema) + |> Credentials.get_schema() do + {:ok, schema} -> loaded_fieldset(assigns, changeset, schema) + {:error, reason} -> unavailable_fieldset(assigns, reason) + end + end + defp loaded_fieldset(assigns, changeset, schema) do body = normalize_body(assigns.current_body) schema_changeset = assigns.schema_changeset || create_changeset(schema, body) @@ -43,6 +52,47 @@ defmodule LightningWeb.CredentialLive.JsonSchemaBodyComponent do """ end + defp unavailable_fieldset(assigns, reason) do + assigns = assign(assigns, :reason, reason) + + ~H""" + {render_slot( + @inner_block, + {Phoenix.LiveView.TagEngine.component( + &schema_unavailable/1, + [reason: @reason, target: @target], + {__ENV__.module, __ENV__.function, __ENV__.file, __ENV__.line} + ), false} + )} + """ + end + + attr :reason, :any, required: true + attr :target, :any, default: nil + + def schema_unavailable(assigns) do + ~H""" +
+

+ This adaptor isn't in the adaptor catalogue. +

+

Couldn't load adaptors. Please try again.

+ +
+ """ + end + defp inner(assigns) do body_form = to_form(assigns.schema_changeset, as: "credential[body]") assigns = assign(assigns, :body_form, body_form) diff --git a/test/lightning/adaptors_test.exs b/test/lightning/adaptors_test.exs index c159b70da1f..ac61018bacb 100644 --- a/test/lightning/adaptors_test.exs +++ b/test/lightning/adaptors_test.exs @@ -66,14 +66,25 @@ defmodule Lightning.AdaptorsTest do assert pkg.source == :npm end - test "an empty catalogue that has never loaded waits, then reports it", + test "an empty catalogue that has never loaded is :not_ready", %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + flunk("a read must not trigger a load") + end) + + assert {:error, :not_ready} = Adaptors.packages(sup) + end + + test "an empty catalogue that has loaded is an empty list", %{sup: sup} do stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> {:ok, []} end) stub(Lightning.Adaptors.StrategyMock, :fetch_icons, fn _opts -> {:ok, %{}} end) + start_scheduler(sup) + + assert :ok = Adaptors.ensure_loaded(sup) assert {:ok, []} = Adaptors.packages(sup) end @@ -147,10 +158,16 @@ defmodule Lightning.AdaptorsTest do Adaptors.schema(sup, "@openfn/language-http") end - test "waits for the first load and returns the schema", %{sup: sup} do + test "is :not_ready until the catalogue has loaded", %{sup: sup} do record = adaptor_record(schema_data: ~s({"type":"object"})) stub_refresh_cycle(record) + assert {:error, :not_ready} = + Adaptors.schema(sup, "@openfn/language-http") + + start_scheduler(sup) + assert :ok = Adaptors.ensure_loaded(sup) + assert {:ok, ~s({"type":"object"})} = Adaptors.schema(sup, "@openfn/language-http") end @@ -255,12 +272,13 @@ defmodule Lightning.AdaptorsTest do assert Adaptors.resolve_name(sup, "unknownish") == {:ok, "unknownish"} end - test "waits for the first load when the catalogue has never loaded", %{ - sup: sup - } do - stub_refresh_cycle(adaptor_record()) + test "answers :not_ready without waiting when the catalogue has never loaded", + %{sup: sup} do + stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> + flunk("resolve_name must not trigger a load") + end) - assert Adaptors.resolve_name(sup, "http") == {:ok, "@openfn/language-http"} + assert Adaptors.resolve_name(sup, "http") == {:error, :not_ready} end test "never resolves the raw and oauth sentinels, even if shadowed in the catalogue", diff --git a/test/lightning/credentials/schema_test.exs b/test/lightning/credentials/schema_test.exs index eaf4f23acb2..8e50ec11b34 100644 --- a/test/lightning/credentials/schema_test.exs +++ b/test/lightning/credentials/schema_test.exs @@ -340,7 +340,7 @@ defmodule Lightning.Credentials.SchemaTest do {:error, :unreachable} end) - schema = Credentials.get_schema("ordered-fixture") + {:ok, schema} = Credentials.get_schema("ordered-fixture") assert schema.fields == [:zeta, :alpha, :mu] end @@ -353,7 +353,7 @@ defmodule Lightning.Credentials.SchemaTest do end test "successfully validates field with json schema email format" do - schema = Credentials.get_schema("godata") + {:ok, schema} = Credentials.get_schema("godata") changeset = Ecto.Changeset.put_change( @@ -370,7 +370,7 @@ defmodule Lightning.Credentials.SchemaTest do end test "returns a changeset with 2 expected formats" do - schema = Credentials.get_schema("postgresql") + {:ok, schema} = Credentials.get_schema("postgresql") changeset = Ecto.Changeset.put_change( @@ -388,7 +388,7 @@ defmodule Lightning.Credentials.SchemaTest do end test "returns a changeset with 1 expected format and 2 allowed types" do - schema = Credentials.get_schema("http") + {:ok, schema} = Credentials.get_schema("http") changeset = Ecto.Changeset.put_change( @@ -406,7 +406,7 @@ defmodule Lightning.Credentials.SchemaTest do end test "treats object types as text (TEMP FIX)" do - schema = Credentials.get_schema("http") + {:ok, schema} = Credentials.get_schema("http") assert schema.types == %{ username: :string, @@ -418,7 +418,7 @@ defmodule Lightning.Credentials.SchemaTest do end test "returns a changeset with expected email format" do - schema = Credentials.get_schema("godata") + {:ok, schema} = Credentials.get_schema("godata") changeset = Ecto.Changeset.put_change( @@ -436,7 +436,7 @@ defmodule Lightning.Credentials.SchemaTest do end test "returns a changeset with no expected format and 2 allowed types" do - schema = Credentials.get_schema("dhis2") + {:ok, schema} = Credentials.get_schema("dhis2") changeset = Ecto.Changeset.put_change( diff --git a/test/lightning/credentials_test.exs b/test/lightning/credentials_test.exs index 543e5731bed..d995de4a664 100644 --- a/test/lightning/credentials_test.exs +++ b/test/lightning/credentials_test.exs @@ -2881,10 +2881,55 @@ defmodule Lightning.CredentialsTest do test "returns the adaptor's schema when one is present" do seed_credential_schema("http") - assert %Credentials.Schema{fields: fields} = + assert {:ok, %Credentials.Schema{fields: fields}} = Credentials.get_schema("@openfn/language-http") assert fields != [] end + + test "returns {:error, :not_found} for an adaptor the catalogue lacks" do + seed_credential_schema("http") + + assert {:error, :not_found} = + Credentials.get_schema("@openfn/language-never-existed") + end + + test "returns an error rather than raising when the catalogue cannot answer", + %{sup: sup} do + :ok = + Supervisor.terminate_child( + sup, + Lightning.Adaptors.Supervisor.highlander_name(sup) + ) + + assert {:error, :not_ready} = + Credentials.get_schema("@openfn/language-http") + end + + test "a save while the catalogue cannot answer says so on the body" do + user = insert(:user) + + Mimic.stub(Lightning.Adaptors, :resolve_name, fn name -> {:ok, name} end) + + Mimic.stub(Lightning.Adaptors, :schema, fn _name -> + {:error, :not_ready} + end) + + attrs = %{ + name: "a http credential", + user_id: user.id, + schema: "@openfn/language-http", + credential_bodies: [%{name: "main", body: %{"baseUrl" => "http://x"}}] + } + + assert {:error, %Ecto.Changeset{} = changeset} = + Credentials.create_credential(attrs, user) + + assert [ + credential_bodies: + {"Environment 1: body adaptor catalogue is not ready yet, try again shortly", + []} + ] = changeset.errors + end end end diff --git a/test/lightning/setup_test.exs b/test/lightning/setup_test.exs new file mode 100644 index 00000000000..e4c4dc5c03e --- /dev/null +++ b/test/lightning/setup_test.exs @@ -0,0 +1,31 @@ +defmodule Lightning.SetupTest do + use Lightning.DataCase, async: false + + import ExUnit.CaptureLog + import Mimic + + describe "with_minimum_setup/1" do + setup :set_mimic_from_context + + setup do + level = Logger.level() + Logger.configure(level: :info) + on_exit(fn -> Logger.configure(level: level) end) + :ok + end + + test "announces the catalogue load and warns when it fails" do + stub(Lightning.Adaptors, :ensure_loaded, fn -> {:error, :timeout} end) + + log = + capture_log(fn -> + assert {:ok, :done, _} = + Lightning.Setup.with_minimum_setup(fn -> :done end) + end) + + assert log =~ "Loading the adaptor catalogue" + assert log =~ ":timeout" + assert log =~ "ADAPTORS.md" + end + end +end diff --git a/test/lightning_web/live/credential_live_test.exs b/test/lightning_web/live/credential_live_test.exs index 46367d9245c..0401a22037a 100644 --- a/test/lightning_web/live/credential_live_test.exs +++ b/test/lightning_web/live/credential_live_test.exs @@ -3909,4 +3909,75 @@ defmodule LightningWeb.CredentialLiveTest do refute html =~ ~s(value="main_user") end end + + describe "when the adaptor catalogue cannot answer" do + setup %{sup: sup} do + Lightning.Repo.delete_all(Lightning.Adaptors.Catalogue.Adaptor) + Cachex.clear(Lightning.Adaptors.Supervisor.cache_name(sup)) + + :ok = + Supervisor.terminate_child( + sup, + Lightning.Adaptors.Supervisor.highlander_name(sup) + ) + + :ok + end + + test "the credential type picker offers a retry", %{conn: conn} do + {:ok, view, _html} = live(conn, ~p"/credentials") + + html = open_create_credential_modal(view) + + assert html =~ "Couldn't load adaptors. Please try again." + + assert has_element?( + view, + "#credential-type-adaptors-error button", + "Retry" + ) + + assert html =~ "Raw JSON" + + seed_credential_schema("http") + + html = + view + |> element("#credential-type-adaptors-error button", "Retry") + |> render_click() + + refute html =~ "Couldn't load adaptors. Please try again." + assert html =~ "credential-schema-picker_selected_@openfn/language-http" + end + + test "the credential form offers a retry instead of crashing", %{ + conn: conn, + user: user + } do + credential = + insert(:credential, user: user, schema: "@openfn/language-http") + + {:ok, view, _html} = live(conn, ~p"/credentials", on_error: :raise) + + html = open_edit_credential_modal(view, credential.id) + + assert html =~ "Couldn't load adaptors. Please try again." + + assert has_element?( + view, + "#credential-schema-unavailable button", + "Retry" + ) + + seed_credential_schema("http") + + html = + view + |> element("#credential-schema-unavailable button", "Retry") + |> render_click() + + refute html =~ "Couldn't load adaptors. Please try again." + assert html =~ "baseUrl" + end + end end From ea9f963a919c66bf105c550923bf6b27da3fd9b7 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Mon, 14 Sep 2026 12:12:07 +0200 Subject: [PATCH 34/37] Trim and correct comments across the adaptors branch --- .../guidelines/testable-supervision-trees.md | 15 +-- ADAPTORS.md | 25 +++-- .../inspector/TemplatePublishPanel.tsx | 3 +- .../stores/createAdaptorStore.ts | 11 +-- assets/js/yaml/util.ts | 11 +-- .../__helpers__/sessionStoreHelpers.ts | 4 +- .../components/ChatInput.test.tsx | 13 ++- .../inspector/CodeViewPanel.test.tsx | 8 +- .../createAdaptorStore.test.ts | 10 +- .../hooks/useAIWorkflowUndo.test.ts | 2 +- .../collaborative-editor/useAdaptors.test.tsx | 1 - .../utils/workflowDiff.test.ts | 4 +- assets/test/utils/nameValidation.test.ts | 37 ++++---- assets/test/yaml/blockScalars.test.ts | 4 +- assets/test/yaml/edgeKeys.test.ts | 8 +- assets/test/yaml/util.test.ts | 30 +++--- bin/adaptor_cache | 5 +- lib/lightning/adaptor_service.ex | 4 +- lib/lightning/adaptors.ex | 19 ++-- lib/lightning/adaptors/catalogue.ex | 61 ++++++------ lib/lightning/adaptors/channel_broadcaster.ex | 30 +++--- lib/lightning/adaptors/dump.ex | 4 +- lib/lightning/adaptors/icon_cache.ex | 46 ++++----- lib/lightning/adaptors/icon_field.ex | 8 +- lib/lightning/adaptors/invalidator.ex | 24 +++-- lib/lightning/adaptors/local.ex | 6 +- lib/lightning/adaptors/node_monitor.ex | 7 +- lib/lightning/adaptors/npm.ex | 75 +++++++-------- lib/lightning/adaptors/npm/github.ex | 47 ++++------ lib/lightning/adaptors/npm/registry.ex | 25 +++-- lib/lightning/adaptors/npm/schema.ex | 18 ++-- lib/lightning/adaptors/package_name.ex | 2 +- lib/lightning/adaptors/scheduler.ex | 71 +++++++------- lib/lightning/adaptors/seed.ex | 6 +- lib/lightning/adaptors/store.ex | 31 +++---- lib/lightning/adaptors/strategy.ex | 93 ++++++++----------- lib/lightning/collections/item.ex | 2 - lib/lightning/config/bootstrap.ex | 22 ++--- lib/lightning/credentials/credential.ex | 2 +- .../credentials/schema_reconciler.ex | 10 +- lib/lightning/export_utils.ex | 5 +- lib/lightning/export_utils/scalar.ex | 7 +- lib/lightning/projects/provisioner.ex | 9 +- lib/lightning/projects/sandboxes.ex | 11 +-- lib/lightning/release.ex | 12 +-- lib/lightning/runs/handlers.ex | 11 +-- lib/lightning/utils/validators.ex | 6 +- lib/lightning/workflows/edge.ex | 2 +- lib/lightning/workflows/job.ex | 3 +- lib/lightning/workflows/workflow.ex | 2 +- lib/lightning/workflows/workflow_template.ex | 5 +- .../channels/workflow_channel.ex | 29 +++--- .../controllers/adaptor_controller.ex | 5 +- .../controllers/adaptor_icon_controller.ex | 39 ++++---- .../live/maintenance_live/index.ex | 12 +-- .../project_live/github_sync_component.ex | 6 +- lib/lightning_web/live/sandbox_live/index.ex | 14 ++- lib/mix/tasks/lightning.adaptors.dump.ex | 15 ++- lib/mix/tasks/lightning.adaptors.import.ex | 2 +- lib/mix/tasks/lightning.adaptors.snapshot.ex | 2 +- mix.exs | 5 +- test/lightning/adaptor_service_test.exs | 2 +- .../adaptors/channel_broadcaster_test.exs | 4 +- .../adaptors/end_to_end_broadcast_test.exs | 2 +- .../adaptors/highlander_integration_test.exs | 8 +- test/lightning/adaptors/invalidator_test.exs | 2 - test/lightning/adaptors/node_monitor_test.exs | 8 +- test/lightning/adaptors/readiness_test.exs | 4 +- test/lightning/adaptors/scheduler_test.exs | 36 +++---- test/lightning/adaptors/store_test.exs | 22 ++--- .../adaptors/supervisor_integration_test.exs | 9 +- test/lightning/collections_test.exs | 3 +- .../lightning/credentials/credential_test.exs | 8 +- .../credentials/schema_reconciler_test.exs | 6 +- test/lightning/export_utils/scalar_test.exs | 33 ++++--- test/lightning/export_utils_test.exs | 25 +++-- test/lightning/projects/provisioner_test.exs | 6 +- test/lightning/sandboxes_test.exs | 4 +- test/lightning/utils/validators_test.exs | 4 +- test/lightning/version_control_test.exs | 2 +- test/lightning/workflow_templates_test.exs | 2 +- test/lightning/workflows/edge_test.exs | 18 ++-- test/lightning/workflows/job_test.exs | 11 +-- test/lightning/workflows/trigger_test.exs | 12 +-- test/lightning/workflows/workflow_test.exs | 6 +- test/lightning/workflows_test.exs | 4 +- .../channels/ai_assistant_channel_test.exs | 3 +- .../channels/workflow_channel_test.exs | 6 +- .../adaptor_icon_controller_test.exs | 22 ++--- .../live/components/common_test.exs | 10 +- .../live/credential_live_test.exs | 2 +- test/lightning_web/live/project_live_test.exs | 2 +- .../live/sandbox_live/index_test.exs | 3 - test/support/adaptor_test_helpers.ex | 4 +- tooling/adaptor_cache/README.md | 10 +- 95 files changed, 580 insertions(+), 729 deletions(-) diff --git a/.claude/guidelines/testable-supervision-trees.md b/.claude/guidelines/testable-supervision-trees.md index d6a60beee25..3e06291c3e9 100644 --- a/.claude/guidelines/testable-supervision-trees.md +++ b/.claude/guidelines/testable-supervision-trees.md @@ -158,13 +158,14 @@ boundary that was already open two lines away and already carrying its siblings across. `Lightning.Adaptors.Supervisor` shows both sides. `strategy` and `source` go -into `:persistent_term` keyed per instance (`lib/lightning/adaptors/supervisor.ex:42-45`), -which is the acceptable shape for the stateless `Store` functions: a caller -holding only the instance name has nowhere else to read boot-fixed config from. -It is the tell for `Scheduler`, whose child spec already carries `cache`, -`tasks` and `source_topic` (`supervisor.ex:58-65`) but not `strategy`, so the -process re-reads it from the global store on every refresh (`scheduler.ex:259`, -`:271`, `:313`). New children take config from the child spec, as `lock_key` does. +into `:persistent_term` keyed per instance (the `:persistent_term.put` in +`lib/lightning/adaptors/supervisor.ex`), which is the acceptable shape for the +stateless `Store` functions: a caller holding only the instance name has nowhere +else to read boot-fixed config from. It is the tell for `Scheduler`, whose child +spec already carries `cache`, `tasks` and `source_topic` but not `strategy`, so +the process re-reads it from the global store on every refresh (each +`AdaptorsSupervisor.strategy(state.sup)` call in `scheduler.ex`). New children +take config from the child spec, as `lock_key` does. ### Scoping mocks without going global diff --git a/ADAPTORS.md b/ADAPTORS.md index 087dc7370bd..d3c82138ed3 100644 --- a/ADAPTORS.md +++ b/ADAPTORS.md @@ -25,16 +25,15 @@ To layer a private checkout over the public one, comma-separate multiple roots: ADAPTORS_LOCAL_REPO=/path/to/private-adaptors,/path/to/adaptors ``` -A package in multiple roots comes from the first; Lightning logs each shadowed -package on every scan, not just at boot. +A package in multiple roots comes from the first. Lightning logs each shadowed +package on every scan. > #### Note {: .info} > -> Lightning still accepts the old names `LOCAL_ADAPTORS=true` and -> `OPENFN_ADAPTORS_REPO`, warning at boot only when it falls back to them: -> `LOCAL_ADAPTORS=true` when `ADAPTORS_STRATEGY` is unset, -> `OPENFN_ADAPTORS_REPO` when the strategy is local and `ADAPTORS_LOCAL_REPO` is -> unset. +> The deprecated `LOCAL_ADAPTORS=true` and `OPENFN_ADAPTORS_REPO` still work. +> Lightning warns at boot only when it falls back to them: `LOCAL_ADAPTORS=true` +> when `ADAPTORS_STRATEGY` is unset, `OPENFN_ADAPTORS_REPO` when the strategy is +> local and `ADAPTORS_LOCAL_REPO` is unset. ## Running without internet access @@ -132,9 +131,8 @@ the catalogue before it can store it, so it waits for the first load, up to 90 seconds, and rejects the save with "adaptor catalogue is not ready yet" if nothing has arrived by then. -So an instance that cannot reach npm at all comes up, serves every page and -retries in the background, but cannot save a workflow until the catalogue has -loaded once. Import a snapshot to give it one. +An instance that cannot reach npm serves every page but cannot save a workflow +until the catalogue has loaded once. Import a snapshot to give it one. ## Troubleshooting @@ -142,7 +140,7 @@ loaded once. Import a snapshot to give it one. then force a refresh with `--name`. - New version not showing: the hourly refresh hasn't run. Force one, or wait. - Icons missing after import: they never reached `ADAPTORS_ICONS_PATH` on this - instance, or the dump predates icon metadata. Redo the dump and copy the + instance. Unpack the icons archive there, or redo the dump and copy the directory. - Local package ignored: an earlier `ADAPTORS_LOCAL_REPO` root has a package of the same name; the log names each shadowed package. @@ -152,7 +150,8 @@ loaded once. Import a snapshot to give it one. 503: the first load has not finished. See [While the catalogue is still loading](#while-the-catalogue-is-still-loading). - Workflow save rejected with "adaptor catalogue is not ready yet": the same - thing, ninety seconds in. If the instance cannot reach npm, import a snapshot; - see [Running without internet access](#running-without-internet-access). + thing, after the save has waited out the first load. If the instance cannot + reach npm, import a snapshot; see + [Running without internet access](#running-without-internet-access). - Refresh exiting `2` with `:empty_listing`: the registry answered but served no `@openfn` packages. Check `ADAPTORS_NPM_REGISTRY_URL`. diff --git a/assets/js/collaborative-editor/components/inspector/TemplatePublishPanel.tsx b/assets/js/collaborative-editor/components/inspector/TemplatePublishPanel.tsx index e323796ca5d..d752a07f370 100644 --- a/assets/js/collaborative-editor/components/inspector/TemplatePublishPanel.tsx +++ b/assets/js/collaborative-editor/components/inspector/TemplatePublishPanel.tsx @@ -21,8 +21,7 @@ import { convertWorkflowStateToSpec } from '#/yaml/util'; logger.ns('TemplatePublishPanel').seal(); -// Validation schema matching backend constraints, meaning -// Lightning.Workflows.WorkflowTemplate.changeset/2. +// Mirrors Lightning.Workflows.WorkflowTemplate.changeset/2. // // name is counted in codepoints (the unit the column is measured in) and // description in graphemes (the unit the server's product cap uses), because diff --git a/assets/js/collaborative-editor/stores/createAdaptorStore.ts b/assets/js/collaborative-editor/stores/createAdaptorStore.ts index d379b398d77..3e7c6501114 100644 --- a/assets/js/collaborative-editor/stores/createAdaptorStore.ts +++ b/assets/js/collaborative-editor/stores/createAdaptorStore.ts @@ -186,9 +186,8 @@ export const createAdaptorStore = (): AdaptorStore => { const existing = state.adaptors; const existingByName = new Map(existing.map(a => [a.name, a])); - // Merge by name to preserve referential identity of unchanged adaptors so - // `withSelector` consumers don't re-render on no-op `adaptors_updated` - // pushes. + // Reuse the previous object where nothing changed, so `withSelector` + // consumers do not re-render on a no-op push. const merged: Adaptor[] = incoming.map(next => { const prev = existingByName.get(next.name); return prev && adaptorsEqual(prev, next) ? prev : next; @@ -273,9 +272,9 @@ export const createAdaptorStore = (): AdaptorStore => { const connectChannel = (provider: PhoenixChannelProvider) => { // The push only signals that named adaptors changed; it carries no // adaptor data. Always re-fetch the catalogue over HTTP rather than - // branching on which names changed -- a brand-new adaptor needs the - // fetch regardless, and 304 caching makes re-fetching a known one just - // as cheap. + // branching on which names changed. A brand-new adaptor needs the fetch + // regardless, and 304 caching makes re-fetching a known one just as + // cheap. const adaptorsUpdatedHandler = () => { logger.debug('Received adaptors_updated signal, refreshing catalogue'); void requestAdaptors(); diff --git a/assets/js/yaml/util.ts b/assets/js/yaml/util.ts index adb79124e52..6a81544ff26 100644 --- a/assets/js/yaml/util.ts +++ b/assets/js/yaml/util.ts @@ -48,8 +48,8 @@ const roundPosition = (pos: Position): Position => { // An edge key is a label. Nothing parses it, and the edge body carries its own // identity in source_job, source_trigger and target_job. That matters because -// the key joins two job keys with `->` and a job may legally hold a `>` since -// #4577: jobs named `a` and `b->c` produce the same key as `a->b` and `c`. +// the key joins two job keys with `->` and a job name may legally hold a `>`. +// Jobs named `a` and `b->c` produce the same key as `a->b` and `c`. // // Mirrors `disambiguate_edge_keys/1` in lib/lightning/export_utils.ex. const disambiguateEdgeKeys = ( @@ -383,10 +383,9 @@ export const parseWorkflowYAML = (yamlString: string): WorkflowSpec => { } } - // Validate job names. A Set rather than an object: a job named - // `constructor` or `toString` used to hit an inherited property and raise - // a duplicate error for a name that appeared once. Compared hyphenated, - // which is what the export side compares. + // A Set rather than an object. On a plain object a job named `constructor` + // or `toString` hits an inherited property and reads as a duplicate. + // Compared hyphenated, which is what the export side compares. const seenKeys = new Set(); Object.entries(parsedYAML['jobs']).forEach( ([key, specJob]: [string, any]) => { diff --git a/assets/test/collaborative-editor/__helpers__/sessionStoreHelpers.ts b/assets/test/collaborative-editor/__helpers__/sessionStoreHelpers.ts index eb414f594f6..abd65b53311 100644 --- a/assets/test/collaborative-editor/__helpers__/sessionStoreHelpers.ts +++ b/assets/test/collaborative-editor/__helpers__/sessionStoreHelpers.ts @@ -34,8 +34,8 @@ export { waitForAsync }; * A session store that tears itself down when the test ends. * * PhoenixChannelProvider registers a `process.on('exit')` handler that only - * `destroy()` removes, so a store left initialised leaks one listener — plus a - * Y.Doc, awareness and channel — per test. + * `destroy()` removes, so a store left initialised leaks one listener per test, + * along with its Y.Doc, awareness and channel. */ export function createTestSessionStore(): SessionStore { const store = createSessionStore(); diff --git a/assets/test/collaborative-editor/components/ChatInput.test.tsx b/assets/test/collaborative-editor/components/ChatInput.test.tsx index 7c6ef64935c..68f1b571946 100644 --- a/assets/test/collaborative-editor/components/ChatInput.test.tsx +++ b/assets/test/collaborative-editor/components/ChatInput.test.tsx @@ -276,8 +276,8 @@ describe('ChatInput', () => { describe('Message Length', () => { const type = async (text: string) => { const textarea = screen.getByPlaceholderText('Ask me anything...'); - // fireEvent, not userEvent: typing ten thousand characters one keystroke - // at a time takes minutes. + // fireEvent rather than userEvent, since typing ten thousand characters + // one keystroke at a time takes minutes. fireEvent.change(textarea, { target: { value: text } }); }; @@ -293,11 +293,10 @@ describe('ChatInput', () => { await type('x'.repeat(9600)); // The comma grouping comes from the locale vitest.config.ts pins for the - // run, not from the component: ChatInput formats with the viewer's own - // locale, so a real en-ZA or de-DE user sees "9 600" / "9.600". If this - // assertion fails on your machine, the locale pin is not in effect — - // don't pass a fixed locale to `toLocaleString()` in the component to - // make it pass. + // run, not from the component. ChatInput formats with the viewer's own + // locale, so a real en-ZA or de-DE user sees "9 600" or "9.600". If this + // assertion fails on your machine the locale pin is not in effect. Do not + // fix it by passing a fixed locale to `toLocaleString()` in the component. expect(screen.getByTestId('chat-input-length')).toHaveTextContent( '9,600 / 10,000' ); diff --git a/assets/test/collaborative-editor/components/inspector/CodeViewPanel.test.tsx b/assets/test/collaborative-editor/components/inspector/CodeViewPanel.test.tsx index 2aa6be4e864..2ca9bec42cb 100644 --- a/assets/test/collaborative-editor/components/inspector/CodeViewPanel.test.tsx +++ b/assets/test/collaborative-editor/components/inspector/CodeViewPanel.test.tsx @@ -14,8 +14,8 @@ * - Click handler for opening publish panel * - Button styling based on enabled/disabled state * - * Note: Download and copy functionality require manual testing due to jsdom - * limitations with DOM manipulation and clipboard API. + * Note: Copy functionality requires manual testing because jsdom has no + * clipboard API. */ import { render, screen } from '@testing-library/react'; @@ -276,8 +276,8 @@ describe('CodeViewPanel', () => { }); test('falls back to a usable name when nothing survives sanitising', async () => { - // A CJK or Arabic name used to sanitise down to nothing and the browser - // was handed a file called ".yaml". + // A CJK or Arabic name sanitises down to nothing, which would hand the + // browser a file called ".yaml". expect(await downloadNameFor('患者確認')).toBe('workflow.yaml'); expect(await downloadNameFor('تسجيل المريض')).toBe('workflow.yaml'); expect(await downloadNameFor('🎉')).toBe('workflow.yaml'); diff --git a/assets/test/collaborative-editor/createAdaptorStore.test.ts b/assets/test/collaborative-editor/createAdaptorStore.test.ts index 5efb968c030..db136e1c97c 100644 --- a/assets/test/collaborative-editor/createAdaptorStore.test.ts +++ b/assets/test/collaborative-editor/createAdaptorStore.test.ts @@ -1,9 +1,5 @@ /** * Tests for createAdaptorStore - * - * Covers the core store interface (subscribe/getSnapshot), state management - * commands, HTTP-backed `requestAdaptors`, Phoenix channel `adaptors_updated` - * live-update handling, and query helpers. */ import { describe, test, expect, vi, beforeEach } from 'vitest'; @@ -90,11 +86,11 @@ describe('createAdaptorStore', () => { unsubscribe2(); store.clearError(); expect(count1).toBe(2); - expect(count2).toBe(1); // unsubscribed, no longer notified + expect(count2).toBe(1); unsubscribe1(); store.setLoading(false); - expect(count1).toBe(2); // unsubscribed, no longer notified + expect(count1).toBe(2); }); test('withSelector returns a referentially stable value until its slice changes', () => { @@ -107,7 +103,7 @@ describe('createAdaptorStore', () => { store.setLoading(true); - expect(selectAdaptors()).toBe(adaptorsBefore); // unrelated slice unchanged + expect(selectAdaptors()).toBe(adaptorsBefore); expect(selectIsLoading()).not.toBe(loadingBefore); }); diff --git a/assets/test/collaborative-editor/hooks/useAIWorkflowUndo.test.ts b/assets/test/collaborative-editor/hooks/useAIWorkflowUndo.test.ts index 48c92102df2..94b722b639e 100644 --- a/assets/test/collaborative-editor/hooks/useAIWorkflowUndo.test.ts +++ b/assets/test/collaborative-editor/hooks/useAIWorkflowUndo.test.ts @@ -183,7 +183,7 @@ describe('useAIWorkflowUndo', () => { it('keeps the direction while the confirmation fades out', () => { // The dialog leaves over 200ms, so it is still on screen after a confirm. - // Reading the direction off the pending restore flipped its copy to the + // Reading the direction off the pending restore would flip its copy to the // other one on the way out. const { result } = setup({ hasChanged: true }); diff --git a/assets/test/collaborative-editor/useAdaptors.test.tsx b/assets/test/collaborative-editor/useAdaptors.test.tsx index 71c08d4ff72..b74a398589e 100644 --- a/assets/test/collaborative-editor/useAdaptors.test.tsx +++ b/assets/test/collaborative-editor/useAdaptors.test.tsx @@ -419,7 +419,6 @@ describe('useAdaptors hooks', () => { '@openfn/language-http', ]); - // adaptorsInUse entries are the same object references as the catalogue entries, not copies. const catalogue = result.current.allAdaptors; for (const a of result.current.adaptorsInUse) { const fromCatalogue = catalogue.find(c => c.name === a.name); diff --git a/assets/test/collaborative-editor/utils/workflowDiff.test.ts b/assets/test/collaborative-editor/utils/workflowDiff.test.ts index 37f4a90d8a2..dbe360b6bb8 100644 --- a/assets/test/collaborative-editor/utils/workflowDiff.test.ts +++ b/assets/test/collaborative-editor/utils/workflowDiff.test.ts @@ -1091,7 +1091,7 @@ describe('deriveSnapshotChanges', () => { it('pairs two id-less webhooks in document order, not in reverse', () => { // Both sides are identical apart from a job body. Pairing the leftovers - // from the end crossed the two triggers and invented a path move on each. + // from the end would cross the two triggers and invent a path move on each. const twoHooks = (body: string) => `id: wf-1 name: Test Workflow jobs: @@ -1122,7 +1122,7 @@ edges: {} }); it('keeps two webhooks apart in the cache salt', () => { - // buildYaml keys triggers by type, so this one is written out: the spec + // buildYaml keys triggers by type, so this one is written out. The spec // allows any key, and two webhooks are what make a type-keyed salt // ambiguous. Both chains stream the same two snapshots on purpose. const twoHooks = (a: string, b: string, body: string) => `id: wf-1 diff --git a/assets/test/utils/nameValidation.test.ts b/assets/test/utils/nameValidation.test.ts index 2d1aeed4b7e..452cb50231e 100644 --- a/assets/test/utils/nameValidation.test.ts +++ b/assets/test/utils/nameValidation.test.ts @@ -1,5 +1,5 @@ /** - * The client-side copy of the job/workflow name rule (#4577). + * The client-side copy of the job/workflow name rule. * * These tests exist to keep the client and `Lightning.Validators.validate_name/3` * saying the same thing. The Elixir counterparts live in @@ -100,7 +100,7 @@ describe('normalizeName', () => { expect('abc\uFEFF'.trim()).toBe('abc'); expect(normalizeName('abc\uFEFF')).toBe('abc\uFEFF'); - // A sample of the rest of the 25, to catch the set being edited down. + // A sample of the rest of the White_Space set, to catch it being edited down. expect(normalizeName('\u3000\u00a0\u2028 abc \u205f')).toBe('abc'); // Not White_Space, so not trimmed by either side. @@ -116,7 +116,6 @@ describe('graphemeLength', () => { expect(family.length).toBe(11); expect(graphemeLength(family)).toBe(1); - // Same for a flag. expect(graphemeLength('🇺🇸')).toBe(1); }); @@ -127,7 +126,7 @@ describe('graphemeLength', () => { // consonant together where Elixir's tables split. Malayalam is the // worst case: 'ന്ദ്ര' is 1 here and 3 to Elixir. // 2. Break-after-ZWJ, another rule revision Elixir's tables predate. - // 3. Characters added in Unicode 16, which Elixir has not caught up to. + // 3. Characters added in Unicode versions newer than Elixir's tables. // // The client is the permissive side every time, which is the safe // direction. The codepoint guard below stops any of this reaching the @@ -185,7 +184,7 @@ describe('JobSchema name', () => { const family = '\u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466}'; // 12 families plus 88 letters: 100 graphemes, which Ecto accepts, but 220 - // UTF-16 code units, which the old `.max(100)` would have refused. + // UTF-16 code units, which a code-unit cap of 100 would refuse. const atCap = 'a'.repeat(88) + family.repeat(12); expect(atCap.length).toBe(220); expect(graphemeLength(atCap)).toBe(NAME_MAX_LENGTH); @@ -231,7 +230,7 @@ describe('JobSchema name', () => { describe('isInvisibleOnly', () => { test('catches a run of joiners, not just a single one', () => { // Grapheme clustering fuses a ZWJ-led run into one cluster, so a - // per-grapheme check caught one joiner and missed two. + // per-grapheme check would count one joiner and miss the rest. for (const name of [ '\u{200D}', '\u{200D}\u{200D}', @@ -275,7 +274,7 @@ describe('createWorkflowSchema (the validator the settings form uses)', () => { test('accepts a name long in UTF-16 units but short in codepoints', () => { // 130 emoji: 130 codepoints, which the column holds, but 260 UTF-16 units, - // which the old `.max(255)` counted and refused. + // which a code-unit cap of 255 would refuse. const name = '\u{1F600}'.repeat(130); expect(name.length).toBe(260); @@ -320,7 +319,7 @@ describe('EdgeSchema condition_label (an inbound schema)', () => { test('accepts a label long in UTF-16 units but short in codepoints', () => { // 128 emoji: 128 codepoints, which the column holds, but 256 UTF-16 units, - // which the old `.max(255)` counted. + // which a code-unit cap of 255 would refuse. const label = '\u{1F600}'.repeat(128); expect(label.length).toBe(256); @@ -367,10 +366,9 @@ describe('isInvisibleOnly agrees with the server', () => { }); test('the client is no stricter than the server', () => { - // These used to differ. V8's tables were ahead of the PCRE build Elixir - // shipped, so the client knew four Arabic and Kaithi number signs as - // Format that the server did not. Erlang 28 brought PCRE2 and closed it. - // Pinned at zero so a future skew in either engine shows up as a diff. + // V8 and the PCRE build Erlang ships each carry their own Unicode tables, + // and they have disagreed on the Format category before. Pinned at zero so + // a skew in either engine shows up as a diff. const extra: string[] = []; for (let c = 0; c <= 0x10ffff; c++) { if (c >= 0xd800 && c <= 0xdfff) continue; @@ -413,7 +411,7 @@ describe('EdgeSchema condition_expression', () => { test('accepts an expression long in UTF-16 units but short in codepoints', () => { // 200 astral emoji: 200 codepoints the server stores fine, 400 UTF-16 - // units the old `.max(255)` counted. This runs on every keystroke. + // units a code-unit cap of 255 would refuse. const expression = '\u{1F600}'.repeat(200); expect(expression.length).toBe(400); @@ -422,8 +420,7 @@ describe('EdgeSchema condition_expression', () => { }); test('the base schema caps it too, not just the js_expression override', () => { - // The server validates the expression on every condition type. The base - // schema had no cap at all, so an over-wide expression sailed through. + // The server validates the expression on every condition type. expect(parseBase('a'.repeat(256)).success).toBe(false); }); @@ -446,9 +443,9 @@ describe('TemplatePublishSchema (the template form)', () => { ...values, }); - // Every case below is chosen so that it passes under the codepoint/grapheme - // rule and fails under the UTF-16 `.max()` it replaced, or the reverse. - // A case that behaves the same under both pins nothing. + // Every case below passes under the codepoint/grapheme rule and fails under + // a UTF-16 code-unit `.max()`, or the reverse. A case that behaves the same + // under both pins nothing. test('name: accepts 200 emoji, which .max(255) on UTF-16 units refused', () => { const name = '\u{1F600}'.repeat(200); @@ -466,8 +463,8 @@ describe('TemplatePublishSchema (the template form)', () => { }); test('name: refuses 256 emoji, which is 512 units and 256 codepoints', () => { - // Over the cap on both counts, so it pins that widening the rule did not - // remove the cap for astral input. + // Over the cap on both counts, so it pins that the cap still applies to + // astral input. expect(parse({ name: '\u{1F600}'.repeat(256) }).success).toBe(false); }); diff --git a/assets/test/yaml/blockScalars.test.ts b/assets/test/yaml/blockScalars.test.ts index bdab8296fa4..b62e9bad1e6 100644 --- a/assets/test/yaml/blockScalars.test.ts +++ b/assets/test/yaml/blockScalars.test.ts @@ -4,8 +4,8 @@ * `test/fixtures/block_scalars.json` is generated from * `Lightning.ExportUtils.Scalar.encode_block/2` and checked against yamerl on * the Elixir side. This file is the other half, and no Elixir test can run - * this parser: a `|2` block with a trailing whitespace-only line round-trips - * in yamerl and loses that line here (#4577). + * this parser. A `|2` block with a trailing whitespace-only line round-trips + * in yamerl and loses that line here. * * If this fails after a change to encode_block/2, regenerate the fixture and * check both parsers. diff --git a/assets/test/yaml/edgeKeys.test.ts b/assets/test/yaml/edgeKeys.test.ts index 28c63d24a17..27aff45e19c 100644 --- a/assets/test/yaml/edgeKeys.test.ts +++ b/assets/test/yaml/edgeKeys.test.ts @@ -6,8 +6,8 @@ * into the same git-synced repos, so if the two disagree the same workflow * exports two different ways depending on which side did it. * - * Reachable only since #4577: a job name could not contain `>` before, so - * `a` + `b->c` and `a->b` + `c` could not both key to `a->b->c`. + * A job name may contain `>`, so `a` + `b->c` and `a->b` + `c` both key to + * `a->b->c`. */ import fs from 'node:fs'; import path from 'node:path'; @@ -74,8 +74,8 @@ describe('edge keys match the server', () => { const spec = specFor(collision as EdgeCase); - // Two edges in, two out. The browser used to keep the last and the spec - // came out an edge short, with no error. + // Two edges in, two out. Keeping only the last would leave the spec an + // edge short, with no error. expect(Object.keys(spec.edges)).toHaveLength(2); // Both edges still name their own source and target, which is what the diff --git a/assets/test/yaml/util.test.ts b/assets/test/yaml/util.test.ts index 201de70b4bd..215332f1e84 100644 --- a/assets/test/yaml/util.test.ts +++ b/assets/test/yaml/util.test.ts @@ -339,9 +339,7 @@ describe('convertWorkflowStateToSpec', () => { // The server writes the canonical spec and the CLI reads it back, so a // key that differs by one hyphen is a different job. ExportUtils.hyphenate/1 - // replaces each single space, so two spaces give two hyphens. This used to - // collapse runs of whitespace and disagreed with the server on exactly - // that input. + // replaces each single space, so two spaces give two hyphens. test('one space, one hyphen, matching the server', () => { const spec = specFor(['a b', 'one two', 'trailing ']); @@ -351,14 +349,14 @@ describe('convertWorkflowStateToSpec', () => { }); // The server refuses this pair rather than exporting a spec with one job - // missing; the browser used to keep the last silently. + // missing. test('refuses two job names that hyphenate to the same key', () => { expect(() => specFor(['a b', 'a-b'])).toThrow(/Duplicate job name/); }); - // A job named `__proto__` assigned onto a plain object ran the prototype - // setter instead of adding a key, so the job silently vanished from the - // spec and the collision check never saw it. + // A job named `__proto__` assigned onto a plain object runs the prototype + // setter instead of adding a key, so the job would vanish from the spec + // and the collision check would never see it. test('keeps a job named __proto__ in the spec', () => { const spec = specFor(['__proto__', 'ordinary']); @@ -394,8 +392,8 @@ describe('convertWorkflowStateToSpec', () => { describe('convertWorkflowSpecToState prototype keys', () => { const specWith = (jobNames: string[]): WorkflowSpec => { - // Null-prototype here too, or the test helper hits the same setter the - // code under test used to and never builds the case it means to. + // Null-prototype here too, or the test helper hits the prototype setter + // itself and never builds the case it means to. const jobs = Object.create(null) as Record; jobNames.forEach(name => { jobs[name] = { @@ -414,8 +412,8 @@ describe('convertWorkflowSpecToState prototype keys', () => { }; test('keeps a job keyed __proto__ on the way in', () => { - // Assigned onto a plain object this ran the prototype setter and the job - // never landed, so the state came back one job short. + // Assigned onto a plain object this runs the prototype setter and the job + // never lands, so the state comes back one job short. const state = convertWorkflowSpecToState( specWith(['__proto__', 'a', 'b', 'c', 'd', 'e']) ); @@ -432,8 +430,8 @@ describe('convertWorkflowSpecToState prototype keys', () => { }); test('an edge naming a job that is not there still fails', () => { - // `toString` used to resolve through the prototype, so JobNotFoundError - // never fired and the edge pointed at nothing. + // On a plain object `toString` resolves through the prototype, so + // JobNotFoundError would never fire and the edge would point at nothing. const spec = specWith(['real']) as unknown as { edges: Record; }; @@ -468,9 +466,9 @@ describe('parseWorkflowYAML duplicate detection', () => { 'edges: {}', ].join('\n'); - // The export side compares hyphenated keys. Comparing raw names here let a - // spec holding both import cleanly and then throw on the way back out, - // leaving a workflow that could not be exported. + // The export side compares hyphenated keys. Comparing raw names here would + // let a spec holding both import cleanly and then throw on the way back out, + // leaving a workflow that cannot be exported. test('refuses two names that hyphenate to the same key', () => { expect(() => parseWorkflowYAML(yamlWith(['a b', 'a-b']))).toThrow( /Duplicate job name/ diff --git a/bin/adaptor_cache b/bin/adaptor_cache index c6536f781a8..bf06012a906 100755 --- a/bin/adaptor_cache +++ b/bin/adaptor_cache @@ -10,8 +10,9 @@ # tooling/adaptor_cache/README.md. # # A forward proxy would not work: Lightning talks to these upstreams through -# Req/Finch, and neither honours HTTP_PROXY/HTTPS_PROXY. Each upstream does -# have a configurable base URL, which is what this reverse proxy plugs into. +# Tesla on the Finch adapter, which does not honour HTTP_PROXY/HTTPS_PROXY. +# Each upstream does have a configurable base URL, which is what this reverse +# proxy plugs into. # ============================================================================= Mix.install([ diff --git a/lib/lightning/adaptor_service.ex b/lib/lightning/adaptor_service.ex index f672718fdd9..e15ab76b007 100644 --- a/lib/lightning/adaptor_service.ex +++ b/lib/lightning/adaptor_service.ex @@ -22,8 +22,8 @@ defmodule Lightning.AdaptorService do elsewhere such as delaying or rejecting processing until the adaptor becomes available. - Every install is gated on the adaptor catalogue (`Lightning.Adaptors`): - `install/2` refuses to run `npm install` for a package name the catalogue + `install/2` checks the package name against the adaptor catalogue + (`Lightning.Adaptors`) and refuses to run `npm install` for a name it doesn't recognise. ## Looking up adaptors diff --git a/lib/lightning/adaptors.ex b/lib/lightning/adaptors.ex index 4547e856461..d3cd97e733d 100644 --- a/lib/lightning/adaptors.ex +++ b/lib/lightning/adaptors.ex @@ -104,17 +104,17 @@ defmodule Lightning.Adaptors do do: Store.schema(sup, pkg) @doc """ - Resolves a possibly-legacy short adaptor name (e.g. `"http"`) to its full - npm package name (`"@openfn/language-http"`), if the full name resolves in - the catalogue. Returns `{:ok, name}` unchanged if it already resolves, or - if neither form does: a loaded catalogue that knows neither is a real - answer, and the name is left as given. + Resolves a legacy short adaptor name such as `"http"` to its full npm + package name, `"@openfn/language-http"`, when the full name is in the + catalogue. A name the catalogue already knows comes back unchanged. So + does a name it knows in neither form, because a loaded catalogue that + has never heard of it is a real answer. `"raw"` and `"oauth"` are sentinels, not adaptor names, and are returned unchanged without consulting the catalogue. - Answers from the catalogue as it stands, without waiting for a first - load: a catalogue that has never loaded is `{:error, :not_ready}`. + Does not wait for a first load. A catalogue that has never loaded is + `{:error, :not_ready}`. """ @spec resolve_name(atom(), String.t()) :: {:ok, String.t()} | {:error, :not_ready} @@ -159,8 +159,7 @@ defmodule Lightning.Adaptors do One read, so the stamp always describes the entries it comes with. - Returns `{:error, term()}` unchanged from `Store.catalogue/1` on a - backing-store failure; callers must handle it. + Returns `{:error, term()}` on a backing-store failure. """ @spec catalogue(atom()) :: {:ok, @@ -213,7 +212,7 @@ defmodule Lightning.Adaptors do end end - # The cached listing first, then the row itself: the listing can lag a + # The cached listing first, then the row itself. The listing can lag a # Scheduler write until the Invalidator drops it, and it leaves out the # excluded and deprecated names a job may still be using. defp resolve_meta(sup, name, cached) do diff --git a/lib/lightning/adaptors/catalogue.ex b/lib/lightning/adaptors/catalogue.ex index 4342ef5b48a..44202086c9b 100644 --- a/lib/lightning/adaptors/catalogue.ex +++ b/lib/lightning/adaptors/catalogue.ex @@ -94,9 +94,9 @@ defmodule Lightning.Adaptors.Catalogue do end @doc """ - Full structs for a source. Heavier than `list_package_metas/1`, which is - what picker traffic uses — this one is for callers that need the whole - row, like the Scheduler's diffing and the dump/seed tooling. + Full structs for a source, for callers that need the whole row, such as + the Scheduler's diffing and the dump/seed tooling. Picker traffic uses + the lighter `list_package_metas/1`. """ @spec list_adaptors(source()) :: [Adaptor.t()] def list_adaptors(source) do @@ -127,23 +127,18 @@ defmodule Lightning.Adaptors.Catalogue do end @doc """ - Idempotent, transactional, diff-aware upsert of one adaptor record - plus its version rows. The source is read from the record, whose keys - may be atoms or strings (as a decoded JSON snapshot gives them). - - Behaviour: - - * On every call, `checked_at` is advanced to "now". - * `updated_at` only advances when at least one non-`checked_at` - field of the adaptor row actually differs from the existing row. - * Version rows are replaced (delete + insert) inside the same - transaction. - * Every row is run through its schema changeset before write, so a - corrupt Strategy response cannot poison the DB. - - Raises if the underlying transaction fails (e.g. invalid input from - a misbehaving strategy) — the success type is the only contract the - Scheduler relies on. + Upserts one adaptor record plus its version rows in one transaction. + The source is read from the record, whose keys may be atoms or strings, + as a decoded JSON snapshot gives them. + + `checked_at` advances on every call. `updated_at` advances only when + some other field of the adaptor row differs from the existing row. + Version rows are deleted and reinserted. Every row goes through its + schema changeset first, so a corrupt strategy response cannot reach + the database. + + Raises if the transaction fails, for example on invalid input from a + misbehaving strategy. The Scheduler relies on the success type only. """ @spec upsert_adaptor(map()) :: {:ok, Adaptor.t()} def upsert_adaptor(record) when is_map(record) do @@ -202,7 +197,7 @@ defmodule Lightning.Adaptors.Catalogue do Advance `checked_at` for a known `(name, source)` row without loading it. No-op when no row matches. - Used by the Scheduler's "polled NPM, nothing changed" path — + The Scheduler uses this when a poll finds nothing changed. It is cheaper than a full upsert and never bumps `updated_at`. """ @spec touch_checked_at(String.t(), source()) :: :ok @@ -252,7 +247,9 @@ defmodule Lightning.Adaptors.Catalogue do @doc """ Maximum `checked_at` seen for `source`, or `nil` when the table is - empty for that source. Backs the Scheduler's smart-init timing. + empty for that source. The Scheduler reads it at boot to time its first + tick, and the Store reads it to tell whether the catalogue has ever + loaded. """ @spec max_checked_at(source()) :: DateTime.t() | nil def max_checked_at(source) do @@ -267,8 +264,8 @@ defmodule Lightning.Adaptors.Catalogue do Full catalogue projection for a source: every adaptor's `name`, `latest_version`, `repository`, icon fields, and full version list. - Excludes the packages listed in `@excluded_names`, any deprecated - adaptor, and — for an otherwise-listed adaptor — any deprecated version. + Excludes the packages listed in `@excluded_names` and any deprecated + adaptor. A listed adaptor's deprecated versions are left out too. """ @spec catalogue(source()) :: [catalogue_entry()] def catalogue(source) do @@ -311,9 +308,8 @@ defmodule Lightning.Adaptors.Catalogue do `MAX(adaptors.updated_at)` and `MAX(adaptor_versions.inserted_at)`, or `nil` when the source has no rows. - `version_row_count` is carried alongside the timestamp because a - removed version doesn't move either max — deleting rows only ever - lowers the count. + `version_row_count` rides alongside the timestamp because deleting a + version row moves neither max. It does lower the count. """ @spec catalogue_stamp(source()) :: {DateTime.t() | nil, non_neg_integer()} def catalogue_stamp(source) do @@ -422,16 +418,15 @@ defmodule Lightning.Adaptors.Catalogue do end # `Ecto.Changeset.cast/3` raises on a map mixing atom and string keys, so - # every map handed to a changeset here is flattened to string keys first — - # that is what a JSON snapshot gives us, and what atom-keyed callers - # convert cleanly into. + # every map handed to a changeset here is flattened to string keys first. + # A JSON snapshot arrives that way already. defp stringify_keys(map) do Map.new(map, fn {k, v} -> {to_string(k), v} end) end - # `source` is read outside the changeset (for the existing-row lookup), - # so it needs its own cast: `Ecto.Enum` fields accept a string via - # `Changeset.cast/3`, but not via `Repo.get_by/3`'s query parameters. + # `source` is read outside the changeset for the existing-row lookup, so + # it needs its own cast. `Ecto.Enum` fields accept a string through + # `Changeset.cast/3` but not through `Repo.get_by/3` query parameters. defp normalize_source(source) when is_atom(source), do: source defp normalize_source(source) when is_binary(source), diff --git a/lib/lightning/adaptors/channel_broadcaster.ex b/lib/lightning/adaptors/channel_broadcaster.ex index 09dbf14f995..8d5ec62d598 100644 --- a/lib/lightning/adaptors/channel_broadcaster.ex +++ b/lib/lightning/adaptors/channel_broadcaster.ex @@ -1,17 +1,16 @@ defmodule Lightning.Adaptors.ChannelBroadcaster do @moduledoc """ - Burst-coalesced fan-out of adaptor changes to connected sessions. + Fans adaptor changes out to connected sessions, coalescing bursts. - Subscribes to `:source_topic` (the cache-coherence topic shared with - `Lightning.Adaptors.Invalidator`) and republishes a single envelope of - changed names to `:client_topic` at most once per 250ms leading-edge + Subscribes to `:source_topic`, the same topic + `Lightning.Adaptors.Invalidator` listens on, and republishes one message + of changed names to `:client_topic` at most once per `debounce_ms/0` window. - Two-topic separation: the source topic is the cache-coherence audience; - the client topic is the display-freshness audience (`WorkflowChannel` - subscribers). This bridges them: the payload tells a session "these - adaptors changed, go refetch" — not what changed about them, so - `:flush` never has to touch the cache or render anything. + The two topics have different audiences. The source topic keeps node + caches coherent. The client topic tells `WorkflowChannel` subscribers + which adaptors changed so they refetch, and nothing about what changed, + so `:flush` never touches the cache or renders anything. """ use GenServer @@ -19,10 +18,8 @@ defmodule Lightning.Adaptors.ChannelBroadcaster do @debounce_ms 250 @doc """ - Leading-edge coalesce window in milliseconds. - - Exposed so integration tests can compute receive timeouts off the - authoritative value rather than hard-coding a duplicate. + Leading-edge coalesce window in milliseconds. Tests derive receive + timeouts from it. """ @spec debounce_ms() :: pos_integer() def debounce_ms, do: @debounce_ms @@ -31,9 +28,9 @@ defmodule Lightning.Adaptors.ChannelBroadcaster do Start the ChannelBroadcaster linked to the calling process. Required opts: - * `:name` — registered GenServer name. - * `:source_topic` — PubSub topic to subscribe to (cache-coherence). - * `:client_topic` — PubSub topic to broadcast the changed names to. + * `:name` - registered GenServer name. + * `:source_topic` - PubSub topic to subscribe to. + * `:client_topic` - PubSub topic to broadcast the changed names to. """ @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts) do @@ -64,7 +61,6 @@ defmodule Lightning.Adaptors.ChannelBroadcaster do {:noreply, %{state | timer: timer, names: MapSet.put(state.names, name)}} end - # Subsequent messages within the debounce window: accumulate, don't flush. def handle_info({:changed, name, _source}, state) do {:noreply, %{state | names: MapSet.put(state.names, name)}} end diff --git a/lib/lightning/adaptors/dump.ex b/lib/lightning/adaptors/dump.ex index 8404181be66..a15b599acec 100644 --- a/lib/lightning/adaptors/dump.ex +++ b/lib/lightning/adaptors/dump.ex @@ -25,8 +25,8 @@ defmodule Lightning.Adaptors.Dump do * `:source` - `:npm` (default) or `:local` """ - # `path` is a mix-task argument (`mix lightning.adaptors.dump`) or a - # release-command argument — an operator's own filesystem, not a request. + # `path` is a mix-task or release-command argument on the operator's own + # filesystem, not a request. # sobelow_skip ["Traversal.FileModule"] @spec dump_to_file(Path.t(), keyword()) :: {:ok, non_neg_integer()} def dump_to_file(path, opts \\ []) do diff --git a/lib/lightning/adaptors/icon_cache.ex b/lib/lightning/adaptors/icon_cache.ex index 707ffdaf382..5d10d51e43e 100644 --- a/lib/lightning/adaptors/icon_cache.ex +++ b/lib/lightning/adaptors/icon_cache.ex @@ -1,34 +1,22 @@ defmodule Lightning.Adaptors.IconCache do @moduledoc """ - Pure filesystem helper owning the on-disk adaptor icon cache. + Stateless filesystem functions over the on-disk adaptor icon cache, + rooted at `Lightning.Adaptors.Config.icon_path/0`. - Not a GenServer. Three stateless functions over - `Lightning.Adaptors.Config.icon_path/0`, which returns `ADAPTORS_ICONS_PATH` - when set and otherwise resolves the `{:tmp, suffix}` default at call time. - - Disk layout is **source-partitioned** and **content-addressed**: + Disk layout: ///.. where `sha8` is the first 8 lowercase hex characters of the icon - sha256 on the adaptor row. Source partitioning means flipping - `ADAPTORS_STRATEGY` between restarts cannot accidentally serve `:npm` - bytes from a row that's now resolved via `:local` (or vice versa). - Putting the sha in the filename means `cached?/5` is a plain - existence check, and a node holding an earlier icon simply misses and - refetches instead of serving it forever; `write!/6` removes the - superseded siblings for that shape. - - Concurrent first-request fetchers are coalesced upstream by Cachex's - courier on `{:icon_bytes, source, name, shape}` inside - `Lightning.Adaptors.Store.icon/3`, and all in-flight peers receive the - courier's result for free. Bytes that verify are left uncommitted, - since this directory is their cache, but bytes that disagree with the - row's sha or extension are committed as an error, so the disagreement - is not re-fetched from the source on every request until the row moves. - The temp-then-rename in `write!/6` is the belt-and-braces guarantee - for the file-write step itself: readers never observe a half-written - file. + sha256 on the adaptor row. Partitioning by source means switching + `ADAPTORS_STRATEGY` between restarts cannot serve `:npm` bytes for a + row now resolved through `:local`, or the reverse. Putting the sha in + the filename makes `cached?/5` a plain existence check, and a node + holding an earlier icon misses and refetches instead of serving it + forever. `write!/6` removes the superseded siblings for that shape. + + Concurrent first-request fetches are coalesced by + `Lightning.Adaptors.Store.icon/3`, not here. """ alias Lightning.Adaptors.Config @@ -46,11 +34,11 @@ defmodule Lightning.Adaptors.IconCache do `@openfn/language-foo`); `Path.join/1` preserves the slash so the scope becomes a real subdirectory. - Raises `ArgumentError` on a name `PackageName` would reject. Icons are - written straight from a strategy's response, before the row reaches - `CatalogueAdaptor.changeset/2` and its name validation, so this is the - only thing standing between a hostile registry entry and a write - outside the cache root. + Raises `ArgumentError` on a name `PackageName` would reject. The + Scheduler writes icons straight from a strategy's response, before the + row reaches `Lightning.Adaptors.Catalogue.Adaptor.changeset/2` and its + name validation, so this is the only thing standing between a hostile + registry entry and a write outside the cache root. """ @spec path(source(), name(), shape(), ext(), binary()) :: Path.t() def path(source, name, shape, ext, sha256) do diff --git a/lib/lightning/adaptors/icon_field.ex b/lib/lightning/adaptors/icon_field.ex index 7159e89e9da..2d721203861 100644 --- a/lib/lightning/adaptors/icon_field.ex +++ b/lib/lightning/adaptors/icon_field.ex @@ -2,11 +2,9 @@ defmodule Lightning.Adaptors.IconField do @moduledoc """ Schema column names for an icon shape. - Every module that reaches for an icon column — the catalogue schema, - the store's projections, the scheduler's record merges, the controller - and the URL builder — goes through here, so `:square` and - `:rectangle` mean the same columns everywhere and no column name is - built by interpolating an atom. + Every module that reaches for an icon column goes through here, so + `:square` and `:rectangle` mean the same columns everywhere and no + column name is built by interpolating an atom. """ @type shape :: :square | :rectangle diff --git a/lib/lightning/adaptors/invalidator.ex b/lib/lightning/adaptors/invalidator.ex index 5342c82f868..6993ee774cf 100644 --- a/lib/lightning/adaptors/invalidator.ex +++ b/lib/lightning/adaptors/invalidator.ex @@ -3,16 +3,14 @@ defmodule Lightning.Adaptors.Invalidator do Subscribes to cluster adaptor-change broadcasts and evicts matching local Cachex entries, keeping each node coherent with Postgres. - Subscribes to `opts[:source_topic]` on `Lightning.PubSub` at init. - On `{:changed, name, source}`, deletes the six cache keys written by - `Lightning.Adaptors.Store`: the four keyed by name (`:schema`, - `:icon_meta` and the two `:icon_bytes` shapes) plus the two - source-wide ones (`:packages`, `:catalogue`), which any change - invalidates. Dropping `:icon_bytes` is what lets a committed icon error - clear: the row moving is the only thing that can resolve it, and the row - moving always broadcasts. No source filtering on the - hot path — a broadcast for a source that isn't active on this node is a - no-op because those keys simply don't exist in Cachex. + Subscribes to `opts[:source_topic]` on `Lightning.PubSub` at init. On + `{:changed, name, source}` it deletes every cache key + `Lightning.Adaptors.Store` writes for that name, plus the source-wide + `:packages` and `:catalogue` keys, which any change invalidates. + Dropping `:icon_bytes` is what lets a committed icon error clear. Only + the row moving can resolve it, and the row moving always broadcasts. + There is no source filtering. A broadcast for a source not active on + this node deletes keys that do not exist, which is a no-op. """ use GenServer @@ -21,9 +19,9 @@ defmodule Lightning.Adaptors.Invalidator do Start the Invalidator linked to the calling process. Required opts: - * `:name` — registered process name. - * `:source_topic` — `Phoenix.PubSub` topic to subscribe to. - * `:cache` — Cachex table atom (from `Lightning.Adaptors.Supervisor.cache_name/1`). + * `:name` - registered process name. + * `:source_topic` - `Phoenix.PubSub` topic to subscribe to. + * `:cache` - Cachex table atom, from `Lightning.Adaptors.Supervisor.cache_name/1`. """ @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts) do diff --git a/lib/lightning/adaptors/local.ex b/lib/lightning/adaptors/local.ex index dfb35d26d3f..a36eb066bfd 100644 --- a/lib/lightning/adaptors/local.ex +++ b/lib/lightning/adaptors/local.ex @@ -9,8 +9,7 @@ defmodule Lightning.Adaptors.Local do `Lightning.Adaptors.Config.strategy_opts(__MODULE__)[:paths]`, an ordered list of root directories. - Each callback walks the filesystem afresh — caching is the Store's - responsibility. The module is stateless; no GenServer, no ETS. + Each callback walks the filesystem afresh. Caching is the Store's job. ## Layout @@ -32,8 +31,7 @@ defmodule Lightning.Adaptors.Local do readable and really is empty, which is a different thing from an npm org listing that comes back with nothing in it. - `source: :local` is **not** set here — the Store stamps it before - upsert. No network calls anywhere in this module. + `source: :local` is not set here. The Scheduler stamps it before upsert. """ @behaviour Lightning.Adaptors.Strategy diff --git a/lib/lightning/adaptors/node_monitor.ex b/lib/lightning/adaptors/node_monitor.ex index 1d7f935aed1..3785f56a60b 100644 --- a/lib/lightning/adaptors/node_monitor.ex +++ b/lib/lightning/adaptors/node_monitor.ex @@ -21,8 +21,8 @@ defmodule Lightning.Adaptors.NodeMonitor do Start a NodeMonitor for the given supervisor instance. Required opts: - * `:name` — registered GenServer name. - * `:sup` — supervisor instance name, forwarded to `Store.warm_from_repo/1`. + * `:name` - registered GenServer name. + * `:sup` - supervisor instance name, forwarded to `Store.warm_from_repo/1`. """ @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts) do @@ -43,8 +43,7 @@ defmodule Lightning.Adaptors.NodeMonitor do {:noreply, state} end - # Deliberate no-op: nodedown does not trigger a re-warm. 302-on-stale-sha - # handles already-issued icon URLs; other reads stay stale until nodeup. + # Deliberate no-op; see the moduledoc. def handle_info({:nodedown, _node, _info}, state) do {:noreply, state} end diff --git a/lib/lightning/adaptors/npm.ex b/lib/lightning/adaptors/npm.ex index fb0eed0356d..ef4af5b83d9 100644 --- a/lib/lightning/adaptors/npm.ex +++ b/lib/lightning/adaptors/npm.ex @@ -3,57 +3,44 @@ defmodule Lightning.Adaptors.NPM do Production implementation of `Lightning.Adaptors.Strategy` that talks to the public NPM registry and the OpenFn adaptors monorepo on GitHub. - Implements the four `Lightning.Adaptors.Strategy` callbacks: - - * `c:Lightning.Adaptors.Strategy.list_adaptors/0` — merges the - `@openfn` org's authoritative package listing with the search API's - cheap version lookup, returning `name + latest_version` for every - `@openfn/language-*` package. See - `Lightning.Adaptors.NPM.Registry` for why this is two calls, not - one. A listing with no `@openfn/language-*` names is an error, not - an empty catalogue. - * `c:Lightning.Adaptors.Strategy.fetch_adaptor/1` — packument fetch + - per-version decode and latest-version schema retrieval via - jsDelivr. Icon fields are **not** stamped here; the Scheduler - joins them on after a bulk - `c:Lightning.Adaptors.Strategy.fetch_icons/1` pass. - * `c:Lightning.Adaptors.Strategy.fetch_icon/2` — single icon raw GET - against `raw.githubusercontent.com`, used by the Store's rare - lazy-miss fallback. - * `c:Lightning.Adaptors.Strategy.fetch_icons/1` — bulk fan-out over - the adaptor listing, one HTTP request per `(name, shape)`. Threads - `:prior_etags` from the caller down into the per-request - `If-None-Match` headers. + * `c:Lightning.Adaptors.Strategy.list_adaptors/0` merges the + `@openfn` org's package listing with the search API's version + lookup. See `Lightning.Adaptors.NPM.Registry` for why this is two + calls, not one. A listing with no `@openfn/language-*` names is an + error, not an empty catalogue. + * `c:Lightning.Adaptors.Strategy.fetch_adaptor/1` fetches the + packument and the latest version's schema from jsDelivr. Icon + fields are not stamped here. The Scheduler joins them on after a + bulk `c:Lightning.Adaptors.Strategy.fetch_icons/1` pass. + * `c:Lightning.Adaptors.Strategy.fetch_icon/2` is a single raw GET + against `raw.githubusercontent.com`, used when the Store misses an + icon on disk. + * `c:Lightning.Adaptors.Strategy.fetch_icons/1` fans out one GET per + `(name, shape)` and sends `:prior_etags` as `If-None-Match`. ## HTTP - This module is a thin orchestrator. The actual HTTP work is delegated - to three sub-modules, each of which owns its own Tesla client and - upstream base URL: - - * `Lightning.Adaptors.NPM.Registry` — npm registry search + packument. - * `Lightning.Adaptors.NPM.Schema` — jsDelivr `configuration-schema.json`. - * `Lightning.Adaptors.NPM.GitHub` — `raw.githubusercontent.com` - icon fetches (one GET per `(name, shape)`). + The HTTP work lives in three sub-modules, each with its own Tesla + client and base URL: `Lightning.Adaptors.NPM.Registry`, + `Lightning.Adaptors.NPM.Schema` and `Lightning.Adaptors.NPM.GitHub`. - Each sub-module issues at most a handful of single-shot Tesla requests - bounded by `http_timeout`. No retry, no backoff, no circuit-breaker — - transient failures (5xx, timeout, nxdomain) of the *primary* request - (`packument` for `fetch_adaptor/1`, the org package listing for - `list_adaptors/0` and `fetch_icons/1`) surface as `{:error, term()}` - unchanged, as does a failed schema fetch inside `fetch_adaptor/1` - (`{:error, {:schema_fetch_failed, reason}}`). Each icon fetch inside - `fetch_icons/1` is best-effort instead: a miss there degrades to an - absent icon shape rather than failing the batch. + Every request is single-shot and bounded by `http_timeout`. There is no + retry, backoff or circuit breaker. A transient failure (5xx, timeout, + nxdomain) of the primary request (the packument for `fetch_adaptor/1`, + the org listing for `list_adaptors/0` and `fetch_icons/1`) surfaces as + `{:error, term()}` unchanged, and a failed schema fetch inside + `fetch_adaptor/1` as + `{:error, {:schema_fetch_failed, reason}}`. Icon fetches inside + `fetch_icons/1` are best-effort instead: a miss degrades to an absent + icon shape rather than failing the batch. ## Configuration - Each sub-module reads `:registry_url`, `:jsdelivr_url`, `:github_url`, - `:github_ref`, and `:http_timeout` via - `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)` — all - three share this module's own config key rather than each having their - own — with defaults baked in so the module works even when no - Application env block is set. + The sub-modules read `:registry_url`, `:jsdelivr_url`, `:github_url`, + `:github_ref` and `:http_timeout` from + `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)`. All + three share this module's config key rather than having their own, and + each key has a default so the module works with no Application env set. """ @behaviour Lightning.Adaptors.Strategy diff --git a/lib/lightning/adaptors/npm/github.ex b/lib/lightning/adaptors/npm/github.ex index 013616335c6..243b29c9aa0 100644 --- a/lib/lightning/adaptors/npm/github.ex +++ b/lib/lightning/adaptors/npm/github.ex @@ -2,27 +2,25 @@ defmodule Lightning.Adaptors.NPM.GitHub do @moduledoc """ Raw `raw.githubusercontent.com` client for adaptor icons. - Icons aren't published inside npm tarballs — they live in the - `OpenFn/adaptors` monorepo. This module fetches them directly via the - raw GitHub content host, one icon per HTTP GET, no tarball walking. + Icons are not published inside npm tarballs. They live in the + `OpenFn/adaptors` monorepo, so this module fetches them from the raw + GitHub content host, one icon per GET. ## URL pattern /OpenFn/adaptors//packages//assets/. where `` strips the `@openfn/` scope and, when present, the - `language-` prefix too — `@openfn/language-common` becomes `common`, + `language-` prefix too, so `@openfn/language-common` becomes `common`, matching the monorepo's `packages/` directory names. Each `(name, - shape)` is probed `png` first then `svg` — matching the ext order used - by `Lightning.Adaptors.Local`. + shape)` is probed `png` first then `svg`, the same order + `Lightning.Adaptors.Local` uses. ## Configuration - Both `:github_url` (default `https://raw.githubusercontent.com`) and - `:github_ref` (default `main`) are read via - `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)`, - symmetric with the existing `:registry_url`, `:jsdelivr_url`, and - `:http_timeout` keys. + `:github_url` (default `https://raw.githubusercontent.com`) and + `:github_ref` (default `main`) come from + `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)`. """ alias Lightning.Adaptors.Config @@ -42,9 +40,8 @@ defmodule Lightning.Adaptors.NPM.GitHub do @doc """ Fetch a single icon for `(name, shape)`. - Tries `png` then `svg`. No conditional GET — this entry point is used - by the Store's lazy-miss fallback, where no prior etag is in scope. - Returns: + Tries `png` then `svg`. No conditional GET, since the Store calls this + when an icon is missing on disk and has no prior etag. Returns: * `{:ok, %{data: binary(), ext: String.t(), etag: String.t() | nil}}` on success. @@ -63,21 +60,13 @@ defmodule Lightning.Adaptors.NPM.GitHub do @doc """ Fetch icons for every `(name, shape)` pair across `names`. - Returns `{:ok, partial_map}` where each entry is keyed by the - package name and contains zero, one, or two shape keys. Absence - is **not** an error — packages with no upstream icon simply do not - appear (or appear with a missing shape). - - When `prior_etags` is supplied as `%{name => %{shape => etag}}`, the - corresponding `If-None-Match` header is sent per `(name, shape)`. A - 304 response is surfaced as a `:not_modified` sentinel in the - per-shape slot — distinct from "absent" which means upstream had no - such shape at all. - - Fans out via `Task.async_stream` with a bounded concurrency. Transport - failures for a single `(name, shape)` are dropped silently — they just - don't appear in the result map. This function always returns - `{:ok, partial_map}`; there is no error return. + Returns `{:ok, map}` keyed by package name, each holding zero, one or + two shape keys. A missing shape is not an error. It means upstream had + no such icon, or the fetch for that pair failed, which is only logged. + + `prior_etags` is `%{name => %{shape => etag}}`. Each etag is sent as + `If-None-Match` for its `(name, shape)`, and a 304 comes back as + `:not_modified` in that shape's slot. """ @spec fetch_all([String.t()], %{ optional(String.t()) => %{ diff --git a/lib/lightning/adaptors/npm/registry.ex b/lib/lightning/adaptors/npm/registry.ex index c355f0856b9..5b874b62927 100644 --- a/lib/lightning/adaptors/npm/registry.ex +++ b/lib/lightning/adaptors/npm/registry.ex @@ -3,27 +3,26 @@ defmodule Lightning.Adaptors.NPM.Registry do NPM registry HTTP client for `Lightning.Adaptors.NPM`. Talks to `registry.npmjs.org`. Responsible for the `list_adaptors/0` - scope listing, and the `packument` endpoint used by `fetch_adaptor/1` - and `fetch_icon/2`. + scope listing and the packument endpoint `fetch_adaptor/1` uses. `list_adaptors/0` deliberately merges two endpoints rather than calling one: * `/-/user/openfn/package` is the authoritative name list for the - `@openfn` org — the actual trust boundary, since it can't return a - name Lightning doesn't already trust. It has no version data. - * `/-/v1/search` is used only as a cheap version lookup for whichever - of those names it happens to cover. npm's relevance ranking demotes - or excludes deprecated packages from search results even on an - exact-name query, so search alone silently drops names — it is not - a safe source of *scope membership*, only of version data for names + `@openfn` org and the trust boundary, since it cannot return a + name Lightning does not already trust. It has no version data. + * `/-/v1/search` is only a cheap version lookup for whichever of those + names it happens to cover. npm's relevance ranking demotes or + excludes deprecated packages from search results even on an + exact-name query, so search alone silently drops names. It is not a + safe source of scope membership, only of version data for names already known to be in scope. Any authoritative name missing from the search results falls back to a - per-name `get_packument/1` + `latest_version/1` call, bounded to the - handful of names search doesn't cover. Do not "simplify" this back to a - single search call or a pagination bump — search's result count is - capped by npm's relevance ranking regardless of `size`/`from`, and + per-name `get_packument/1` and `latest_version/1` call, bounded to the + handful of names search does not cover. Do not "simplify" this back to + a single search call or a pagination bump. Search's result count is + capped by npm's relevance ranking regardless of `size` and `from`, and deprecated packages are excluded from ranking entirely, so no amount of paging recovers them. diff --git a/lib/lightning/adaptors/npm/schema.ex b/lib/lightning/adaptors/npm/schema.ex index 8a8d935fa9c..26c5dd185cb 100644 --- a/lib/lightning/adaptors/npm/schema.ex +++ b/lib/lightning/adaptors/npm/schema.ex @@ -3,11 +3,8 @@ defmodule Lightning.Adaptors.NPM.Schema do jsDelivr CDN client for adaptor configuration schemas. Fetches `/npm/@/configuration-schema.json` from - `cdn.jsdelivr.net`, checks it decodes, and returns `{:ok, - {schema_data, schema_sha256}}` with the body kept as the bytes - served. A genuine 404 (schema removed upstream) is `{:ok, {nil, - nil}}`; any other failure (timeout, other HTTP status, network error, - undecodable body) is `{:error, reason}`. + `cdn.jsdelivr.net`, checks it decodes, and keeps the body as the bytes + served. Base URL via `Lightning.Adaptors.Config.strategy_opts(Lightning.Adaptors.NPM)[:jsdelivr_url]`, default `https://cdn.jsdelivr.net`. @@ -21,11 +18,12 @@ defmodule Lightning.Adaptors.NPM.Schema do @doc """ Fetch the configuration schema for `name@version` from jsDelivr. - Returns `{:ok, {schema_data, schema_sha256}}` on success, `{:ok, - {nil, nil}}` on a genuine 404 (schema removed upstream), and - `{:error, reason}` on any other failure, since a transient failure - must not be mistaken for genuine absence by callers that persist the - result. + Returns `{:ok, {schema_data, schema_sha256}}` on success and + `{:ok, {nil, nil}}` on a 404. jsDelivr answers 404 both for a package + with no schema and for a version it has not mirrored yet, so the + Scheduler decides what a `nil` means. Any other failure (timeout, other status, undecodable body) is + `{:error, reason}`, so callers that persist the result never mistake a + transient failure for absence. """ @spec schema(String.t(), String.t()) :: {:ok, {String.t(), String.t()}} | {:ok, {nil, nil}} | {:error, term()} diff --git a/lib/lightning/adaptors/package_name.ex b/lib/lightning/adaptors/package_name.ex index afa5d32b43b..b3f74e1bd60 100644 --- a/lib/lightning/adaptors/package_name.ex +++ b/lib/lightning/adaptors/package_name.ex @@ -7,7 +7,7 @@ defmodule Lightning.Adaptors.PackageName do # # A segment may not begin with `.` or `_`, which is npm's own rule. That # keeps `.` and `..` out, so a name is always safe to use as a path - # segment — see `Lightning.Adaptors.IconCache`. + # segment. See `Lightning.Adaptors.IconCache`. @segment "[a-zA-Z0-9-][\\w.-]*" @strict_format ~r{\A(@?#{@segment}(?:/#{@segment})?)(?:@([\w.-]+))?\z} diff --git a/lib/lightning/adaptors/scheduler.ex b/lib/lightning/adaptors/scheduler.ex index b82635814d2..91c7110d899 100644 --- a/lib/lightning/adaptors/scheduler.ex +++ b/lib/lightning/adaptors/scheduler.ex @@ -10,13 +10,14 @@ defmodule Lightning.Adaptors.Scheduler do catalogue ticks at once. An interval of `0` disables the timer and leaves only on-demand refreshes. - A tick lists the source, fetches the adaptors whose `latest_version` - changed or whose stored row has no schema and whose version landed - within the last hour (a refetch that still finds no schema counts as - touched; the window covers jsDelivr's mirroring lag, after which a - missing schema is taken as really missing), fetches icons in parallel, - and upserts each changed adaptor with its icons. `refresh_package/2` refetches one - adaptor without icons. + A tick lists the source and fetches every adaptor whose `latest_version` + changed. It also refetches an adaptor whose stored row has no schema, for + the grace period set by `@schema_grace_ms` after the row's `updated_at`. + jsDelivr mirrors a new version with some lag, so a schema missing inside + that window may still arrive. After it, a missing schema is taken as + really missing. A refetch that still finds no schema counts as touched. + Icons are fetched in parallel and each changed adaptor is upserted with + its icons. `refresh_package/2` refetches one adaptor without icons. """ use GenServer @@ -38,10 +39,10 @@ defmodule Lightning.Adaptors.Scheduler do `:cache`, `:tasks`, `:source_topic`, `:refresh_interval` (tick interval in milliseconds; `0` disables the timer) and `:warn_when_empty` (whether booting on an empty catalogue with the timer disabled logs a warning). - Optional: `:checked_at` (1-arity fn, - default `&Catalogue.max_checked_at/1`) reads the source's last-checked - timestamp; called once at boot to schedule the delay before the scheduler's - initial tick. + `:checked_at` is optional. It is a 1-arity function, defaulting to + `&Catalogue.max_checked_at/1`, that reads the source's last-checked + timestamp. It is called once at boot to work out the delay before the + first tick. """ @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts) do @@ -125,14 +126,14 @@ defmodule Lightning.Adaptors.Scheduler do Whether a cycle has completed against a source that listed no adaptors at all, since this Scheduler started. - That is the one outcome no row can record: an upstream answering with an + That is the one outcome no row can record. An upstream answering with an empty list has told us there are no adaptors, and the empty catalogue it leaves behind is loaded rather than unloaded. Every other completed cycle leaves rows, which answer for themselves and keep answering after - a restart — so this deliberately says nothing about them, and a source + a restart. So this deliberately says nothing about them, and a source whose rows are later deleted reloads as it did before. - A cycle that failed to list, fetch or write is not a completed one: we + A cycle that failed to list, fetch or write is not a completed one. We cannot tell a source with nothing in it from one we could not read. Answers `false` for a Scheduler that is unreachable. @@ -176,9 +177,8 @@ defmodule Lightning.Adaptors.Scheduler do @impl true def handle_continue(:check_catalogue, state) do - # Read runs, and delay is computed, even when interval_ms == 0 — that's - # the only way an interval=0 (disabled) deployment still learns its - # catalogue is empty. Don't skip it for that branch. + # The read runs even when interval_ms == 0. It is the only way a + # deployment with the timer disabled learns its catalogue is empty. checked_at = case read_checked_at(state) do nil -> @@ -211,7 +211,7 @@ defmodule Lightning.Adaptors.Scheduler do end # An empty catalogue only needs an operator's attention when no timer will - # fill it; with an interval set the first tick is already due immediately. + # fill it. With an interval set the first tick is already due. defp log_empty_catalogue(state) do cond do state.interval_ms > 0 -> @@ -436,8 +436,7 @@ defmodule Lightning.Adaptors.Scheduler do started_at = System.monotonic_time(:millisecond) strategy = AdaptorsSupervisor.strategy(state.sup) - # Single DB round-trip serves both the icons-task input (prior etags) - # and the version diff used below to decide which adaptors to fetch. + # One query feeds both the icons task and the version diff below. existing_rows = Catalogue.list_adaptors(state.source) prior_etags = prior_etags_from_rows(existing_rows) @@ -476,10 +475,9 @@ defmodule Lightning.Adaptors.Scheduler do |> Enum.map(fn record -> persist_with_icons(record, icons, state) end) |> Enum.count(&(&1 == :ok)) - # Rows fetched this tick already have fresh icons from - # persist_with_icons/3. Everything else — touched or errored — is - # reconciled here too, so an icon-only upstream change still lands - # even when the version doesn't bump. + # Rows fetched this tick got their icons in persist_with_icons/3. + # The rest, touched or errored, get theirs here, so an icon-only + # upstream change lands even when the version doesn't bump. fetched_names = MapSet.new(fetched, & &1.name) unfetched_rows = @@ -534,7 +532,7 @@ defmodule Lightning.Adaptors.Scheduler do else case strategy.fetch_adaptor(name) do # Refetched only because the stored schema was nil, and upstream - # still has none: nothing to persist, so don't broadcast a change. + # still has none. Nothing to persist, so don't broadcast a change. {:ok, %{schema_data: nil}} when same_version? -> Catalogue.touch_checked_at(name, state.source) :touched @@ -563,7 +561,7 @@ defmodule Lightning.Adaptors.Scheduler do # jsDelivr 404s for a version it has not mirrored yet, which is # indistinguishable from a schema the source really dropped. On the - # periodic path we keep what we have; an operator refresh takes upstream + # periodic path we keep what we have. An operator refresh takes upstream # as-is and is where a real removal lands. defp keep_stored_schema( %{schema_data: nil} = record, @@ -643,9 +641,7 @@ defmodule Lightning.Adaptors.Scheduler do end :not_modified -> - # Upstream confirmed unchanged — leave row's existing icon and - # etag in place. Counted in the tick summary via - # count_not_modified/1. + # Upstream answered 304, so the row's icon and etag stay as they are. record _ -> @@ -653,9 +649,8 @@ defmodule Lightning.Adaptors.Scheduler do end end - # Stamp the etag onto the record only when the strategy supplied one - # (NPM 200 entries always have the key; Local omits it). A nil etag is - # not stamped — we preserve whatever was already on the row. + # A nil etag (the Local strategy sends none) leaves whatever the row + # already has in place rather than clearing it. defp maybe_put_etag(record, _shape, nil), do: record defp maybe_put_etag(record, shape, etag) when is_binary(etag) do @@ -673,8 +668,6 @@ defmodule Lightning.Adaptors.Scheduler do end) end - # `row` is an Adaptor struct (from list_adaptors/1), which exposes - # :name and the icon sha256 fields, which is all we need. defp apply_icons_to_existing(_row, package_icons, _state) when map_size(package_icons) == 0, do: :unchanged @@ -708,16 +701,14 @@ defmodule Lightning.Adaptors.Scheduler do case Map.get(package_icons, shape) do %{data: bytes, ext: ext, sha256: sha} = entry when is_binary(bytes) -> if Map.get(row, sha_key) == sha do - # Same bytes already on disk; the etag may still need - # refreshing if the strategy gave us a new (non-nil) value - # that differs from what we have. nil never clobbers. + # Same bytes already on disk, but the etag may still have moved. maybe_accumulate_etag(acc, etag_key, row, Map.get(entry, :etag)) else accumulate_fetched_icon(acc, shape, row, entry, ext, sha, bytes, state) end :not_modified -> - # 304 confirmed — nothing to write, etag already current. + # 304, so nothing to write. acc _ -> @@ -807,8 +798,8 @@ defmodule Lightning.Adaptors.Scheduler do end # A row or shape with no etag is left out rather than kept as an empty - # entry — the strategy already treats an absent entry as "no prior etag, - # don't send If-None-Match". + # entry. The strategy treats an absent entry as no prior etag and sends + # no If-None-Match. @spec prior_etags_from_rows([map()]) :: %{ String.t() => %{optional(:square | :rectangle) => String.t()} } diff --git a/lib/lightning/adaptors/seed.ex b/lib/lightning/adaptors/seed.ex index 1862c63fa36..3400c2b72ef 100644 --- a/lib/lightning/adaptors/seed.ex +++ b/lib/lightning/adaptors/seed.ex @@ -27,8 +27,8 @@ defmodule Lightning.Adaptors.Seed do * `:sup` - supervisor instance whose topic the broadcasts go to, defaulting to `Lightning.Adaptors.Config.default_instance/0` """ - # `path` is a mix-task argument (`mix lightning.adaptors.import`) or a - # release-command argument — an operator's own filesystem, not a request. + # `path` is a mix-task or release-command argument, so it names the + # operator's own filesystem rather than request input. # sobelow_skip ["Traversal.FileModule"] @spec seed_from_file(Path.t(), keyword()) :: {:ok, non_neg_integer()} def seed_from_file(path, opts \\ []) do @@ -64,7 +64,7 @@ defmodule Lightning.Adaptors.Seed do # `Lightning.Release.seed_adaptors/2` seeds through # `Ecto.Migrator.with_repo/2`, which starts the repo without the rest of - # the app — there is no PubSub to broadcast on, and no cache to evict. + # the app. There is no PubSub to broadcast on and no cache to evict. defp broadcast_changed(sup, source, names) do if Process.whereis(Lightning.PubSub) do topic = AdaptorsSupervisor.source_topic(sup) diff --git a/lib/lightning/adaptors/store.ex b/lib/lightning/adaptors/store.ex index 5d92d434a9c..5c582ffe6dd 100644 --- a/lib/lightning/adaptors/store.ex +++ b/lib/lightning/adaptors/store.ex @@ -3,8 +3,7 @@ defmodule Lightning.Adaptors.Store do Cached reads over `Lightning.Adaptors.Catalogue`. Every read checks the instance's Cachex first and falls back to the - catalogue table. Reads never write to the catalogue: the - `Lightning.Adaptors.Scheduler` is the only writer, so a row with no + catalogue table. Reads never fill the catalogue, so a row with no schema means the source has none and `schema/2` answers `"{}"`, while an unknown name returns `{:error, :not_found}`. `icon/3` returns a path on disk, fetching the bytes from the strategy on the first miss. `catalogue/1` @@ -13,12 +12,12 @@ defmodule Lightning.Adaptors.Store do ## The first load - Reads never block. A read that comes back empty or not-found asks whether - the catalogue has ever loaded, and answers `{:error, :not_ready}` when it - has not. An empty answer from a catalogue that has loaded is a real answer - and is returned as-is — including the empty catalogue a source with no - adaptors leaves behind, which the Scheduler reports as loaded despite there - being no row to find. + Reads never wait for the first load. A read that comes back empty or + not-found asks whether the catalogue has ever loaded, and answers + `{:error, :not_ready}` when it has not. An empty answer from a catalogue + that has loaded is a real answer and is returned as-is. That includes the + empty catalogue a source with no adaptors leaves behind, which the + Scheduler reports as loaded despite there being no row to find. Waiting for the first load is opt-in, through `ensure_loaded/2`. """ @@ -379,17 +378,11 @@ defmodule Lightning.Adaptors.Store do end end - # `Cachex.fetch/4` returns one of: - # * `{:ok, value}` — cache hit (or coalesced peer of a `:commit`) - # * `{:commit, value}` — fallback ran and committed - # * `{:ignore, value}` — fallback ran and chose not to cache - # * `{:error, term}` — Cachex-side failure (fallback raised, etc.) - # - # Every fallback returns an inner `{:ok, _} | {:error, _}`, whichever - # wrapper it chooses, so the wrapper tuple's second element is itself - # the public value we want to return, including a committed - # `{:error, _}`, which comes back as `{:ok, {:error, _}}` on a later - # hit. Cachex-side `{:error, _}` passes through unchanged. + # Every fallback returns an inner `{:ok, _} | {:error, _}` inside the + # Cachex wrapper, so the wrapper's second element is the public value. + # A committed `{:error, _}` comes back as `{:ok, {:error, _}}` on a later + # hit and unwraps the same way. A Cachex-side `{:error, _}` (the fallback + # raised) passes through unchanged. @spec unwrap(tuple()) :: {:ok, term()} | {:error, term()} defp unwrap({:ok, inner}), do: inner defp unwrap({:commit, inner}), do: inner diff --git a/lib/lightning/adaptors/strategy.ex b/lib/lightning/adaptors/strategy.ex index a1683250790..f0f79cba91f 100644 --- a/lib/lightning/adaptors/strategy.ex +++ b/lib/lightning/adaptors/strategy.ex @@ -1,31 +1,26 @@ defmodule Lightning.Adaptors.Strategy do @moduledoc """ - Behaviour shared by every adaptor strategy (NPM, Local, and the test - mock). + Behaviour every adaptor strategy implements. A strategy is the sole boundary between the `Lightning.Adaptors.*` subsystem and the outside world. It defines four callbacks: - * `c:fetch_adaptor/1` — given a package name, return a structured - `t:adaptor_record/0` covering version history, integrity hashes, - and dependency metadata. Icon fields are not part of this - record; the Scheduler stamps them on separately after joining - the bulk icon pipeline. - * `c:fetch_icon/2` — given a package name and an icon variant, - return the raw bytes plus extension. Used by the Store's rare - lazy-miss fallback. - * `c:fetch_icons/1` — bulk icon fetch for every adaptor known to - the strategy. The Scheduler invokes this once per tick in parallel - with its per-adaptor fan-out. Accepts a keyword list of options; - see the callback docs for `:prior_etags`. - * `c:list_adaptors/0` — the cheap change-signal: one call returning - `name + latest_version` for every `@openfn/*` package, used by - the scheduler to diff against the `adaptors` table. + * `c:fetch_adaptor/1` returns a `t:adaptor_record/0` for one package + name. Icon fields are not part of this record. The Scheduler stamps + them on separately after joining the bulk icon pipeline. + * `c:fetch_icon/2` returns the raw bytes and extension of one icon + variant. The Store calls it when an icon is missing on disk. + * `c:fetch_icons/1` bulk-fetches icons for every adaptor the strategy + knows. The Scheduler runs it once per tick in parallel with its + per-adaptor fan-out. See the callback docs for `:prior_etags`. + * `c:list_adaptors/0` is the cheap change signal: `name` and + `latest_version` for every package the strategy knows, which the + Scheduler diffs against the `adaptors` table. The active strategy module is resolved at runtime via - `Lightning.Adaptors.Config.strategy/0`. Implementations must surface - transient failures (5xx, timeout, nxdomain) as `{:error, term()}`; - retry policy lives at the scheduler/store layer, not here. + `Lightning.Adaptors.Config.strategy/0`. Implementations surface + transient failures (5xx, timeout, nxdomain) as `{:error, term()}` and + do not retry. """ @typedoc """ @@ -44,9 +39,8 @@ defmodule Lightning.Adaptors.Strategy do } @typedoc """ - The structured adaptor record returned by `c:fetch_adaptor/1`. Icon - fields are persisted separately by the Scheduler after joining - `c:fetch_icons/1` — they are not stamped onto this record. + The record returned by `c:fetch_adaptor/1`. Icon fields are not on it. + The Scheduler persists them separately after joining `c:fetch_icons/1`. `schema_data` is the credential schema as a JSON binary. `nil` means the source sees no schema for this version; the Scheduler decides @@ -66,11 +60,10 @@ defmodule Lightning.Adaptors.Strategy do } @typedoc """ - Fresh-fetch icon entry inside the `c:fetch_icons/1` result map. The - optional `:etag` field carries the upstream-provided cache validator - (verbatim from the HTTP response) and is `nil` when the upstream - didn't supply one — strategies without a transport-level validator - (e.g. `Lightning.Adaptors.Local`) omit the key entirely. + Fresh-fetch icon entry inside the `c:fetch_icons/1` result map. `:etag` + is the upstream cache validator verbatim from the HTTP response, `nil` + when upstream sent none. Strategies with no transport-level validator, + such as `Lightning.Adaptors.Local`, omit the key entirely. """ @type icon_entry :: %{ required(:data) => binary(), @@ -81,20 +74,16 @@ defmodule Lightning.Adaptors.Strategy do @typedoc """ Per-shape value inside the `c:fetch_icons/1` result map. Either a - fresh `t:icon_entry/0` (200 response) or the `:not_modified` sentinel - (304 response — upstream confirmed unchanged; only ever returned when - the caller supplied a prior etag via the `:prior_etags` option). + fresh `t:icon_entry/0` (a 200) or `:not_modified` (a 304), which is + only ever returned when the caller supplied a prior etag via + `:prior_etags`. """ @type icon_shape_value :: icon_entry() | :not_modified @typedoc """ - Bulk icon map returned by `c:fetch_icons/1`. Three branches matter: - - * shape **entirely absent** — upstream had no such icon for this - package; - * shape present as `:not_modified` — upstream confirmed the icon - is unchanged since the prior etag was issued; - * shape present as a map — apply the bytes (a fresh fetch). + Bulk icon map returned by `c:fetch_icons/1`. A shape that is absent + means upstream has no such icon. `:not_modified` means it is unchanged + since the prior etag. A map is a fresh fetch to apply. """ @type icons_map :: %{ required(String.t()) => %{ @@ -121,30 +110,26 @@ defmodule Lightning.Adaptors.Strategy do Bulk fetch every available icon for every adaptor known to the strategy. - Returns `{:ok, partial_map}` where each per-shape slot is either - absent (no icon upstream), a fresh `t:icon_entry/0` (200), or the - `:not_modified` sentinel (304 — only when a prior etag was sent). - A top-level `{:error, term()}` is only returned when the whole - pipeline can't proceed (e.g. an upstream `list_adaptors/0` call - inside the bulk implementation fails). + Returns `{:ok, icons_map}` (see `t:icons_map/0`). A top-level + `{:error, term()}` is only + returned when the whole pipeline cannot proceed, for example when the + `list_adaptors/0` call inside the bulk implementation fails. ## Options - * `:prior_etags` — a map of the form - `%{name => %{optional(:square | :rectangle) => etag_string}}` - whose values are sent as `If-None-Match` per `(name, shape)`. - Defaults to `%{}`. Unknown keys in the keyword list are - ignored. Strategies without a transport-level cache validator - (e.g. `Lightning.Adaptors.Local`) ignore this option entirely - and never return `:not_modified`. + * `:prior_etags` - `%{name => %{optional(:square | :rectangle) => etag}}`, + sent as `If-None-Match` per `(name, shape)`. Defaults to `%{}`. + Strategies with no transport-level cache validator, such as + `Lightning.Adaptors.Local`, ignore it and never return + `:not_modified`. """ @callback fetch_icons(opts :: keyword()) :: {:ok, icons_map()} | {:error, term()} @doc """ - Cheap change-signal listing: `name + latest_version` for every - `@openfn/*` package known to the strategy. The scheduler diffs this - against the `adaptors` table to compute its work list. + Cheap change signal: `name` and `latest_version` for every package the + strategy knows. The Scheduler diffs this against the `adaptors` table to + compute its work list. `{:ok, []}` means the strategy looked and there is genuinely nothing there, which settles the Store's first-load gate. A strategy that cannot diff --git a/lib/lightning/collections/item.ex b/lib/lightning/collections/item.ex index a8b6e6252e0..a36e0661148 100644 --- a/lib/lightning/collections/item.ex +++ b/lib/lightning/collections/item.ex @@ -43,8 +43,6 @@ defmodule Lightning.Collections.Item do "value is too long, please use a shorter one", 1_000_000 ) - # Width, not a null byte, so this is separate from the Collections jsonb - # work still outstanding. |> Lightning.Validators.validate_name_fits_column( :key, "key is too long, please use a shorter one" diff --git a/lib/lightning/config/bootstrap.ex b/lib/lightning/config/bootstrap.ex index b8fd9076054..febd4ef71c1 100644 --- a/lib/lightning/config/bootstrap.ex +++ b/lib/lightning/config/bootstrap.ex @@ -244,15 +244,13 @@ defmodule Lightning.Config.Bootstrap do configure_adaptors_strategy(local_adaptors_repos, use_local_adaptors_repos?) - # Upstreams for the NPM strategy. Each key reaches exactly one sub-module - # through Lightning.Adaptors.Config.strategy_opts/1: registry_url is the - # npm search and packument endpoint (NPM.Registry), jsdelivr_url serves - # configuration schemas (NPM.Schema), and github_url plus github_ref locate - # the raw icon files under OpenFn/adaptors (NPM.GitHub). Defaults live in - # the @default_* attributes on those modules; bootstrap only overrides one - # when its env var is set. + # Upstreams for the NPM strategy. registry_url is the npm search and + # packument endpoint, jsdelivr_url serves configuration schemas, and + # github_url plus github_ref locate the raw icon files under + # OpenFn/adaptors. Defaults live on the Lightning.Adaptors.NPM.* modules, + # and an unset env var leaves the default alone. # - # Point them at `bin/adaptor_cache` to serve all three from a local disk + # Point all of them at `bin/adaptor_cache` to serve from a local disk # cache while working on adaptors. config :lightning, Lightning.Adaptors.NPM, @@ -1069,11 +1067,9 @@ defmodule Lightning.Config.Bootstrap do end end - # ADAPTORS_LOCAL_REPO wins outright when set. When it's unset, fall back - # to the OPENFN_ADAPTORS_REPO value parsed above, but only warn about it - # when Lightning.Adaptors is actually running the Local strategy — an - # operator running the npm strategy can leave OPENFN_ADAPTORS_REPO set for - # the ws-worker without being warned about a var they still need. + # The deprecation warning only fires under the Local strategy. An operator + # on the npm strategy can leave OPENFN_ADAPTORS_REPO set for the ws-worker + # and should not be told to drop a var they still need. defp resolve_local_strategy_paths(local_adaptors_repos, adaptors_strategy) do case env!("ADAPTORS_LOCAL_REPO", :string, nil) |> parse_repo_list() do [] -> diff --git a/lib/lightning/credentials/credential.ex b/lib/lightning/credentials/credential.ex index dd34abddad6..6c77b5af12a 100644 --- a/lib/lightning/credentials/credential.ex +++ b/lib/lightning/credentials/credential.ex @@ -110,7 +110,7 @@ defmodule Lightning.Credentials.Credential do end # Expanding a legacy short name needs a loaded catalogue. When it cannot - # answer the name is stored as typed rather than failing the save: the + # answer, the name is stored as typed rather than failing the save. The # short form is a supported legacy shape that `get_schema/1` resolves on # read and `Credentials.reconcile_legacy_schema_names/1` rewrites later. defp resolve_schema_name(changeset) do diff --git a/lib/lightning/credentials/schema_reconciler.ex b/lib/lightning/credentials/schema_reconciler.ex index 659c558aea1..82ffe0226af 100644 --- a/lib/lightning/credentials/schema_reconciler.ex +++ b/lib/lightning/credentials/schema_reconciler.ex @@ -8,15 +8,15 @@ defmodule Lightning.Credentials.SchemaReconciler do broadcast (`Lightning.Adaptors.ChannelBroadcaster`) only fires when an adaptor row actually changes, so a catalogue that is already warm (e.g. after a restart, backed by a persistent store) may complete its first - refresh without changing a single row and would never emit anything — a + refresh without changing a single row and would never emit anything. A subscriber that waited only for the broadcast would then never sweep. The on-start run covers that case. The sweep is idempotent (each pass only touches rows still on a short name), so running it twice, from two triggers, or on every node in a cluster (the PubSub topic is cluster-wide) is safe. There is deliberately - no "done" flag gating it — that flag was the bug this module replaces: it - could get set after a failed run and then never retry. + no "done" flag gating it. A flag set after a failed run would stop every + later retry. Each sweep issues a `SELECT DISTINCT schema` over `credentials` (no index on that column) plus one catalogue lookup per distinct legacy name, and @@ -35,8 +35,8 @@ defmodule Lightning.Credentials.SchemaReconciler do @doc """ Starts the reconciler. Required opts: `:name`, `:sup`. Optional: `:reconcile` (1-arity fn, default - `&Lightning.Credentials.reconcile_legacy_schema_names/1`) and `:retry_ms` - (default 5 minutes) controlling the retry delay after a failed sweep. + `&Lightning.Credentials.reconcile_legacy_schema_names/1`) and `:retry_ms`, + the delay before a failed sweep is retried. """ @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts) do diff --git a/lib/lightning/export_utils.ex b/lib/lightning/export_utils.ex index b39675512a5..616e444c80f 100644 --- a/lib/lightning/export_utils.ex +++ b/lib/lightning/export_utils.ex @@ -561,9 +561,8 @@ defmodule Lightning.ExportUtils do ) end - # build_yaml_tree/2 raises from several levels down, so the alternative to - # rescuing here is threading an error tuple through every builder. One rescue - # at the single public boundary is the smaller change. + # build_yaml_tree/2 raises from several levels down. One rescue at the + # public boundary beats threading an error tuple through every builder. defp with_duplicate_key_error(build) do {:ok, build.()} rescue diff --git a/lib/lightning/export_utils/scalar.ex b/lib/lightning/export_utils/scalar.ex index 0d973b14628..5ab80e1250f 100644 --- a/lib/lightning/export_utils/scalar.ex +++ b/lib/lightning/export_utils/scalar.ex @@ -5,8 +5,8 @@ defmodule Lightning.ExportUtils.Scalar do `Lightning.ExportUtils` builds the project spec by concatenating strings, so every name, label and identifier has to be quoted and escaped here. - Output has to stay byte-identical for anything already emitted correctly: - customers keep their project spec in git, so a change in quoting style is a + Output has to stay byte-identical for anything already emitted correctly. + Customers keep their project spec in git, so a change in quoting style is a diff in every synced repo. That is why the bare and single-quoted shapes below are the historic ones rather than what a general purpose YAML writer would pick. @@ -19,7 +19,8 @@ defmodule Lightning.ExportUtils.Scalar do @bare_key ~r/\A[a-zA-Z0-9][a-zA-Z0-9_\-@\.>]*[a-zA-Z0-9]\z/ # The spellings that do not survive a round trip as the string we wrote. - # Measured against yamerl and yaml@2.7.1, in both key and value position; + # Measured against yamerl and the npm `yaml` parser, in both key and value + # position; # the corpus is in scalar_test.exs. Deliberately narrower than YAML 1.1: # `on`/`off` and `2026-08-27` are left bare because neither parser resolves # them and both were legal job names, so quoting would churn synced repos. diff --git a/lib/lightning/projects/provisioner.ex b/lib/lightning/projects/provisioner.ex index a5b1535e1dc..1fc88536d36 100644 --- a/lib/lightning/projects/provisioner.ex +++ b/lib/lightning/projects/provisioner.ex @@ -979,11 +979,10 @@ defmodule Lightning.Projects.Provisioner do new_project_creds_to_add = Enum.map(new_credential_params, fn cred_params -> - # Both sides normalised. The stored name went through - # `validate_name/3`; the spec body never did, so a client that writes - # a decomposed accent asks for a credential that renders identically - # to the stored one and is refused, with an error naming a credential - # the user can see on screen. + # The stored name went through `validate_name/3` and the spec body + # never did. Without normalising both sides, a spec carrying a + # decomposed accent is refused with an error naming a credential the + # user can see on screen. wanted = Validators.normalize_name_for_match(cred_params["name"]) credential = diff --git a/lib/lightning/projects/sandboxes.ex b/lib/lightning/projects/sandboxes.ex index 3794cfbc993..e9fffde9f8a 100644 --- a/lib/lightning/projects/sandboxes.ex +++ b/lib/lightning/projects/sandboxes.ex @@ -354,10 +354,8 @@ defmodule Lightning.Projects.Sandboxes do ) do selected_credential_ids = Map.get(opts, :selected_credential_ids, []) - # The merge creates every collection the target is missing except the - # names the caller explicitly skipped. A malformed skip list raises - # rather than silently changing what gets created. There is no deletion - # half: a merge never deletes target collections, whatever options a + # A malformed skip list raises rather than silently changing what gets + # created. A merge never deletes target collections, whatever options a # caller passes. skip_collection_names = validate_skip_collections!(Map.get(opts, :skip_collections, [])) @@ -1540,9 +1538,8 @@ defmodule Lightning.Projects.Sandboxes do "got: #{inspect(other)}" end - # Names that exist in the source but not the target. The single source of - # truth for both the merge-time sync and the preview the merge screen - # shows. + # Shared by the merge-time sync and the merge screen's preview so the two + # cannot disagree. defp source_only_collection_names(source, target) do source_names = source |> Collections.list_project_collections() |> names() target_names = target |> Collections.list_project_collections() |> names() diff --git a/lib/lightning/release.ex b/lib/lightning/release.ex index 0b9534e18cf..2bd42a0a97f 100644 --- a/lib/lightning/release.ex +++ b/lib/lightning/release.ex @@ -38,9 +38,9 @@ defmodule Lightning.Release do @doc """ Populate the adaptor catalogue from a JSON snapshot file, without - reaching npm. This is the release equivalent of - `mix lightning.adaptors.import` — a release has no Mix, so run this - through `bin/lightning eval` instead. + reaching npm. It is the release equivalent of + `mix lightning.adaptors.import`. A release has no Mix, so run this + through `bin/lightning eval`. ## Usage @@ -60,9 +60,9 @@ defmodule Lightning.Release do @doc """ Write the adaptor catalogue to a JSON snapshot file, without the rest - of the app running. This is the release equivalent of - `mix lightning.adaptors.dump` — a release has no Mix, so run this - through `bin/lightning eval` instead. + of the app running. It is the release equivalent of + `mix lightning.adaptors.dump`. A release has no Mix, so run this + through `bin/lightning eval`. ## Usage diff --git a/lib/lightning/runs/handlers.ex b/lib/lightning/runs/handlers.ex index 90d17230366..e3d7095f2d5 100644 --- a/lib/lightning/runs/handlers.ex +++ b/lib/lightning/runs/handlers.ex @@ -557,12 +557,11 @@ defmodule Lightning.Runs.Handlers do # Older workers JSON-encode output_dataclip into a string before sending # it; newer workers send it already decoded. A bare string is ambiguous - # either way — a job can legitimately return "24", "true", or "{}" as - # its literal state — so we try to JSON-decode it and fall back to the - # raw string if that fails. A literal string that happens to look like - # JSON (a job returning the string "24") ends up stored as the decoded - # value instead; we accept that ambiguity since returning a bare string - # as step state is already rare. + # either way, since a job can legitimately return "24", "true" or "{}" + # as its literal state. We try to JSON-decode it and fall back to the + # raw string. A literal string that looks like JSON (a job returning the + # string "24") is stored as the decoded value instead. Bare-string step + # state is rare enough to accept that. defp maybe_decode_dataclip(value) when is_binary(value) do case Jason.decode(value) do {:ok, decoded} -> decoded diff --git a/lib/lightning/utils/validators.ex b/lib/lightning/utils/validators.ex index 3d1f7ead1e5..b63b708a189 100644 --- a/lib/lightning/utils/validators.ex +++ b/lib/lightning/utils/validators.ex @@ -35,8 +35,8 @@ defmodule Lightning.Validators do # # Control characters are out because job names are written into the # `workflow_snapshots.jobs` jsonb column, and Postgres refuses a NUL inside - # jsonb, so a name carrying one crashes the snapshot insert (#4893). We - # reject rather than strip. + # jsonb, so a name carrying one crashes the snapshot insert. We reject + # rather than strip. # # `Lightning.LogMessage` keeps a narrower regex that it strips rather than # rejects. That is deliberate for log lines, which legitimately hold tabs and @@ -59,7 +59,7 @@ defmodule Lightning.Validators do # A name that merely contains one of these is fine: a joiner is how an emoji # sequence, a Devanagari conjunct and an Arabic ligature are written. # Written on one line on purpose: PCRE's /x does not ignore whitespace inside - # a character class, so laying this out over several lines silently put a + # a character class, so laying this out over several lines would put a # literal space and newline into the set. @invisible_regex ~r/\A[\p{Cf}\x{034F}\x{115F}\x{1160}\x{17B4}\x{17B5}\x{180B}-\x{180F}\x{2065}\x{2800}\x{3164}\x{FE00}-\x{FE0F}\x{FFA0}\x{FFF0}-\x{FFF8}\x{13430}-\x{1343F}\x{E0000}-\x{E0FFF}]+\z/u diff --git a/lib/lightning/workflows/edge.ex b/lib/lightning/workflows/edge.ex index fe42aa8f1c7..0e08e2a6b64 100644 --- a/lib/lightning/workflows/edge.ex +++ b/lib/lightning/workflows/edge.ex @@ -157,7 +157,7 @@ defmodule Lightning.Workflows.Edge do # # There is deliberately no `valid?: false` short circuit: the changeset is # invalid for unrelated missing fields on plenty of real save paths, and - # skipping these checks there is how the hole stayed open. + # skipping these checks there would let a NUL through to the snapshot. defp validate_condition_expression(changeset) do changeset |> Validators.validate_no_null_bytes( diff --git a/lib/lightning/workflows/job.ex b/lib/lightning/workflows/job.ex index 2fcc57f30a4..2749190a959 100644 --- a/lib/lightning/workflows/job.ex +++ b/lib/lightning/workflows/job.ex @@ -102,8 +102,7 @@ defmodule Lightning.Workflows.Job do ) |> validate_required(:name, message: "job name can't be blank") |> validate_required(:body, message: "job body can't be blank") - # Only the NUL: a body is code and legitimately has newlines and tabs in it - # (#4893). + # Only the NUL: a body is code and legitimately has newlines and tabs in it. |> Validators.validate_no_null_bytes( :body, "job body can't contain a null byte" diff --git a/lib/lightning/workflows/workflow.ex b/lib/lightning/workflows/workflow.ex index 174513d190b..7b0d34b7419 100644 --- a/lib/lightning/workflows/workflow.ex +++ b/lib/lightning/workflows/workflow.ex @@ -112,7 +112,7 @@ defmodule Lightning.Workflows.Workflow do "workflow name can't contain control characters" ) # positions is written straight into the workflow_snapshots.positions jsonb, - # keys and all, and Postgres refuses a NUL anywhere inside jsonb (#4893). + # keys and all, and Postgres refuses a NUL anywhere inside jsonb. |> Validators.validate_no_null_bytes_deep( :positions, "positions can't contain a null byte" diff --git a/lib/lightning/workflows/workflow_template.ex b/lib/lightning/workflows/workflow_template.ex index f02c5e92d5a..48012ddd543 100644 --- a/lib/lightning/workflows/workflow_template.ex +++ b/lib/lightning/workflows/workflow_template.ex @@ -42,8 +42,9 @@ defmodule Lightning.Workflows.WorkflowTemplate do max: 1000, message: "Description must be less than 1000 characters" ) - # positions is jsonb and code, description and tags are text columns, all - # copied into the snapshot. publish_template exposes every one (#4893). + # positions is jsonb and code, description and tags are text columns. + # Postgres refuses a NUL in either, and publish_template passes every one + # of them straight from the client. |> Lightning.Validators.validate_no_null_bytes_deep( :positions, "Positions can't contain a null byte" diff --git a/lib/lightning_web/channels/workflow_channel.ex b/lib/lightning_web/channels/workflow_channel.ex index 355fe03bb46..ad6a0f27ec2 100644 --- a/lib/lightning_web/channels/workflow_channel.ex +++ b/lib/lightning_web/channels/workflow_channel.ex @@ -291,9 +291,9 @@ defmodule LightningWeb.WorkflowChannel do @doc """ Saves the current Y.Doc state through the Session. - The reply is deferred: `Session.save_workflow/2` may wait on the adaptor + The reply is deferred. `Session.save_workflow/2` may wait on the adaptor catalogue's first load, so the call runs off the channel process and - the reply is sent with `Phoenix.Channel.reply/2` when it finishes. + the reply goes out with `Phoenix.Channel.reply/2` when it finishes. Success: `{:ok, %{saved_at: DateTime, lock_version: integer}}` Error: `{:error, %{errors: map, type: string}}` @@ -883,10 +883,9 @@ defmodule LightningWeb.WorkflowChannel do {:reply, {:ok, %{}}, socket} end - # Catch-all for any event this channel doesn't recognise (e.g. a stale - # client tab still sending an event removed in a later deploy). Replies - # with an error instead of raising FunctionClauseError, which would kill - # this client's channel process and drop its connection. + # A stale client tab may still send an event removed in a later deploy. + # Replying with an error instead of raising FunctionClauseError keeps this + # client's channel process and connection alive. @impl true def handle_in(event, _payload, socket) do warn_unhandled_message("handle_in", event) @@ -1205,10 +1204,9 @@ defmodule LightningWeb.WorkflowChannel do {:noreply, socket} end - # Catch-all for any internal message this channel doesn't recognise (e.g. a - # PubSub broadcast for an event type removed in a later deploy). Logs and - # keeps the channel alive instead of raising FunctionClauseError, which - # would kill this client's channel process and drop its connection. + # A PubSub broadcast for an event type removed in a later deploy can still + # arrive here. Logging instead of raising FunctionClauseError keeps this + # client's channel process and connection alive. @impl true def handle_info(message, socket) do warn_unhandled_message("handle_info", unhandled_message_type(message)) @@ -1264,7 +1262,7 @@ defmodule LightningWeb.WorkflowChannel do defp refresh_lifecycle_from_broadcast(socket, _payload), do: socket - # Unlinked on purpose: a GenServer.call timeout or dead target exits, and + # Unlinked on purpose. A GenServer.call timeout or dead target exits, and # a linked task would take the channel down. `catch :exit` turns it into # an error reply instead. defp async_task(socket, event, task_fn) do @@ -1317,9 +1315,8 @@ defmodule LightningWeb.WorkflowChannel do {:noreply, socket} end - # Logs and reports to Sentry that a channel message went unhandled, by - # event name only. The full message/payload is never logged since it may - # carry user or workflow data. + # Only the event name goes to the log and Sentry. The full message or + # payload may carry user or workflow data. defp warn_unhandled_message(kind, event) do Logger.warning("WorkflowChannel: unhandled #{kind} event: #{event}") @@ -1678,8 +1675,8 @@ defmodule LightningWeb.WorkflowChannel do end end - # Returns the bare reply payload, not `{:reply, ..., socket}`, so deferred - # replies can use it too. + # Returns the bare reply payload so both `handle_in` and the deferred + # `handle_info` clauses can use it. defp workflow_error_reply({:error, %{type: type, message: message}}) do {:error, %{ diff --git a/lib/lightning_web/controllers/adaptor_controller.ex b/lib/lightning_web/controllers/adaptor_controller.ex index f1c747eaab3..27c2f52627d 100644 --- a/lib/lightning_web/controllers/adaptor_controller.ex +++ b/lib/lightning_web/controllers/adaptor_controller.ex @@ -7,8 +7,9 @@ defmodule LightningWeb.AdaptorController do matching `If-None-Match` answers 304 without touching Postgres, and a miss on the ETag still serves an already-rendered payload. - A catalogue that has never loaded answers 503 with a `retry-after`, not - an empty list: the picker shows its retry state instead of no adaptors. + A catalogue that has never loaded answers 503 with a `retry-after` rather + than an empty list, so the picker shows its retry state instead of no + adaptors. """ use LightningWeb, :controller diff --git a/lib/lightning_web/controllers/adaptor_icon_controller.ex b/lib/lightning_web/controllers/adaptor_icon_controller.ex index ab44ce5b029..c0100ffc7f0 100644 --- a/lib/lightning_web/controllers/adaptor_icon_controller.ex +++ b/lib/lightning_web/controllers/adaptor_icon_controller.ex @@ -1,17 +1,16 @@ defmodule LightningWeb.AdaptorIconURL do @moduledoc """ - Single source of truth for content-addressable adaptor-icon URLs. + Builds content-addressable adaptor icon URLs. - `sha8` is the first 4 raw bytes of the icon's sha256, hex-encoded - to 8 lowercase characters, yielding a deterministic content-addressable - path segment. + `sha8` is the first 4 raw bytes of the icon's sha256, hex-encoded to 8 + lowercase characters. """ @doc """ Build a content-addressable icon URL for `name`/`shape`. - Returns `nil` when `meta` has no ext or sha256 for the requested shape - — i.e. when no icon is available. + Returns `nil` when `meta` has no ext or sha256 for the requested shape, + meaning no icon is available. """ alias Lightning.Adaptors.IconField @@ -35,15 +34,17 @@ defmodule LightningWeb.AdaptorIconController do Route: `/adaptors/icons/:name/:shape-:sha8.:ext` - `sha8` is the first 4 raw bytes of the stored sha256 hex-encoded to - 8 lowercase characters. The controller compares `sha8` against the - DB-projected metadata and responds with one of: - - - **200** — sha matches; serves bytes with a 1-year immutable cache. - - **302** — sha is stale but the adaptor still has an icon; redirects - to the canonical (current-sha) URL with `Cache-Control: no-store` - on the redirect itself. - - **404** — adaptor unknown, ext mismatch, bad shape, or no icon. + `sha8` is defined in `LightningWeb.AdaptorIconURL`. The controller + compares it against the stored icon metadata and responds with one of: + + - **200** when `sha8` matches. Serves the bytes with a one-year immutable + cache. + - **302** when `sha8` is stale but the adaptor still has an icon. + Redirects to the current-sha URL with `Cache-Control: no-store` on the + redirect itself. + - **404** when the adaptor is unknown, the ext mismatches, the shape is + bad, or there is no icon. + - **503** with `retry-after` when the catalogue has never loaded. """ use LightningWeb, :controller @@ -53,10 +54,10 @@ defmodule LightningWeb.AdaptorIconController do @immutable_cache "public, max-age=31536000, immutable" - # Router-shaped params: a single `:filename` segment of the form - # `-.` because Phoenix path matchers permit only one - # dynamic segment per path component. We split here and delegate to the - # 4-key clause below, which is also what the unit tests call directly. + # Phoenix path matchers allow one dynamic segment per path component, so + # the router hands over a single `:filename` of the form + # `-.`. The first clause splits it and delegates to the + # 4-key clause, which stays separate so tests can call it directly. @filename_regex ~r/\A(?[a-z]+)-(?[A-Fa-f0-9]+)\.(?[A-Za-z0-9]+)\z/ @doc false diff --git a/lib/lightning_web/live/maintenance_live/index.ex b/lib/lightning_web/live/maintenance_live/index.ex index 4102b86f642..01e15d44e8b 100644 --- a/lib/lightning_web/live/maintenance_live/index.ex +++ b/lib/lightning_web/live/maintenance_live/index.ex @@ -1,13 +1,11 @@ defmodule LightningWeb.MaintenanceLive.Index do @moduledoc """ - Superuser-only maintenance page for on-demand operations against - `Lightning.Adaptors`. + Superuser-only page for on-demand `Lightning.Adaptors` maintenance. - Exposes two actions: "Refresh Adaptor Registry" (`refresh/0`) and - "Refresh Adaptor Icons" (`refresh_icons/0`). Neither blocks the LiveView: - the registry refresh is fire-and-forget on the cluster-singleton scheduler, - while the icon refresh runs under `start_async` (the underlying call can - take up to two minutes) and flashes its result when it completes. + Neither action blocks the LiveView. The registry refresh is queued on the + scheduler and returns at once. The icon refresh can run for up to two + minutes, so it goes through `start_async` and flashes its result when it + completes. """ use LightningWeb, :live_view diff --git a/lib/lightning_web/live/project_live/github_sync_component.ex b/lib/lightning_web/live/project_live/github_sync_component.ex index eef0c134b9a..aba25b506cb 100644 --- a/lib/lightning_web/live/project_live/github_sync_component.ex +++ b/lib/lightning_web/live/project_live/github_sync_component.ex @@ -216,9 +216,9 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do %Tesla.Env{body: %{} = body} -> error_message(body) - # Usually ours: the export pre-flight fails with a plain string naming - # both colliding entities, which the generic message below would lose. - # Not exclusively, though: refresh_oauth_token/1 passes GitHub's body + # Usually this is our export pre-flight failing with a plain string that + # names both colliding entities, which the generic message below would + # lose. Not always, though. refresh_oauth_token/1 passes GitHub's body # straight through. message when is_binary(message) -> message diff --git a/lib/lightning_web/live/sandbox_live/index.ex b/lib/lightning_web/live/sandbox_live/index.ex index 17784c19d82..44abf8266cc 100644 --- a/lib/lightning_web/live/sandbox_live/index.ex +++ b/lib/lightning_web/live/sandbox_live/index.ex @@ -787,9 +787,9 @@ defmodule LightningWeb.SandboxLive.Index do |> assign(:merge_selected_collection_names, MapSet.new(to_create)) end - # The change event fires for every input in the merge form (toggling any - # checkbox re-submits it), so only recompute the collections preview - and - # reset the row selections - when the target actually changed. + # Toggling any checkbox re-submits the merge form, so the change event + # fires for every input. Recomputing the preview also resets the row + # selections, so only do it when the target changed. defp maybe_assign_merge_collections(socket, sandbox, target_project) do new_target_id = target_project && target_project.id @@ -1117,11 +1117,9 @@ defmodule LightningWeb.SandboxLive.Index do MergeProjects.diverged_workflows(target_project, source) end - # The merge always creates whatever collections the target lacks; only - # names the user explicitly unchecked are skipped. The unchecked set is - # honored only for the target it was previewed against - for any other - # target nothing is skipped. Either way a collection added to the sandbox - # after the preview still gets created: skipping is explicit, creating is + # Only names the user explicitly unchecked are skipped, and only for the + # target they were previewed against. A collection added to the sandbox + # after the preview is still created. Skipping is explicit, creating is # the default. defp skipped_collection_names(assigns, target) do if assigns.merge_collections_target_id == target.id do diff --git a/lib/mix/tasks/lightning.adaptors.dump.ex b/lib/mix/tasks/lightning.adaptors.dump.ex index 26b34dcfdf8..84b79e45cc9 100644 --- a/lib/mix/tasks/lightning.adaptors.dump.ex +++ b/lib/mix/tasks/lightning.adaptors.dump.ex @@ -3,15 +3,14 @@ defmodule Mix.Tasks.Lightning.Adaptors.Dump do @moduledoc """ Write this instance's adaptor catalogue to a JSON file, in the shape - `Lightning.Adaptors.Catalogue.upsert_adaptor/1` accepts — the shape + `Lightning.Adaptors.Catalogue.upsert_adaptor/1` accepts and `mix lightning.adaptors.import` reads back. - This is the catalogue-to-file leg of mirroring adaptors into an - airgapped environment: hydrate an online instance as usual, dump it - here, carry the file across, and import it on the offline instance. - `mix lightning.adaptors.snapshot` produces the same kind of file by - fetching npm directly, for when there is no populated catalogue to - dump from. + This is how adaptors get mirrored into an airgapped environment. Hydrate + an online instance as usual, dump it here, carry the file across, and + import it on the offline instance. `mix lightning.adaptors.snapshot` + produces the same kind of file by fetching npm directly, for when there + is no populated catalogue to dump from. ## Usage @@ -24,7 +23,7 @@ defmodule Mix.Tasks.Lightning.Adaptors.Dump do so they don't travel in this file. It carries the icon metadata (extension, sha256, etag) so an imported row can serve an icon already present at `ADAPTORS_ICONS_PATH` on the target instance instead of - refetching from GitHub; copy that directory across alongside this file. + refetching from GitHub. Copy that directory across alongside this file. """ use Mix.Task diff --git a/lib/mix/tasks/lightning.adaptors.import.ex b/lib/mix/tasks/lightning.adaptors.import.ex index bac35b9fdbf..e4095ba3352 100644 --- a/lib/mix/tasks/lightning.adaptors.import.ex +++ b/lib/mix/tasks/lightning.adaptors.import.ex @@ -5,7 +5,7 @@ defmodule Mix.Tasks.Lightning.Adaptors.Import do Populate the `adaptors` table from a JSON file, without reaching npm. The file is a JSON array of adaptor records in the shape - `Lightning.Adaptors.Catalogue.upsert_adaptor/1` accepts — the same shape + `Lightning.Adaptors.Catalogue.upsert_adaptor/1` accepts, the same shape `mix lightning.adaptors.snapshot` writes. ## Usage diff --git a/lib/mix/tasks/lightning.adaptors.snapshot.ex b/lib/mix/tasks/lightning.adaptors.snapshot.ex index 85fb353e910..eaa7d349286 100644 --- a/lib/mix/tasks/lightning.adaptors.snapshot.ex +++ b/lib/mix/tasks/lightning.adaptors.snapshot.ex @@ -7,7 +7,7 @@ defmodule Mix.Tasks.Lightning.Adaptors.Snapshot do the shape `Lightning.Adaptors.Catalogue.upsert_adaptor/1` accepts. Nothing here touches the catalogue, so this works on an instance with - no database yet — it is the cold-start way to produce a snapshot. + no database yet. It is the cold-start way to produce a snapshot. `mix lightning.adaptors.dump` is the equivalent for an instance whose catalogue is already populated. Either file can be read back by `mix lightning.adaptors.import`. diff --git a/mix.exs b/mix.exs index dfc2fe9e2bb..7f819adb9d5 100644 --- a/mix.exs +++ b/mix.exs @@ -156,8 +156,9 @@ defmodule Lightning.MixProject do {:phoenix_live_view, "~> 1.0.17"}, {:cors_plug, "~> 3.0"}, {:plug_cowboy, "~> 2.5"}, - # highlander_pg 1.0.8 caps postgrex at ~> 0.21; it only issues advisory - # locks, so override rather than hold the rest of the app back. + # highlander_pg declares a narrower postgrex range than the rest of the + # app needs (see deps/highlander_pg/mix.exs). It only issues advisory + # locks, so override rather than hold the app back. {:postgrex, ">= 0.0.0", override: true}, {:prom_ex, "~> 1.11.0"}, {:rambo, "~> 0.3.4"}, diff --git a/test/lightning/adaptor_service_test.exs b/test/lightning/adaptor_service_test.exs index 83f4e83bdd0..45603606912 100644 --- a/test/lightning/adaptor_service_test.exs +++ b/test/lightning/adaptor_service_test.exs @@ -7,7 +7,7 @@ defmodule Lightning.AdaptorServiceTest do back as `{:catalogue_unavailable, reason}` rather than a refusal. """ - # set_mox_global: the load runs in a Task owned by the Scheduler. + # The load runs in a Task owned by the Scheduler, so Mox must be in global mode. use Lightning.DataCase, async: false import Mox diff --git a/test/lightning/adaptors/channel_broadcaster_test.exs b/test/lightning/adaptors/channel_broadcaster_test.exs index 04c480362e3..84a7540924f 100644 --- a/test/lightning/adaptors/channel_broadcaster_test.exs +++ b/test/lightning/adaptors/channel_broadcaster_test.exs @@ -12,8 +12,6 @@ defmodule Lightning.Adaptors.ChannelBroadcasterTest do setup do sup = :"cb_test_#{System.unique_integer([:positive])}" - # The supervisor starts the ChannelBroadcaster automatically, registered - # under `channel_broadcaster_name(sup)`. start_supervised!( {AdaptorsSupervisor, name: sup, @@ -72,7 +70,7 @@ defmodule Lightning.Adaptors.ChannelBroadcasterTest do refute_receive %{event: "adaptors_updated"}, 100 # A second, separate burst must not still carry the first burst's - # names — the accumulator has to reset after :flush. + # names. The accumulator has to reset after :flush. changed(source_topic, "@openfn/language-dhis2") assert_receive %{ diff --git a/test/lightning/adaptors/end_to_end_broadcast_test.exs b/test/lightning/adaptors/end_to_end_broadcast_test.exs index 4e26c8295a1..8a78b28e6c4 100644 --- a/test/lightning/adaptors/end_to_end_broadcast_test.exs +++ b/test/lightning/adaptors/end_to_end_broadcast_test.exs @@ -5,7 +5,7 @@ defmodule Lightning.Adaptors.EndToEndBroadcastTest do the per-instance client topic (which `WorkflowChannel` subscribers use for display freshness) as a single coalesced `adaptors_updated` envelope. - This is the only test that exercises the full wiring across Invalidator, + Exercises the full wiring across Invalidator, NodeMonitor, ChannelBroadcaster, and Scheduler. """ diff --git a/test/lightning/adaptors/highlander_integration_test.exs b/test/lightning/adaptors/highlander_integration_test.exs index 8dda75166ad..da49b344593 100644 --- a/test/lightning/adaptors/highlander_integration_test.exs +++ b/test/lightning/adaptors/highlander_integration_test.exs @@ -6,15 +6,15 @@ defmodule Lightning.Adaptors.HighlanderIntegrationTest do Both supervisors share an explicit `:lock_key` so they race for the same `pg_try_advisory_lock` bucket, but each keeps its own derived - `:global` Scheduler name. Exactly one of them — the leader — registers - a Scheduler under its `{:global, …}` name; the other's HighlanderPG + `:global` Scheduler name. Exactly one of them, the leader, registers + a Scheduler under its `{:global, …}` name. The other's HighlanderPG polls and waits. When the leading supervisor stops (releasing its Postgres session and thus its advisory lock), the surviving instance must acquire the lock within ~2× the default 300ms polling interval and register its own Scheduler under its own `{:global, …}` name. """ - # async: false — real advisory locks coordinate against the test DB; + # async: false because real advisory locks coordinate against the test DB. # set_mox_global so the StrategyMock is visible to the wrapped child # processes started by HighlanderPG. use Lightning.DataCase, async: false @@ -28,7 +28,7 @@ defmodule Lightning.Adaptors.HighlanderIntegrationTest do setup :verify_on_exit! # Both supervisors come up with refresh_interval=0 (default in config/test.exs) - # so the inert Scheduler's init does no DB work; the test only cares about + # so the inert Scheduler's init does no DB work. The test only cares about # HighlanderPG's leader election, not refresh behaviour. test "two supervisors sharing one lock_key: only one runs the Scheduler at a time; failover on leader shutdown" do diff --git a/test/lightning/adaptors/invalidator_test.exs b/test/lightning/adaptors/invalidator_test.exs index b1a5e57bbd6..917efa790eb 100644 --- a/test/lightning/adaptors/invalidator_test.exs +++ b/test/lightning/adaptors/invalidator_test.exs @@ -6,8 +6,6 @@ defmodule Lightning.Adaptors.InvalidatorTest do setup do sup = :"inv_test_#{System.unique_integer([:positive])}" - # The supervisor starts the Invalidator automatically, registered under - # `invalidator_name(sup)`. start_supervised!( {AdaptorsSupervisor, name: sup, diff --git a/test/lightning/adaptors/node_monitor_test.exs b/test/lightning/adaptors/node_monitor_test.exs index ed40f2e3aa1..d134d7d36b7 100644 --- a/test/lightning/adaptors/node_monitor_test.exs +++ b/test/lightning/adaptors/node_monitor_test.exs @@ -13,8 +13,6 @@ defmodule Lightning.Adaptors.NodeMonitorTest do setup do sup = :"nm_test_#{System.unique_integer([:positive])}" - # The supervisor starts the NodeMonitor automatically, registered under - # `node_monitor_name(sup)`. start_supervised!( {AdaptorsSupervisor, name: sup, @@ -28,9 +26,9 @@ defmodule Lightning.Adaptors.NodeMonitorTest do nm_pid = Process.whereis(nm_name) Ecto.Adapters.SQL.Sandbox.allow(Lightning.Repo, self(), nm_pid) - # Scheduler is auto-started too (wrapped in HighlanderPG, registered - # via :global). It may not be up yet at setup time — HighlanderPG - # polls at 300ms — so this is best-effort. + # The Scheduler is auto-started too, wrapped in HighlanderPG and registered + # via :global. HighlanderPG polls at 300ms, so it may not be up yet at + # setup time. This is best-effort. {:global, global_sched_name} = AdaptorsSupervisor.global_scheduler_name(sup) sched_pid = :global.whereis_name(global_sched_name) diff --git a/test/lightning/adaptors/readiness_test.exs b/test/lightning/adaptors/readiness_test.exs index 1dc1c483d12..0d0b314a7aa 100644 --- a/test/lightning/adaptors/readiness_test.exs +++ b/test/lightning/adaptors/readiness_test.exs @@ -45,8 +45,8 @@ defmodule Lightning.Adaptors.ReadinessTest do start_supervised!({ Scheduler, # Its boot-time max_checked_at read runs in a process with no - # $callers chain back to this test — the Sandbox.allow/3 below - # races it, so skip the read rather than risk an OwnershipError. + # $callers chain back to this test, and the Sandbox.allow/3 below + # races it. Skip the read rather than risk an OwnershipError. name: AdaptorsSupervisor.global_scheduler_name(sup), sup: sup, lock_key: AdaptorsSupervisor.lock_key(sup), diff --git a/test/lightning/adaptors/scheduler_test.exs b/test/lightning/adaptors/scheduler_test.exs index 7fdbc4876fc..e63897c1bfa 100644 --- a/test/lightning/adaptors/scheduler_test.exs +++ b/test/lightning/adaptors/scheduler_test.exs @@ -1,5 +1,5 @@ defmodule Lightning.Adaptors.SchedulerTest do - # async: false — DataCase's shared sandbox mode means every process can + # async: false because DataCase's shared sandbox mode lets every process # reach the DB without an allow/3 call, and set_mox_global is only safe # when tests run serially. use Lightning.DataCase, async: false @@ -30,9 +30,9 @@ defmodule Lightning.Adaptors.SchedulerTest do start_supervised!({ AdaptorsSupervisor, - # Keeps the auto-started scheduler a true inert no-op ahead of - # start_scheduler/2 below — otherwise its boot-time max_checked_at - # read logs an empty-catalogue warning on every test in this file. + # checked_at spares the auto-started scheduler its boot-time DB read. + # It never ticks (refresh_interval is 0 in config/test.exs) and + # start_scheduler/2 replaces it. name: sup, strategy: Lightning.Adaptors.StrategyMock, checked_at: fn _source -> nil end @@ -60,9 +60,7 @@ defmodule Lightning.Adaptors.SchedulerTest do global_name = AdaptorsSupervisor.global_scheduler_name(sup) source_topic = AdaptorsSupervisor.source_topic(sup) - # Stop the supervisor's auto-started HighlanderPG (and its wrapped - # Scheduler) so we can start a replacement under the controlled - # interval without name collision. + # Terminate first so the replacement can take the same global name. :ok = Supervisor.terminate_child(sup, AdaptorsSupervisor.highlander_name(sup)) @@ -147,7 +145,7 @@ defmodule Lightning.Adaptors.SchedulerTest do {:ok, []} end) - # Empty table → max_checked_at returns nil → delay 0 → tick fires on init. + # The table is empty, so the first tick fires on init. start_scheduler(sup) assert_receive :list_adaptors_called, 2000 @@ -156,13 +154,11 @@ defmodule Lightning.Adaptors.SchedulerTest do test "tick re-arms itself", %{sup: sup} do test_pid = self() - # Stub allows repeated calls; each fires a message so we can count them. stub(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> send(test_pid, :tick_ran) {:ok, []} end) - # 30ms interval → two ticks fire well within 2s. start_scheduler(sup, interval: 30) assert_receive :tick_ran, 2000 @@ -291,8 +287,8 @@ defmodule Lightning.Adaptors.SchedulerTest do :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) start_scheduler(sup) - # With a recently-inserted adaptor, max_checked_at is "now", so the smart- - # init delay is ~99,999 seconds. Trigger an explicit tick via refresh_now. + # A recently-inserted adaptor makes max_checked_at "now", so the init + # tick is a full interval away. Trigger one explicitly via refresh_now. sched_name = AdaptorsSupervisor.global_scheduler_name(sup) Scheduler.refresh_now(sched_name) @@ -603,7 +599,7 @@ defmodule Lightning.Adaptors.SchedulerTest do ]} end) - # Single multi-clause expectation — Mox routes by pattern within + # Single multi-clause expectation. Mox routes by pattern within # one slot, so Scheduler's async_stream_nolink can fan out to the # two adaptors in either order. Two separate `expect/4` calls # would dispatch FIFO and crash with FunctionClauseError when the @@ -676,7 +672,6 @@ defmodule Lightning.Adaptors.SchedulerTest do start_scheduler(sup) - # Wait for init tick. assert_receive :tick_ran, 2000 sched_name = AdaptorsSupervisor.global_scheduler_name(sup) @@ -706,7 +701,7 @@ defmodule Lightning.Adaptors.SchedulerTest do {:global, gname} = sched_name pid = :global.whereis_name(gname) - # Init tick, then two manual refresh_now calls — each waited out so it + # Init tick, then two manual refresh_now calls, each waited out so it # starts its own cycle instead of coalescing into the previous one. assert_receive :tick_ran, 2000 assert_eventually(:sys.get_state(pid).refresh == nil, 2000) @@ -914,7 +909,7 @@ defmodule Lightning.Adaptors.SchedulerTest do new_sha = :crypto.hash(:sha256, new_bytes) # Same version and a stored schema, so the diff path marks this - # adaptor :touched instead of re-fetching it — only the icon changed. + # adaptor :touched instead of re-fetching it. expect(Lightning.Adaptors.StrategyMock, :list_adaptors, fn -> {:ok, [%{name: "@openfn/language-http", latest_version: "1.0.0"}]} end) @@ -986,8 +981,8 @@ defmodule Lightning.Adaptors.SchedulerTest do :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) start_scheduler(sup) - # The pre-seeded row pushes max_checked_at to "now", so init - # delay = full interval — drive the tick explicitly. + # The pre-seeded row pushes max_checked_at to "now", so the init tick + # is a full interval away. Drive it explicitly. sched_name = AdaptorsSupervisor.global_scheduler_name(sup) assert {:ok, %{errors: 0}} = Scheduler.await_refresh(sched_name, 5_000) @@ -1022,7 +1017,7 @@ defmodule Lightning.Adaptors.SchedulerTest do :ok = Phoenix.PubSub.subscribe(Lightning.PubSub, source_topic) start_scheduler(sup) - # Drain the init tick (table is empty → delay 0 → fires immediately). + # Drain the init tick, which fires immediately on an empty table. assert_receive :init_list_adaptors_called, 2000 sched_name = AdaptorsSupervisor.global_scheduler_name(sup) @@ -1102,8 +1097,7 @@ defmodule Lightning.Adaptors.SchedulerTest do {:ok, []} end) - # Exactly one fetch_icons call — the init tick. If refresh_package - # also fetched icons the count would be 2 and Mox would fail. + # The one allowed fetch_icons call belongs to the init tick. expect(Lightning.Adaptors.StrategyMock, :fetch_icons, 1, fn _opts -> send(test_pid, :icons_called) {:ok, %{}} diff --git a/test/lightning/adaptors/store_test.exs b/test/lightning/adaptors/store_test.exs index af8380fa74e..98bd182efd9 100644 --- a/test/lightning/adaptors/store_test.exs +++ b/test/lightning/adaptors/store_test.exs @@ -14,10 +14,9 @@ defmodule Lightning.Adaptors.StoreTest do setup :verify_on_exit! setup do - # Each test owns an isolated `Lightning.Adaptors.Supervisor` instance, - # parameterised on a unique `name:` so cache table / persistent_term - # entries don't collide across the async suite. The `:strategy` opt - # is threaded explicitly — no `Application.put_env` mutation. + # Each test owns an isolated supervisor on a unique `name:` so cache + # table and persistent_term entries don't collide across the async + # suite. Passing `:strategy` directly keeps app env untouched. sup = :"store_test_#{System.unique_integer([:positive])}" start_supervised!( @@ -421,11 +420,9 @@ defmodule Lightning.Adaptors.StoreTest do end describe "icon/3" do - # Each test uses a unique adaptor name so the on-disk cache (shared - # default {:tmp, "lightning/adaptor_icons"} path) does not collide - # across this `async: true` suite. Directories created here are not - # cleaned up — they live under System.tmp_dir! and are namespaced - # per-name so they cannot collide. + # Each test uses a unique adaptor name so the shared on-disk icon cache + # does not collide across this `async: true` suite. Directories are + # left behind under System.tmp_dir!, namespaced per name. defp unique_name(prefix) do "@openfn/language-#{prefix}-#{System.unique_integer([:positive])}" end @@ -576,7 +573,7 @@ defmodule Lightning.Adaptors.StoreTest do assert {:ok, path} = Store.icon(sup, name, :square) assert File.read!(path) == "LAZY_BYTES" - # Courier returned {:ignore, _} → no committed entry on the bytes key. + # The courier returned {:ignore, _}, so nothing was committed on the bytes key. assert {:ok, nil} = Cachex.get(cache, {:icon_bytes, source, name, :square}) end @@ -624,8 +621,7 @@ defmodule Lightning.Adaptors.StoreTest do ) ) - # Single Mox expectation → if both callers reach the strategy - # the second hits "no expectation" and Mox raises. + # Single Mox expectation, so Mox raises if both callers reach the strategy. expect(Lightning.Adaptors.StrategyMock, :fetch_icon, 1, fn ^name, :square -> send(test_pid, :fetch_started) @@ -669,7 +665,7 @@ defmodule Lightning.Adaptors.StoreTest do ) ) - # Single multi-clause expectation with count: 2 — Mox routes by + # Single multi-clause expectation with count: 2. Mox routes by # pattern within one slot, so the two parallel courier calls can # arrive in either order. Two separate `expect/3` calls would # queue FIFO and crash with FunctionClauseError when the task diff --git a/test/lightning/adaptors/supervisor_integration_test.exs b/test/lightning/adaptors/supervisor_integration_test.exs index 47835a9f5b6..a20513fca2e 100644 --- a/test/lightning/adaptors/supervisor_integration_test.exs +++ b/test/lightning/adaptors/supervisor_integration_test.exs @@ -15,7 +15,7 @@ defmodule Lightning.Adaptors.SupervisorIntegrationTest do # Children with their *registered* names. We look up live PIDs by name # (Process.whereis/1) rather than by child id from which_children/1, # because module-based child specs share child ids like `Cachex` or - # `Lightning.Adaptors.Invalidator` — those don't carry the per-instance + # `Lightning.Adaptors.Invalidator`, which don't carry the per-instance # name we derive in the Supervisor. The Scheduler is registered via # `:global` (HighlanderPG-wrapped) so it needs a `:global.whereis_name/1` # lookup instead. @@ -85,15 +85,14 @@ defmodule Lightning.Adaptors.SupervisorIntegrationTest do "child pid #{inspect(child_pid)} is not alive" end) - # Locally-registered children are up under their derived names. Enum.each(local_named_children(sup), fn {role, registered_name} -> pid = Process.whereis(registered_name) assert is_pid(pid), "expected #{role} to be registered and alive" assert Process.alive?(pid) end) - # The HighlanderPG-wrapped Scheduler registers globally once it - # acquires the advisory lock — give it up to ~3s to do so. + # The HighlanderPG-wrapped Scheduler registers globally only once it + # acquires the advisory lock. assert_eventually(is_pid(scheduler_pid(sup)), @scheduler_wait_ms) assert Process.alive?(scheduler_pid(sup)) end @@ -113,7 +112,7 @@ defmodule Lightning.Adaptors.SupervisorIntegrationTest do # Two victims only: the supervisor's default max_restarts is 3 in 5s, # and hitting that ceiling would take the whole instance down for # reasons that have nothing to do with what we're asserting. `:tasks` - # is the interesting one — the Scheduler uses it on every tick, and + # is the interesting one. The Scheduler uses it on every tick, and # still doesn't need restarting alongside it. test "a sibling crash restarts only that sibling, leaving the Scheduler's leadership intact", %{sup: sup} do diff --git a/test/lightning/collections_test.exs b/test/lightning/collections_test.exs index 29d3a3f9ec8..15d19b24dcc 100644 --- a/test/lightning/collections_test.exs +++ b/test/lightning/collections_test.exs @@ -6,8 +6,7 @@ defmodule Lightning.CollectionsTest do alias Lightning.Collections.Item describe "Item.changeset/2 key width" do - # Width, not a null byte, so this is separate from the Collections jsonb - # work still outstanding. + # Key width only. Null bytes in item keys are not covered here. test "an over-long key is a changeset error, not a 22001" do collection = insert(:collection) diff --git a/test/lightning/credentials/credential_test.exs b/test/lightning/credentials/credential_test.exs index 926306062b1..095d218cd9f 100644 --- a/test/lightning/credentials/credential_test.exs +++ b/test/lightning/credentials/credential_test.exs @@ -73,7 +73,7 @@ defmodule Lightning.Credentials.CredentialTest do test "validates name format" do user = insert(:user) - # `@` and `#` are ordinary characters now. A control character is not. + # `@` and `#` are ordinary characters. A control character is not. changeset = Credential.changeset(%Credential{}, %{ name: "Invalid@Name#", @@ -98,8 +98,8 @@ defmodule Lightning.Credentials.CredentialTest do end test "an over-long schema is a changeset error, not a 500" do - # credentials.schema is varchar(40), not 255, so a 41 character schema - # was a 500 on plain ASCII through POST /api/credentials. + # The schema cap is 40, tighter than the name columns, so a 41 character + # schema on plain ASCII has to fail here and not in Postgres. user = insert(:user) changeset = @@ -142,7 +142,7 @@ defmodule Lightning.Credentials.CredentialTest do test "accepts the names the export fix exists for" do user = insert(:user) - # #2808's motivating example. Until #4577 a user could not create it. + # The names the YAML export has to quote. Creating them has to work too. for name <- ["MailChimp June'24", "Vérifier l'état", "患者確認", "step 🎉"] do changeset = Credential.changeset(%Credential{}, %{ diff --git a/test/lightning/credentials/schema_reconciler_test.exs b/test/lightning/credentials/schema_reconciler_test.exs index 716ba0305fd..d7aa5419f5b 100644 --- a/test/lightning/credentials/schema_reconciler_test.exs +++ b/test/lightning/credentials/schema_reconciler_test.exs @@ -1,7 +1,7 @@ defmodule Lightning.Credentials.SchemaReconcilerTest do - # async: false because: - # 1. DataCase uses shared sandbox mode (all processes access DB without allow/3) - # 2. isolated_adaptors stubs Config.default_instance/0 via Mimic globally + # async: false. DataCase then runs the sandbox in shared mode, so every + # process reaches the DB without allow/3, and isolated_adaptors stubs + # Config.default_instance/0 through Mimic for the whole VM. use Lightning.DataCase, async: false import Eventually diff --git a/test/lightning/export_utils/scalar_test.exs b/test/lightning/export_utils/scalar_test.exs index 59970157d54..944bca41064 100644 --- a/test/lightning/export_utils/scalar_test.exs +++ b/test/lightning/export_utils/scalar_test.exs @@ -3,7 +3,7 @@ defmodule Lightning.ExportUtils.ScalarTest do alias Lightning.ExportUtils.Scalar - # The shapes the export used to emit bare. Anything matching these has to + # The shapes the export has always emitted bare. Anything matching these has to # keep coming out byte for byte the same, otherwise every customer repo that # tracks a project spec picks up a diff. @old_value_regex ~r/\A[a-zA-Z0-9][a-zA-Z0-9_\-@\.> ]*[a-zA-Z0-9]\z/ @@ -139,7 +139,7 @@ defmodule Lightning.ExportUtils.ScalarTest do end describe "YAML typed lookalikes" do - # Narrowed in #4577 to what yamerl and yaml@2.7.1 actually resolve. + # Only what yamerl and the npm yaml parser actually resolve. @booleans_and_null ~w(true True TRUE false False FALSE null Null NULL ~) @integers ~w(0 7 08 2026 +5 -5 0x1F 0xff 0o17 007) @@ -148,8 +148,8 @@ defmodule Lightning.ExportUtils.ScalarTest do 1.0 0.5 .5 1e3 1E3 1.5e-3 -1.5 .inf .Inf .INF -.Inf +.inf .nan .NaN .NAN ) - # Neither parser resolves any of these, in either position, and every one - # was a legal job name under the charset rule this branch removed. + # Neither parser resolves any of these, in either position, so quoting + # them would only cost a diff. @not_typed ~w( y Y n N yes Yes YES no No NO on On ON off Off OFF 1_000 0b1010 1:30 0X1F 0O17 @@ -220,9 +220,9 @@ defmodule Lightning.ExportUtils.ScalarTest do describe "the trailing newline hole in the old regexes" do test "a trailing newline no longer slips through as a bare scalar" do - # The old regexes were anchored with ^ and $, and $ matches before a - # newline at the end of the subject, so a trailing newline was emitted - # bare and injected a blank line into the spec. + # A regex anchored with ^ and $ still matches before a newline at the end + # of the subject, so a body ending in one would go out bare and inject a + # blank line into the spec. assert Regex.match?( ~r/^[a-zA-Z0-9][a-zA-Z0-9_\-@\.> ]*[a-zA-Z0-9]$/, "workflow 1\n" @@ -385,7 +385,7 @@ defmodule Lightning.ExportUtils.ScalarTest do ] ++ byte_compat_corpus() end - # A deterministic sweep of the alphabet the old regexes allowed, so the byte + # A deterministic sweep of the alphabet the bare shapes allow, so the byte # compatibility claim is checked against more than a handful of examples. defp byte_compat_corpus do alphanumeric = Enum.concat([?a..?z, ?A..?Z, ?0..?9]) @@ -459,8 +459,8 @@ defmodule Lightning.ExportUtils.ScalarTest do describe "typed?/1 corpus (what the parsers actually resolve)" do # Every entry here was measured against both parsers this project ships - # against, as a value and as a key: yamerl through YamlElixir, and - # yaml@2.7.1 in assets/node_modules. Do not add to @resolved without + # against, as a value and as a key: yamerl through YamlElixir, and the + # yaml package in assets/node_modules. Do not add to @resolved without # running the string through both first, and do not move anything out of # @plain without doing the same. Over-quoting is not free: every one of # these is a legal name, and a quoting change is a diff in every synced @@ -583,9 +583,9 @@ defmodule Lightning.ExportUtils.ScalarTest do describe "encode_block/2" do # The corpus below was round-tripped through both parsers this project - # ships against, yamerl and yaml@2.7.1. Before #4577 the leading-space, CR - # and multiple-trailing-newline rows all lost data or failed to parse - # (issue #2966). + # ships against, yamerl and the npm yaml package. The leading-space, CR and + # multiple-trailing-newline rows are the ones that lose data or fail to + # parse when the indicators are chosen wrongly. @block_cases [ {"plain", "fn(state => state)"}, {"two lines", "line1\nline2"}, @@ -600,8 +600,8 @@ defmodule Lightning.ExportUtils.ScalarTest do {"empty", ""}, {"two trailing newlines", "a\n\n"}, # Crossed shapes. The indicator and the chomping indicator are chosen - # independently, so a body that needs both used to get only the first - # and lose the newlines the second exists to keep. + # independently. A body that needs both must not get only the first and + # lose the newlines the second exists to keep. {"leading space and two trailing", " indented\nnext\n\n"}, {"leading space and three trailing", " x\n\n\n"}, {"leading tab and two trailing", "\tx\n\n"}, @@ -686,8 +686,7 @@ defmodule Lightning.ExportUtils.ScalarTest do # test/fixtures/block_scalars.json is the one corpus both parsers see. This # half pins the encoder output and checks yamerl; assets/test/yaml/ # blockScalars.test.ts parses the same documents with the npm parser, which - # no Elixir test can run and which disagreed with yamerl on the `|2` shape - # until #4577. + # no Elixir test can run. @fixture "test/fixtures/block_scalars.json" setup do diff --git a/test/lightning/export_utils_test.exs b/test/lightning/export_utils_test.exs index 34f55ecb0c2..fd87e7afc9c 100644 --- a/test/lightning/export_utils_test.exs +++ b/test/lightning/export_utils_test.exs @@ -8,10 +8,10 @@ defmodule Lightning.ExportUtilsTest do @fixture "test/fixtures/unicode_project.yaml" - # The names below are the ones YAML gets wrong if we concatenate strings - # without quoting: an apostrophe closes a single quoted scalar, `off` and a - # bare date come back as a boolean and a date, and accented or CJK text falls - # outside the character class the export used to test against. + # YAML gets these names wrong if we concatenate strings without quoting. An + # apostrophe closes a single quoted scalar, and `off` and a bare date come + # back as a boolean and a date. The accented and CJK names have to survive + # untouched. @workflow_one "Flujo 1: Registro en PS y gestión de perfiles" @workflow_two "off" @workflow_three "MailChimp June'24" @@ -165,8 +165,8 @@ defmodule Lightning.ExportUtilsTest do end test "two workflows that hyphenate to the same key are refused" do - # This used to be a silent Map.put overwrite: the project exported - # cleanly and came back with one workflow instead of two. + # Without this check the second workflow silently overwrites the first. + # The project exports cleanly and comes back with one workflow, not two. project = insert(:project, name: "workflow-key-collision", @@ -383,10 +383,9 @@ defmodule Lightning.ExportUtilsTest do describe "hyphenate/1 parity with the client" do # ExportUtils.hyphenate/1 replaces each single space and leaves every other - # whitespace character alone. The JS half pins this in - # assets/test/yaml/util.test.ts; this side had nothing, so widening the - # server back to ~r/\s+/ left the whole Elixir suite green while the two - # ends silently disagreed about what a job's key is. + # whitespace character alone. assets/test/yaml/util.test.ts pins the same + # rule on the client. Both halves are needed, or one side can widen to + # ~r/\s+/ and stay green while the two ends disagree about a job's key. test "two spaces give two hyphens, and other whitespace is left alone" do trigger = build(:trigger, type: :webhook, enabled: true) @@ -428,9 +427,9 @@ defmodule Lightning.ExportUtilsTest do describe "the edge key disambiguation fixture" do # test/fixtures/edge_key_disambiguation.json is the one corpus both sides - # see. This half pins the server; assets/test/yaml/edgeKeys.test.ts asserts - # the browser produces the same keys. Only the JS half read it at first, - # which let the server drift and the fixture go stale in silence. + # see. This half pins the server and assets/test/yaml/edgeKeys.test.ts + # asserts the browser produces the same keys. Both halves have to read it, + # or one side drifts and the fixture goes stale in silence. @edge_fixture "test/fixtures/edge_key_disambiguation.json" test "the server still produces exactly the keys in it" do diff --git a/test/lightning/projects/provisioner_test.exs b/test/lightning/projects/provisioner_test.exs index ccf96abd3e0..0f44f9d32ad 100644 --- a/test/lightning/projects/provisioner_test.exs +++ b/test/lightning/projects/provisioner_test.exs @@ -229,9 +229,9 @@ defmodule Lightning.Projects.ProvisionerTest do test "a control character in a workflow name is a changeset error, not a 500" do # This path builds its own changeset and calls Workflow.validate/1 - # directly, so it used to skip the name rule entirely. A NUL reached - # Postgres and came back as a 22021 that action_fallback does not handle, - # which is a 500 on POST /api/provision (#4893). + # directly, so the name rule has to run there too. Otherwise a NUL + # reaches Postgres and comes back as a 22021 that action_fallback does + # not handle, which is a 500 on POST /api/provision. for name <- [ "before\u{0000}after", "tab\u{0009}here", diff --git a/test/lightning/sandboxes_test.exs b/test/lightning/sandboxes_test.exs index fabdc19567b..a0da2db493e 100644 --- a/test/lightning/sandboxes_test.exs +++ b/test/lightning/sandboxes_test.exs @@ -1048,8 +1048,8 @@ defmodule Lightning.Projects.SandboxesTest do end test "deletion-shaped merge options passed by a caller are ignored" do - # A stale or crafted caller might still send the old deletion options; - # the merge must keep the collection regardless, even for an owner. + # A stale or crafted caller might still send deletion options. The merge + # must keep the collection regardless, even for an owner. assert "target-only" in merge_target_only_collection( :owner, fn collection -> diff --git a/test/lightning/utils/validators_test.exs b/test/lightning/utils/validators_test.exs index 44c9fca6d91..6cd3e847e1b 100644 --- a/test/lightning/utils/validators_test.exs +++ b/test/lightning/utils/validators_test.exs @@ -111,8 +111,8 @@ defmodule Lightning.ValidatorsTest do end test "a run of them is caught, not just one" do - # A per-grapheme check used to fuse a joiner-led run into one cluster and - # miss it. + # A per-grapheme check fuses a joiner-led run into one cluster and misses + # it. assert Validators.invisible_only?("\u{200D}\u{200D}") assert Validators.invisible_only?("\u{200B}\u{FEFF}\u{00AD}\u{FE0F}") assert Validators.invisible_only?(String.duplicate("\u{200D}", 20)) diff --git a/test/lightning/version_control_test.exs b/test/lightning/version_control_test.exs index c7b2434e91e..658922ffe65 100644 --- a/test/lightning/version_control_test.exs +++ b/test/lightning/version_control_test.exs @@ -569,7 +569,7 @@ defmodule Lightning.VersionControlTest do repo_connection: repo_connection } do # The pre-flight generates the spec and throws it away, so a project - # without a collision has to reach GitHub exactly as before. + # without a collision has to reach GitHub as normal. expect_create_installation_token(repo_connection.github_installation_id) expect_get_repo(repo_connection.repo) expect_create_workflow_dispatch(repo_connection.repo, "openfn-pull.yml") diff --git a/test/lightning/workflow_templates_test.exs b/test/lightning/workflow_templates_test.exs index 3839084ea04..f66434535c4 100644 --- a/test/lightning/workflow_templates_test.exs +++ b/test/lightning/workflow_templates_test.exs @@ -26,7 +26,7 @@ defmodule Lightning.WorkflowTemplatesTest do test "a NUL in positions is a changeset error, not a 22P05" do # positions is a jsonb map, and Postgres refuses a NUL anywhere inside - # jsonb, keys included (#4893). + # jsonb, keys included. for positions <- [ %{"node\u{0000}id" => %{"x" => 1}}, %{"node" => %{"label" => "a\u{0000}b"}} diff --git a/test/lightning/workflows/edge_test.exs b/test/lightning/workflows/edge_test.exs index 3735559f887..76aec97a7b5 100644 --- a/test/lightning/workflows/edge_test.exs +++ b/test/lightning/workflows/edge_test.exs @@ -20,8 +20,8 @@ defmodule Lightning.Workflows.EdgeTest do end test "a control character in the label is rejected on every condition type" do - # This check used to sit inside the :js_expression branch, so an :always - # edge could carry a NUL in its label straight into the snapshot jsonb. + # An :always edge carries its label into the snapshot jsonb just like a + # :js_expression one, so the check cannot live in that branch. for condition_type <- [ :always, :on_job_success, @@ -72,9 +72,8 @@ defmodule Lightning.Workflows.EdgeTest do test "a NUL in the expression is rejected on every condition type" do # cast/3 accepts an expression whatever the condition type is, and it - # reaches the snapshot jsonb either way. This check used to be reachable - # only through the :js_expression branch, and even there it sat behind a - # `valid?: false` short circuit, so it never actually ran. + # reaches the snapshot jsonb either way, so the check cannot live in the + # :js_expression branch or behind a `valid?` short circuit. for condition_type <- [ :always, :on_job_success, @@ -151,8 +150,8 @@ defmodule Lightning.Workflows.EdgeTest do test "the checks run even when the changeset is invalid for other reasons" do # A minimal changeset is invalid for unrelated missing fields on plenty - # of real save paths. Skipping the jsonb checks there is how the hole - # stayed open. + # of real save paths, so the jsonb checks have to run on an invalid + # changeset too. changeset = Edge.changeset(%Edge{}, %{ condition_type: :always, @@ -463,9 +462,8 @@ defmodule Lightning.Workflows.EdgeTest do } ) - # Asserted by field rather than as an ordered list: the label check moved - # out of the :js_expression branch so it runs after the expression one, - # and the order of changeset.errors is not what this test is about. + # Asserted by field rather than as an ordered list. The order of + # changeset.errors is not what this test is about. errors = errors_on(changeset) assert errors[:condition_expression] == [ diff --git a/test/lightning/workflows/job_test.exs b/test/lightning/workflows/job_test.exs index d144c5d5d42..3965b114ad7 100644 --- a/test/lightning/workflows/job_test.exs +++ b/test/lightning/workflows/job_test.exs @@ -340,7 +340,7 @@ defmodule Lightning.Workflows.JobTest do test "consecutive joiners do not slip past the blank check" do # String.graphemes/1 fuses a ZWJ-led run into one cluster, so a - # per-grapheme check caught one joiner and missed two. + # per-grapheme check would catch one joiner and miss two. for name <- [ "\u{200D}\u{200D}", "\u{200D}\u{200D}\u{200D}", @@ -363,8 +363,7 @@ defmodule Lightning.Workflows.JobTest do end test "a body containing a NUL is a changeset error, not a jsonb crash" do - # Only the NUL: a body is code and legitimately holds newlines and tabs - # (#4893). + # Only the NUL. A body is code and legitimately holds newlines and tabs. errors = Job.changeset(%Job{}, %{ name: "step", @@ -444,9 +443,9 @@ defmodule Lightning.Workflows.JobTest do end test "the name is trimmed before it is validated, not after" do - # 100 characters plus trailing space. Trimming after validation, which is - # what this changeset used to do, made this 105 characters and rejected - # a name that is exactly at the cap. + # 100 characters plus trailing space. The changeset has to trim before it + # measures, or this is 105 characters and a name exactly at the cap is + # rejected. name = String.duplicate("a", 100) <> " " changeset = Job.changeset(%Job{}, %{name: name}) diff --git a/test/lightning/workflows/trigger_test.exs b/test/lightning/workflows/trigger_test.exs index 15dfce8ed64..7f305ee3f21 100644 --- a/test/lightning/workflows/trigger_test.exs +++ b/test/lightning/workflows/trigger_test.exs @@ -5,7 +5,7 @@ defmodule Lightning.Workflows.TriggerTest do describe "jsonb-bound trigger fields" do test "a NUL in a comment is a changeset error" do - # Both are copied into the workflow_snapshots.triggers jsonb (#4893). + # Both are copied into the workflow_snapshots.triggers jsonb. for {field, message} <- [ {:comment, "comment can't contain a null byte"} ] do @@ -21,8 +21,8 @@ defmodule Lightning.Workflows.TriggerTest do end test "an over-long comment is a changeset error, not a 22001" do - # Both columns are varchar(255) and neither had a length guard, so a 300 - # character comment gave valid? == true and then raised on insert. + # Both columns are varchar(255). Without a length guard a 300 character + # comment is valid? == true and then raises on insert. for {field, message} <- [ {:comment, "comment is too long, please use a shorter one"} ] do @@ -38,9 +38,9 @@ defmodule Lightning.Workflows.TriggerTest do end test "an over-long cron_expression is a changeset error, not a 22001" do - # The third field on the same cast/3, same varchar(255), and the only one - # that had no guard. Crontab parses this happily, so the changeset said - # valid? and the insert raised. Reachable through POST /api/provision. + # Same varchar(255). Crontab parses an over-long expression happily, so + # without this guard the changeset says valid? and the insert raises. + # Reachable through POST /api/provision. expression = "*/1 " <> String.duplicate("1,", 130) <> "1 * * *" assert String.length(expression) > 255 diff --git a/test/lightning/workflows/workflow_test.exs b/test/lightning/workflows/workflow_test.exs index c5c445726b5..f7eb00d86e6 100644 --- a/test/lightning/workflows/workflow_test.exs +++ b/test/lightning/workflows/workflow_test.exs @@ -191,8 +191,8 @@ defmodule Lightning.Workflows.WorkflowTest do test "a name past the width of the column is a changeset error, not a 500", %{project: project} do - # This used to reach the database and come back as Postgrex 22001 - # (string_data_right_truncation), which the user saw as a 500. + # Without the guard this reaches the database and comes back as Postgrex + # 22001 (string_data_right_truncation), which the user sees as a 500. name = String.duplicate("a", 300) changeset = @@ -281,7 +281,7 @@ defmodule Lightning.Workflows.WorkflowTest do project: project } do # positions goes straight into the workflow_snapshots.positions jsonb, - # keys and all, and Postgres refuses a NUL anywhere inside jsonb (#4893). + # keys and all, and Postgres refuses a NUL anywhere inside jsonb. for positions <- [ %{"node\u{0000}id" => %{"x" => 1, "y" => 2}}, %{"node" => %{"x" => 1, "label" => "a\u{0000}b"}}, diff --git a/test/lightning/workflows_test.exs b/test/lightning/workflows_test.exs index a5d6dab7feb..bdd63c6513a 100644 --- a/test/lightning/workflows_test.exs +++ b/test/lightning/workflows_test.exs @@ -23,8 +23,8 @@ defmodule Lightning.WorkflowsTest do describe "soft delete with a name at the column width" do test "a 255 character name is deleted without raising 22001" do # The _del suffix is appended after every validation has run, so a name - # already at the column width used to raise a bare Postgrex error out of - # the dashboard delete button (#4577). + # already at the column width would raise a bare Postgrex error out of + # the dashboard delete button. project = insert(:project) user = insert(:user) name = String.duplicate("a", 255) diff --git a/test/lightning_web/channels/ai_assistant_channel_test.exs b/test/lightning_web/channels/ai_assistant_channel_test.exs index 1f193cb6e50..b39a1167297 100644 --- a/test/lightning_web/channels/ai_assistant_channel_test.exs +++ b/test/lightning_web/channels/ai_assistant_channel_test.exs @@ -352,7 +352,8 @@ defmodule LightningWeb.AiAssistantChannelTest do assert %{"content" => [message]} = errors assert message =~ "should be at most 10000 character(s)" - # Shown to the reader as it stands, so it cannot be a code. + # The reason is shown to the user verbatim, so it is a message rather + # than an error code. assert reason =~ "should be at most 10000 character(s)" end end diff --git a/test/lightning_web/channels/workflow_channel_test.exs b/test/lightning_web/channels/workflow_channel_test.exs index 238c9f9856c..1912155e8d1 100644 --- a/test/lightning_web/channels/workflow_channel_test.exs +++ b/test/lightning_web/channels/workflow_channel_test.exs @@ -3275,8 +3275,8 @@ defmodule LightningWeb.WorkflowChannelTest do workflow: workflow, sup: _sup } do - # Global mode: the refresh runs in a Task owned by the isolated - # instance's Scheduler. + # The refresh runs in a Task owned by the Scheduler, outside this test's + # caller chain, so the stub has to be global. Lightning.Adaptors.Catalogue.delete_all_for_source(:npm) Mox.set_mox_global(Lightning.Adaptors.StrategyMock) @@ -4105,7 +4105,7 @@ defmodule LightningWeb.WorkflowChannelTest do ) # The export refuses rather than dropping one of the pair, and it refuses - # before any GitHub call. No GitHub mocks are set on purpose: verify_on_exit! + # before any GitHub call. No GitHub mocks are set on purpose. verify_on_exit! # turns a dispatch into a failure, so this also asserts we never fired one. for name <- ["My Flow", "My-Flow"] do {:ok, _} = diff --git a/test/lightning_web/controllers/adaptor_icon_controller_test.exs b/test/lightning_web/controllers/adaptor_icon_controller_test.exs index 04f3964f961..f6f9c54522a 100644 --- a/test/lightning_web/controllers/adaptor_icon_controller_test.exs +++ b/test/lightning_web/controllers/adaptor_icon_controller_test.exs @@ -1,5 +1,7 @@ defmodule LightningWeb.AdaptorIconControllerTest do - # async: false — all tests share the Lightning.Adaptors supervisor name. + # Every test goes through the application's Lightning.Adaptors supervisor, + # which runs with Lightning.Adaptors.StrategyMock in test (config/test.exs). + # Sharing that one name is why this module is not async. use LightningWeb.ConnCase, async: false import Mox @@ -12,11 +14,6 @@ defmodule LightningWeb.AdaptorIconControllerTest do setup :verify_on_exit! - # The production `Lightning.Adaptors.Supervisor` is started in - # `application.ex` under the name `Lightning.Adaptors` and — in test — - # uses `Lightning.Adaptors.StrategyMock` (see `config/test.exs`). No - # per-test supervisor start is needed. - defp sha8_from_bytes(bytes) do :crypto.hash(:sha256, bytes) |> binary_part(0, 4) @@ -325,7 +322,8 @@ defmodule LightningWeb.AdaptorIconControllerTest do assert result.status == 302 [location] = get_resp_header(result, "location") - # sha8 segment is lowercase; percent-encoded chars use uppercase hex per RFC 3986 + # Only the sha8 segment is asserted lowercase. Percent-encoded characters + # in the name are uppercase hex per RFC 3986. assert location =~ "square-#{current_sha8}.png" end @@ -461,9 +459,8 @@ defmodule LightningWeb.AdaptorIconControllerTest do %{ conn: conn } do - # Adaptor row exists but has no icon for the square shape. - # Even though there's a stale sha8 in the URL, there's no canonical - # URL to redirect to — 404 instead of 302. + # With no stored icon there is no canonical URL to redirect to, so 404 + # rather than 302. name = unique_adaptor_name() insert_adaptor(name) @@ -480,8 +477,6 @@ defmodule LightningWeb.AdaptorIconControllerTest do end end - # The tests above call the controller directly; these confirm the route - # and pipeline wire up to it too. describe "GET /adaptors/icons/... (full router pipeline)" do test "200 on sha match", %{conn: conn} do name = unique_adaptor_name() @@ -573,7 +568,8 @@ defmodule LightningWeb.AdaptorIconControllerTest do url = AdaptorIconURL.build("@openfn/language-http", meta, :square) - # sha8 is lowercase; percent-encoded chars use uppercase hex per RFC 3986 + # Only the sha8 segment is asserted lowercase. Percent-encoded characters + # in the name are uppercase hex per RFC 3986. assert url =~ "square-#{expected_sha8}.png" assert expected_sha8 == String.downcase(expected_sha8) end diff --git a/test/lightning_web/live/components/common_test.exs b/test/lightning_web/live/components/common_test.exs index 153ccc668c9..b90830a74a8 100644 --- a/test/lightning_web/live/components/common_test.exs +++ b/test/lightning_web/live/components/common_test.exs @@ -4,11 +4,11 @@ defmodule LightningWeb.Components.CommonTest do import Phoenix.LiveViewTest describe "wrapper_tooltip/1 and HTML" do - # The hook used to read aria-label off the DOM *property*, which undoes the - # escaping HEEx applied to the attribute, and hand the result to tippy with - # allowHTML on, so a workflow name holding markup became live elements for - # anyone viewing the project (#4577). The content now goes in a