ci: give CI an oplog, so the HA tests stop being skipped in silence

.github/scripts/mongo-rs-init.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+#
+# FILE            mongo-rs-init.sh
+#
+# Copyright 2026 Seamware
+# SPDX-License-Identifier: Apache-2.0
+#
+# Initiate the single-node replica set that the HA tests need, and wait until it
+# is PRIMARY.
+#
+# WHY A REPLICA SET AT ALL: the HA cache sync (--high-availability mongo) rides on
+# a mongo CHANGE STREAM, and a change stream reads the oplog - which a standalone
+# mongod does not have. corTestParams.sh probes isMaster.setName and, on a
+# standalone, ha_cache_sync.test simply leaves the run set. SILENTLY: nothing in
+# the log says the HA paths went unexercised, which is how CI came to report 108
+# lines and 10 functions of the broker as untested code rather than as an
+# environment nobody had stood up. See doc/coverage.md.
+#
+# WHY IT IS NOT IN THE IMAGE: quay.io/seamware/mongo-rs starts mongod with
+# --replSet, and that is all it can do. A member is addressed by the name its
+# CLIENTS use, and the image cannot know that name - here it is the service
+# alias, on a workstation it is localhost. So the set is initiated from the
+# consumer side, which is this.
+#
+# WHY IT IS A SCRIPT: three jobs need it - ci.yml's functest matrix and the
+# nightly's coverage and valgrind jobs - and three copies of a retry loop that
+# must agree is three copies that will not.
+#
+# Idempotent on purpose. A re-run against a service container that is already
+# initiated must be a no-op, not an AlreadyInitialized failure.
+#
+set -eu
+
+host=${COR_MONGO_HOST:-mongo}
+port=${COR_MONGO_PORT:-27017}
+
+mongo() { mongosh --host "$host" --port "$port" --quiet --eval "$1"; }
+
+#
+# mongod first, then the set. Nothing else in these jobs waits for mongo - the
+# build is long enough that it has always been up by the time the suite starts -
+# so this is where the wait lives now.
+#
+i=0
+while [ "$i" -lt 60 ]; do
+  mongo 'db.runCommand({ping: 1}).ok' >/dev/null 2>&1 && break
+  i=$((i + 1))
+  sleep 2
+done
+[ "$i" -lt 60 ] || { echo "mongod at $host:$port never answered"; exit 1; }
+
+if [ "$(mongo 'try { rs.status().set } catch (e) { "" }')" = "" ]; then
+  echo "initiating replica set rs0 with a single member at $host:$port"
+  mongo "rs.initiate({_id: 'rs0', members: [{_id: 0, host: '$host:$port'}]})" >/dev/null
+else
+  echo "replica set already initiated"
+fi
+
+#
+# Initiated is not the same as usable: an election takes a moment, and a write
+# before it lands gets NotWritablePrimary. The suite would see that as a broker
+# bug.
+#
+i=0
+while [ "$i" -lt 60 ]; do
+  [ "$(mongo 'db.adminCommand({isMaster: 1}).ismaster')" = "true" ] && break
+  i=$((i + 1))
+  sleep 1
+done
+[ "$i" -lt 60 ] || { echo "the set never elected a primary"; exit 1; }
+
+#
+# The assertion that matters, because it is the exact probe the harness makes:
+# a setName is what puts the HA tests in the run set.
+#
+name=$(mongo 'db.adminCommand({isMaster: 1}).setName')
+[ "$name" = "rs0" ] || { echo "setName is '$name', expected rs0"; exit 1; }
+echo "replica set rs0 is PRIMARY - the HA tests are in the run set"

.github/workflows/ci.yml

@@ -186,8 +186,18 @@ jobs:
       image: quay.io/seamware/coraine-ci:2026-08-24-814ad08
 
     services:
+      #
+      # A REPLICA SET, not a standalone. The HA cache sync rides on a mongo change
+      # stream, a change stream reads the oplog, and a standalone mongod has none -
+      # so corTestParams.sh detects the environment and ha_cache_sync.test silently
+      # leaves the run set on a standalone, which is what happened here from the day
+      # this workflow was written. `services:` passes docker-create OPTIONS but not
+      # a COMMAND, so --replSet cannot be added from this file: hence an image whose
+      # entire content is that command line. The set still has to be INITIATED - see
+      # the step below.
+      #
       mongo:
-        image: mongo:8.0
+        image: quay.io/seamware/mongo-rs:2026-09-02-c1b6cba
         ports:
           - 27017:27017
 
@@ -260,19 +270,21 @@ jobs:
           mkdir -p /opt/seamware/plugins /opt/seamware/etc
           make di
 
-      - name: Wait for the databases
+      #
+      # The mongo half of the old wait now lives in the script, because initiating
+      # the set has to wait for mongod anyway and two waits would only disagree.
+      #
+      - name: Initiate the mongo replica set
+        run: stack/coraine/.github/scripts/mongo-rs-init.sh
+
+      - name: Wait for timescale
         run: |
-          for i in $(seq 1 30); do
-            mongosh --host mongo --quiet --eval 'db.runCommand({ping:1}).ok' >/dev/null 2>&1 && break
-            [ "$i" = 30 ] && { echo "mongo never became reachable"; exit 1; }
-            sleep 2
-          done
           for i in $(seq 1 30); do
             psql -h timescale -U postgres -c 'SELECT 1' >/dev/null 2>&1 && break
             [ "$i" = 30 ] && { echo "timescale never became reachable"; exit 1; }
             sleep 2
           done
-          echo "both databases are up"
+          echo "timescale is up"
 
       #
       # 36 tests hardcode http://localhost:7080/jsonldContexts/... and assert that

.github/workflows/nightly.yml

@@ -65,8 +65,17 @@ jobs:
     container:
       image: quay.io/seamware/coraine-ci-nightly:2026-08-26-dd3bf56
     services:
+      #
+      # A REPLICA SET - the HA cache sync watches a change stream, a change stream
+      # reads the oplog, and a standalone mongod has none. On a standalone
+      # ha_cache_sync.test leaves the run set SILENTLY, which is why the coverage
+      # figure reported the HA paths as untested code for weeks. `services:` passes
+      # docker-create options but not a command, so --replSet cannot be set from
+      # here - hence an image whose whole content is that command. The set is
+      # initiated by a step; see .github/scripts/mongo-rs-init.sh.
+      #
       mongo:
-        image: mongo:8.0
+        image: quay.io/seamware/mongo-rs:2026-09-02-c1b6cba
         ports: ['27017:27017']
       timescale:
         image: timescale/timescaledb-ha:pg16
@@ -125,6 +134,13 @@ jobs:
       # "report generated" instead. Tee it, so the numbers reach both the log and
       # the run summary - which is what the comment above this job always claimed.
       #
+      #
+      # The set is initiated here rather than in the image: a member is addressed by
+      # the name its clients use, and the image cannot know that name.
+      #
+      - name: Initiate the mongo replica set
+        run: stack/coraine/.github/scripts/mongo-rs-init.sh
+
       - name: Coverage - mongoc
         working-directory: stack/coraine
         run: |
@@ -288,8 +304,17 @@ jobs:
     container:
       image: quay.io/seamware/coraine-ci-nightly:2026-08-26-dd3bf56
     services:
+      #
+      # A REPLICA SET - the HA cache sync watches a change stream, a change stream
+      # reads the oplog, and a standalone mongod has none. On a standalone
+      # ha_cache_sync.test leaves the run set SILENTLY, which is why the coverage
+      # figure reported the HA paths as untested code for weeks. `services:` passes
+      # docker-create options but not a command, so --replSet cannot be set from
+      # here - hence an image whose whole content is that command. The set is
+      # initiated by a step; see .github/scripts/mongo-rs-init.sh.
+      #
       mongo:
-        image: mongo:8.0
+        image: quay.io/seamware/mongo-rs:2026-09-02-c1b6cba
         ports: ['27017:27017']
       timescale:
         image: timescale/timescaledb-ha:pg16
@@ -325,6 +350,13 @@ jobs:
         run: |
           mkdir -p /opt/seamware/plugins /opt/seamware/etc
           make di
+      #
+      # The set is initiated here rather than in the image: a member is addressed by
+      # the name its clients use, and the image cannot know that name.
+      #
+      - name: Initiate the mongo replica set
+        run: stack/coraine/.github/scripts/mongo-rs-init.sh
+
       - name: Suite under valgrind - tests ${{ matrix.shard.name }}
         working-directory: stack/coraine
         run: |
@@ -376,6 +408,10 @@ jobs:
     container:
       image: quay.io/seamware/coraine-ci-nightly:2026-08-26-dd3bf56
     services:
+      #
+      # Standalone: the TP suite has no HA test, so there is nothing here that needs
+      # a change stream, and an oplog would only be overhead.
+      #
       mongo:
         image: mongo:8.0
         ports: ['27017:27017']
@@ -560,6 +596,12 @@ jobs:
     container:
       image: quay.io/seamware/coraine-ci-nightly:2026-08-26-dd3bf56
     services:
+      #
+      # STANDALONE on purpose, unlike the coverage and valgrind jobs above. This job
+      # compares against RECORDED HISTORY, and a replica set puts every write through
+      # an oplog - the baseline would move under it and the comparison would be
+      # measuring the change of database topology, not the change in the broker.
+      #
       mongo:
         image: mongo:8.0
         ports: ['27017:27017']