[minor] Add off-host backup and restore proof

.github/workflows/cloud-smoke-cleanup.yml

@@ -19,6 +19,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - name: DigitalOcean ISLE
+            kind: app
+            provider: digitalocean
+            template: isle
+            environment: cloud-smoke-cleanup-digitalocean
+            concurrency_group: cloud-compose-smoke-digitalocean-isle
           - name: DigitalOcean WordPress
             kind: app
             provider: digitalocean
@@ -93,7 +99,10 @@ jobs:
       CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.event.workflow_run.id }}
       GCLOUD_OIDC_POOL: ${{ vars.GCLOUD_OIDC_POOL || secrets.GCLOUD_OIDC_POOL }}
       GCLOUD_PROJECT: ${{ vars.GCLOUD_PROJECT || secrets.GCLOUD_PROJECT }}
+      GCLOUD_FRESH_REGION: ${{ vars.GCLOUD_FRESH_REGION || secrets.GCLOUD_FRESH_REGION }}
+      GCLOUD_FRESH_ZONE: ${{ vars.GCLOUD_FRESH_ZONE || secrets.GCLOUD_FRESH_ZONE }}
       GCLOUD_REGION: ${{ vars.GCLOUD_REGION || secrets.GCLOUD_REGION }}
+      GCLOUD_ZONE: ${{ vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }}
       GSA: ${{ vars.GSA || secrets.GSA }}
     steps:
       - name: Checkout repository
@@ -127,5 +136,12 @@ jobs:
       - name: Install gcloud
         uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3
 
-      - name: Sweep GCP smoke resources
+      - name: Sweep GCP upgrade-region resources
+        run: ci/cloud-smoke.sh sweep-gcp-wp
+
+      - name: Sweep GCP fresh-smoke region
+        if: ${{ env.GCLOUD_FRESH_REGION != '' && env.GCLOUD_FRESH_REGION != env.GCLOUD_REGION }}
+        env:
+          GCLOUD_REGION: ${{ env.GCLOUD_FRESH_REGION }}
+          GCLOUD_ZONE: ${{ env.GCLOUD_FRESH_ZONE }}
         run: ci/cloud-smoke.sh sweep-gcp-wp

.github/workflows/cloud-smoke.yml

@@ -78,7 +78,9 @@ jobs:
       cancel-in-progress: false
     env:
       CLOUD_COMPOSE_SMOKE_AUTO_APPROVE: "true"
-      CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "1800"
+      CLOUD_COMPOSE_SMOKE_BOOT_TIMEOUT: "900"
+      CLOUD_COMPOSE_SMOKE_CONFIG_MANAGEMENT_TIMEOUT: "2400"
+      CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "900"
       CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS: "true"
       CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }}
     steps:
@@ -103,12 +105,14 @@ jobs:
           terraform_wrapper: false
 
       - name: Run Linode config-management smoke test
+        timeout-minutes: 100
         env:
           LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
         run: make config-management-cloud-smoke METHOD=${{ matrix.method }}
 
       - name: Destroy Linode config-management smoke resources
         if: always()
+        timeout-minutes: 20
         env:
           LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
         run: ci/config-management-cloud-smoke.sh destroy-${{ matrix.method }}-drupal
@@ -126,21 +130,30 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - name: DigitalOcean ISLE
+            provider: digitalocean
+            template: isle
+            # Exceed the 90-minute application service ceiling with enough
+            # room to observe the terminal unit state before cleanup.
+            boot_timeout: "6000"
           - name: DigitalOcean WordPress
             provider: digitalocean
             template: wp
+            boot_timeout: "1200"
           - name: Linode WordPress
             provider: linode
             template: wp
+            boot_timeout: "1200"
     concurrency:
       group: cloud-compose-smoke-${{ matrix.provider }}-${{ matrix.template }}
       cancel-in-progress: false
     env:
       CLOUD_COMPOSE_SMOKE_AUTO_APPROVE: "true"
+      CLOUD_COMPOSE_SMOKE_BOOT_TIMEOUT: ${{ matrix.boot_timeout }}
       CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "1800"
       CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS: "true"
       CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }}
-      CLOUD_COMPOSE_SOURCE_REF: ${{ github.event.pull_request.head.sha }}
+      CLOUD_COMPOSE_SOURCE_REF: ${{ github.sha }}
 
     steps:
       - name: Checkout repository
@@ -199,12 +212,15 @@ jobs:
       CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "1800"
       CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS: "true"
       CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }}
-      CLOUD_COMPOSE_SOURCE_REF: ${{ github.event.pull_request.head.sha }}
+      CLOUD_COMPOSE_SOURCE_REF: ${{ github.sha }}
       CLOUD_COMPOSE_UPGRADE_BASE_SHA: f33117cdbbf4a9c7d59006a4db986baef118e6bb
       CLOUD_COMPOSE_UPGRADE_CURRENT_REF: ${{ github.sha }}
       GCLOUD_OIDC_POOL: ${{ vars.GCLOUD_OIDC_POOL || secrets.GCLOUD_OIDC_POOL }}
       GCLOUD_PROJECT: ${{ vars.GCLOUD_PROJECT || secrets.GCLOUD_PROJECT }}
+      GCLOUD_FRESH_REGION: ${{ vars.GCLOUD_FRESH_REGION || secrets.GCLOUD_FRESH_REGION }}
+      GCLOUD_FRESH_ZONE: ${{ vars.GCLOUD_FRESH_ZONE || secrets.GCLOUD_FRESH_ZONE }}
       GCLOUD_REGION: ${{ vars.GCLOUD_REGION || secrets.GCLOUD_REGION }}
+      GCLOUD_ZONE: ${{ vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }}
       GCLOUD_NETWORK_PROJECT_ID: ${{ vars.GCLOUD_NETWORK_PROJECT_ID || secrets.GCLOUD_NETWORK_PROJECT_ID }}
       GCLOUD_NETWORK_NAME: ${{ vars.GCLOUD_NETWORK_NAME || secrets.GCLOUD_NETWORK_NAME }}
       GCLOUD_SUBNETWORK_NAME: ${{ vars.GCLOUD_SUBNETWORK_NAME || secrets.GCLOUD_SUBNETWORK_NAME }}
@@ -264,6 +280,9 @@ jobs:
 
       - name: Run fresh smoke test
         if: ${{ !startsWith(github.event.pull_request.title, '[major]') }}
+        env:
+          GCLOUD_REGION: ${{ vars.GCLOUD_FRESH_REGION || secrets.GCLOUD_FRESH_REGION || vars.GCLOUD_REGION || secrets.GCLOUD_REGION }}
+          GCLOUD_ZONE: ${{ vars.GCLOUD_FRESH_ZONE || secrets.GCLOUD_FRESH_ZONE || vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }}
         run: make smoke-test PROVIDER=gcp TEMPLATE=wp
 
       - name: Run 0.10.2 upgrade smoke test
@@ -272,6 +291,9 @@ jobs:
 
       - name: Destroy fresh smoke resources
         if: ${{ always() && !startsWith(github.event.pull_request.title, '[major]') && env.GCLOUD_OIDC_POOL != '' && env.GSA != '' && env.GCLOUD_PROJECT != '' }}
+        env:
+          GCLOUD_REGION: ${{ vars.GCLOUD_FRESH_REGION || secrets.GCLOUD_FRESH_REGION || vars.GCLOUD_REGION || secrets.GCLOUD_REGION }}
+          GCLOUD_ZONE: ${{ vars.GCLOUD_FRESH_ZONE || secrets.GCLOUD_FRESH_ZONE || vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }}
         run: ci/cloud-smoke.sh destroy-gcp-wp
 
       - name: Destroy upgrade smoke resources

.github/workflows/github-release.yaml

@@ -15,3 +15,24 @@ jobs:
       contents: write
       actions: write
     secrets: inherit
+
+  verify-rootfs-assets:
+    name: Verify rootfs release assets
+    needs: release
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout merged source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
+
+      - name: Wait for and verify immutable rootfs assets
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: >-
+          ci/verify-rootfs-release.sh commit
+          "${{ github.event.pull_request.merge_commit_sha }}"

.github/workflows/publish-rootfs.yaml

@@ -20,11 +20,17 @@ jobs:
           ci/rootfs-package-contract.sh
           ci/package-rootfs.sh dist
 
-      - name: Upload rootfs asset and checksum
+      - name: Upload rootfs asset, archive checksum, and source contract
         env:
           GH_TOKEN: ${{ github.token }}
         run: >-
           gh release upload "$GITHUB_REF_NAME"
           dist/cloud-compose-rootfs.tar.gz
           dist/cloud-compose-rootfs.tar.gz.sha256
+          dist/cloud-compose-rootfs.contract.sha256
           --clobber
+
+      - name: Verify published rootfs release assets
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: ci/verify-rootfs-release.sh tag "$GITHUB_REF_NAME"

Makefile

@@ -1,4 +1,4 @@
-.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci
+.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security inline-data-program-contract cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci
 .PHONY: terraform-fmt terraform-fmt-check terraform-validate terraform-validate-contract terraform-lint-check terraform-docs terraform-docs-check
 .PHONY: config-management-cloud-smoke config-management-cloud-smoke-ansible-drupal config-management-cloud-smoke-salt-drupal
 .PHONY: destroy-config-management-cloud-smoke destroy-config-management-cloud-smoke-ansible-drupal destroy-config-management-cloud-smoke-salt-drupal
@@ -16,9 +16,9 @@ export CLOUD_COMPOSE_CI_BIN
 GO_MODULE_FILES := $(wildcard go.mod go.sum)
 GO_SOURCES := $(shell find cmd internal -type f -name '*.go')
 
-lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
+lint: terraform-fmt actionlint shell-lint host-runtime-security inline-data-program-contract cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
 
-lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
+lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security inline-data-program-contract cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
 
 actionlint:
 	go run github.com/rhysd/actionlint/cmd/actionlint@$(ACTIONLINT_VERSION)
@@ -58,6 +58,9 @@ app-filesystem-convergence-contract:
 backup-contract:
 	bash ci/backup-contract.sh
 
+disaster-recovery-contract:
+	bash ci/disaster-recovery-contract.sh
+
 overlay-contract:
 	bash ci/overlay-contract.sh
 
@@ -102,6 +105,9 @@ rootfs-package-contract:
 host-runtime-security:
 	bash ci/host-runtime-security.sh
 
+inline-data-program-contract:
+	bash ci/inline-data-program-contract.sh
+
 cos-jq-portability-contract:
 	bash ci/cos-jq-portability-contract.sh