From e3b313a47da4899a54115a010c8b853a06546819 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:56:00 +0000 Subject: [PATCH 01/67] [minor] Add off-host backup and restore proof --- Makefile | 9 +- README.md | 10 +- ansible/README.md | 15 ++ ansible/roles/cloud_compose/tasks/main.yml | 13 +- ci/config-management-input-contract.sh | 7 + ci/config-management-smoke-inner.sh | 7 + ci/disaster-recovery-contract.sh | 237 +++++++++++++++++ ci/systemd-contract.sh | 18 ++ docs/disaster-recovery.md | 177 +++++++++++++ docs/index.md | 7 +- docs/mkdocs.yml | 1 + docs/non-gcp-providers.md | 15 +- docs/runtime-contracts.md | 32 ++- .../contracttest/disaster_recovery_test.go | 96 +++++++ main.tf | 16 +- modules/digitalocean/main.tf | 56 ++-- modules/digitalocean/variables.tf | 14 + modules/gcp/main.tf | 84 +++--- modules/gcp/variables.tf | 21 ++ modules/linode/main.tf | 56 ++-- modules/linode/variables.tf | 14 + .../disaster_recovery.tftest.hcl | 48 ++++ modules/linux-vm-runtime/main.tf | 100 +++---- modules/linux-vm-runtime/variables.tf | 21 ++ providers/do/variables.tf | 14 + providers/gcp/main.tf | 16 +- providers/gcp/variables.tf | 14 + providers/linode/variables.tf | 14 + .../system/cloud-compose-mariadb-backup.timer | 4 +- .../cloud-compose-offhost-backup.service | 17 ++ .../system/cloud-compose-restore-test.service | 16 ++ .../system/cloud-compose-restore-test.timer | 13 + rootfs/etc/tmpfiles.d/cloud-compose.conf | 1 + .../cloud-compose/disaster-recovery-lib.sh | 211 +++++++++++++++ rootfs/home/cloud-compose/offhost-backup.sh | 249 ++++++++++++++++++ rootfs/home/cloud-compose/restore-test.sh | 83 ++++++ rootfs/home/cloud-compose/run.sh | 5 + runtime_contracts.tftest.hcl | 50 ++++ salt/cloud-compose/README.md | 16 ++ salt/cloud-compose/init.sls | 14 +- .../ansible/invalid-disaster-recovery.yml | 16 ++ tests/config-management/ansible/smoke.yml | 7 + .../salt-pillar/invalid-disaster-recovery.sls | 12 + tests/config-management/salt-pillar/top.sls | 2 + .../config-management/salt-pillar/wp-prod.sls | 3 + variables.tf | 14 + 46 files changed, 1680 insertions(+), 185 deletions(-) create mode 100644 ci/disaster-recovery-contract.sh create mode 100644 docs/disaster-recovery.md create mode 100644 internal/contracttest/disaster_recovery_test.go create mode 100644 modules/linux-vm-runtime/disaster_recovery.tftest.hcl create mode 100644 rootfs/etc/systemd/system/cloud-compose-offhost-backup.service create mode 100644 rootfs/etc/systemd/system/cloud-compose-restore-test.service create mode 100644 rootfs/etc/systemd/system/cloud-compose-restore-test.timer create mode 100644 rootfs/home/cloud-compose/disaster-recovery-lib.sh create mode 100644 rootfs/home/cloud-compose/offhost-backup.sh create mode 100644 rootfs/home/cloud-compose/restore-test.sh create mode 100644 tests/config-management/ansible/invalid-disaster-recovery.yml create mode 100644 tests/config-management/salt-pillar/invalid-disaster-recovery.sls diff --git a/Makefile b/Makefile index cb7e0e5..ed0b7fb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci +.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci .PHONY: terraform-fmt terraform-fmt-check terraform-validate terraform-validate-contract terraform-lint-check terraform-docs terraform-docs-check .PHONY: config-management-cloud-smoke config-management-cloud-smoke-ansible-drupal config-management-cloud-smoke-salt-drupal .PHONY: destroy-config-management-cloud-smoke destroy-config-management-cloud-smoke-ansible-drupal destroy-config-management-cloud-smoke-salt-drupal @@ -16,9 +16,9 @@ export CLOUD_COMPOSE_CI_BIN GO_MODULE_FILES := $(wildcard go.mod go.sum) GO_SOURCES := $(shell find cmd internal -type f -name '*.go') -lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check +lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check -lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check +lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check actionlint: go run github.com/rhysd/actionlint/cmd/actionlint@$(ACTIONLINT_VERSION) @@ -58,6 +58,9 @@ app-filesystem-convergence-contract: backup-contract: bash ci/backup-contract.sh +disaster-recovery-contract: + bash ci/disaster-recovery-contract.sh + overlay-contract: bash ci/overlay-contract.sh diff --git a/README.md b/README.md index 6e8962a..9fd235c 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,14 @@ They must not recreate or destroy the singleton foundation. The cover the state boundary, Shared VPC setup, and Cloud Run Direct VPC egress requirements. +Production operators can require the provider-neutral +[`runtime.disaster_recovery`](docs/disaster-recovery.md) contract. Cloud Compose +then hands complete database, application-file, and volume-topology coverage to +a root-owned operator driver and accepts only an encrypted off-host receipt plus +scheduled disposable restore proof. Storage credentials remain outside +Terraform and the runtime environment. Same-disk logical dumps and provider +snapshots are never presented as disaster recovery. + ## Requirements @@ -68,7 +76,7 @@ No resources. | [name](#input\_name) | Deployment name. | `string` | n/a | yes | | [cloud\_provider](#input\_cloud\_provider) | Compatibility selector for the root GCP entrypoint. Use providers/do or providers/linode for other clouds. | `string` | `"gcp"` | no | | [gcp](#input\_gcp) | Google Cloud infrastructure settings. | 
object({
    project_id     = optional(string, "")
    project_number = optional(string, "")
    region         = optional(string, "us-east5")
    zone           = optional(string, "us-east5-b")

    identity = optional(object({
      vm_service_account_email  = optional(string, "")
      app_service_account_email = optional(string, "")
      app_credentials_enabled   = optional(bool, false)
    }), {})

    instance = optional(object({
      machine_type = optional(string, "n4-standard-2")
      os           = optional(string, "cos-125-19216-220-185")
      production   = optional(bool, false)
    }), {})

    disks = optional(object({
      type                   = optional(string, "hyperdisk-balanced")
      data_size_gb           = optional(number, 20)
      docker_volumes_size_gb = optional(number, 50)
    }), {})

    network = optional(object({
      create                   = optional(bool, true)
      project_id               = optional(string, "")
      name                     = optional(string, "")
      subnetwork               = optional(string, "")
      ip_cidr_range            = optional(string, "10.42.0.0/24")
      mtu                      = optional(number, 1460)
      power_button_allowed_ips = optional(list(string), [])
      power_button_ip_depth    = optional(number)
      ssh_ipv4                 = optional(list(string), [])
      ssh_ipv6                 = optional(list(string), [])
    }), {})

    snapshots = optional(object({
      enabled = optional(bool, true)
    }), {})

    overlay = optional(object({
      source_instance = optional(string, "")
      volume_names    = optional(list(string), [])
    }), {})

    cloud_init = optional(object({
      initcmd = optional(list(string), [])
      runcmd  = optional(list(string), [])
    }), {})

    artifact_registry = optional(object({
      repository = optional(string, "")
      location   = optional(string, "us")
    }), {})

    power_management = optional(object({
      enabled      = optional(bool, false)
      start_role   = optional(string, "")
      suspend_role = optional(string, "")
      frontend = optional(object({
        image  = string
        port   = optional(number, 8080)
        cpu    = optional(string, "1000m")
        memory = optional(string, "1Gi")
      }), null)
    }), {})

    rollout = optional(object({
      enabled        = optional(bool, false)
      release_url    = optional(string, "")
      release_sha256 = optional(string, "")
      port           = optional(number, 8081)
      jwks_uri       = optional(string, "")
      jwt_audience   = optional(string, "")
      custom_claims  = optional(string, "")
      allowed_ipv4   = optional(list(string), ["10.0.0.0/8"])
    }), {})
  })
| `{}` | no | -| [runtime](#input\_runtime) | Provider-neutral compose/runtime settings. | 
object({
    rootfs                = optional(string, "")
    rootfs_archive_url    = optional(string, "")
    rootfs_archive_sha256 = optional(string, "")
    users                 = optional(map(list(string)), {})

    compose = optional(object({
      primary      = optional(string, "")
      ingress_port = optional(number, 80)
      ingress = optional(object({
        letsencrypt     = optional(bool, false)
        bot_mitigation  = optional(bool, false)
        mode            = optional(string, "")
        domain          = optional(string, "")
        acme_email      = optional(string, "")
        trusted_ips     = optional(list(string), [])
        max_upload_size = optional(string, "")
        upload_timeout  = optional(string, "")
      }), {})
      repo   = optional(string, "")
      branch = optional(string, "")
      projects = optional(map(object({
        docker_compose_repo   = string
        docker_compose_branch = optional(string)
        project_dir           = optional(string)
        compose_project_name  = optional(string)
        ingress_port          = optional(number)
        ingress = optional(object({
          letsencrypt     = optional(bool)
          bot_mitigation  = optional(bool)
          mode            = optional(string)
          domain          = optional(string)
          acme_email      = optional(string)
          trusted_ips     = optional(list(string))
          max_upload_size = optional(string)
          upload_timeout  = optional(string)
        }), {})
        sitectl_context_name   = optional(string)
        sitectl_plugin         = optional(string)
        sitectl_environment    = optional(string)
        sitectl_packages       = optional(list(string))
        sitectl_verify_args    = optional(list(string))
        docker_compose_init    = optional(list(string))
        docker_compose_up      = optional(list(string))
        docker_compose_down    = optional(list(string))
        docker_compose_rollout = optional(list(string))
      })), {})
      init    = optional(list(string))
      up      = optional(list(string))
      down    = optional(list(string))
      rollout = optional(list(string))
    }), {})

    sitectl = optional(object({
      packages         = optional(list(string))
      version          = optional(string, "latest")
      package_versions = optional(map(string), {})
      context_name     = optional(string, "")
      plugin           = optional(string, "core")
      environment      = optional(string, "production")
      verify_args      = optional(list(string), [])
    }), {})

    docker = optional(object({
      # renovate: datasource=github-releases depName=docker-compose packageName=docker/compose versioning=semver
      compose_version = optional(string, "v5.3.1")
      # renovate: datasource=github-releases depName=docker-buildx packageName=docker/buildx versioning=semver
      buildx_version = optional(string, "v0.35.0")
    }), {})

    managed_runtime = optional(object({
      enabled                       = optional(bool, true)
      internal_services_enabled     = optional(bool, false)
      internal_services_auto_update = optional(bool, false)
      artifacts = optional(list(object({
        name    = string
        url     = string
        sha256  = string
        path    = string
        mode    = optional(string, "0755")
        owner   = optional(string, "root")
        group   = optional(string, "root")
        restart = optional(string, "")
      })), [])
    }), {})

    vault = optional(object({
      addr                    = optional(string, "")
      namespace               = optional(string, "")
      role                    = optional(string, "")
      agent_enabled           = optional(bool, false)
      auth_method             = optional(string, "auto")
      gcp_auth_mount_path     = optional(string, "auth/gcp")
      agent_token_path        = optional(string, "/mnt/disks/data/vault/token")
      agent_additional_config = optional(string, "")
      agent_templates = optional(list(object({
        destination = string
        contents    = string
        perms       = optional(string, "0640")
        command     = optional(string, "")
      })), [])
    }), {})

    extra_env = optional(map(string), {})
  })
| `{}` | no | +| [runtime](#input\_runtime) | Provider-neutral compose/runtime settings. | 
object({
    rootfs                = optional(string, "")
    rootfs_archive_url    = optional(string, "")
    rootfs_archive_sha256 = optional(string, "")
    users                 = optional(map(list(string)), {})

    disaster_recovery = optional(object({
      required    = optional(bool, false)
      driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver")
    }), {})

    compose = optional(object({
      primary      = optional(string, "")
      ingress_port = optional(number, 80)
      ingress = optional(object({
        letsencrypt     = optional(bool, false)
        bot_mitigation  = optional(bool, false)
        mode            = optional(string, "")
        domain          = optional(string, "")
        acme_email      = optional(string, "")
        trusted_ips     = optional(list(string), [])
        max_upload_size = optional(string, "")
        upload_timeout  = optional(string, "")
      }), {})
      repo   = optional(string, "")
      branch = optional(string, "")
      projects = optional(map(object({
        docker_compose_repo   = string
        docker_compose_branch = optional(string)
        project_dir           = optional(string)
        compose_project_name  = optional(string)
        ingress_port          = optional(number)
        ingress = optional(object({
          letsencrypt     = optional(bool)
          bot_mitigation  = optional(bool)
          mode            = optional(string)
          domain          = optional(string)
          acme_email      = optional(string)
          trusted_ips     = optional(list(string))
          max_upload_size = optional(string)
          upload_timeout  = optional(string)
        }), {})
        sitectl_context_name   = optional(string)
        sitectl_plugin         = optional(string)
        sitectl_environment    = optional(string)
        sitectl_packages       = optional(list(string))
        sitectl_verify_args    = optional(list(string))
        docker_compose_init    = optional(list(string))
        docker_compose_up      = optional(list(string))
        docker_compose_down    = optional(list(string))
        docker_compose_rollout = optional(list(string))
      })), {})
      init    = optional(list(string))
      up      = optional(list(string))
      down    = optional(list(string))
      rollout = optional(list(string))
    }), {})

    sitectl = optional(object({
      packages         = optional(list(string))
      version          = optional(string, "latest")
      package_versions = optional(map(string), {})
      context_name     = optional(string, "")
      plugin           = optional(string, "core")
      environment      = optional(string, "production")
      verify_args      = optional(list(string), [])
    }), {})

    docker = optional(object({
      # renovate: datasource=github-releases depName=docker-compose packageName=docker/compose versioning=semver
      compose_version = optional(string, "v5.3.1")
      # renovate: datasource=github-releases depName=docker-buildx packageName=docker/buildx versioning=semver
      buildx_version = optional(string, "v0.35.0")
    }), {})

    managed_runtime = optional(object({
      enabled                       = optional(bool, true)
      internal_services_enabled     = optional(bool, false)
      internal_services_auto_update = optional(bool, false)
      artifacts = optional(list(object({
        name    = string
        url     = string
        sha256  = string
        path    = string
        mode    = optional(string, "0755")
        owner   = optional(string, "root")
        group   = optional(string, "root")
        restart = optional(string, "")
      })), [])
    }), {})

    vault = optional(object({
      addr                    = optional(string, "")
      namespace               = optional(string, "")
      role                    = optional(string, "")
      agent_enabled           = optional(bool, false)
      auth_method             = optional(string, "auto")
      gcp_auth_mount_path     = optional(string, "auth/gcp")
      agent_token_path        = optional(string, "/mnt/disks/data/vault/token")
      agent_additional_config = optional(string, "")
      agent_templates = optional(list(object({
        destination = string
        contents    = string
        perms       = optional(string, "0640")
        command     = optional(string, "")
      })), [])
    }), {})

    extra_env = optional(map(string), {})
  })
| `{}` | no | | [template](#input\_template) | Optional compose template preset. Supported values are archivesspace, ojs, isle, drupal, wp, omeka-s, and omeka-classic. Explicit runtime settings override preset defaults. | `string` | `""` | no | ## Outputs diff --git a/ansible/README.md b/ansible/README.md index b849020..665539c 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -62,6 +62,21 @@ is meaningful: it disables that phase and is not replaced by the default. Runtime feature switches must be YAML booleans, not quoted strings; ambiguous values are rejected before host mutation. +The provider-neutral disaster-recovery interface is shared with Terraform: + +```yaml +cloud_compose_runtime: + disaster_recovery: + required: true + driver_path: /usr/local/libexec/cloud-compose/offhost-backup-driver +``` + +Install that executable and its credentials separately as root. The role +renders only the boolean requirement and executable path; storage endpoints, +credentials, encryption keys, and retention policy must not be placed in +inventory. See [the driver contract](../docs/disaster-recovery.md) for receipt +and scheduled restore-proof requirements. + Set `cloud_compose_runtime.rollout` to enable the same authenticated rollout listener used by Terraform. Supply a pinned HTTPS `release_url`, its lowercase `release_sha256`, an HTTPS `jwks_uri`, `jwt_audience`, and optional JSON-object diff --git a/ansible/roles/cloud_compose/tasks/main.yml b/ansible/roles/cloud_compose/tasks/main.yml index 7057042..1dd1912 100644 --- a/ansible/roles/cloud_compose/tasks/main.yml +++ b/ansible/roles/cloud_compose/tasks/main.yml @@ -55,6 +55,7 @@ _cc_sitectl: "{{ cloud_compose_runtime.sitectl | default({}) }}" _cc_managed: "{{ cloud_compose_runtime.managed_runtime | default({}) }}" _cc_vault: "{{ cloud_compose_runtime.vault | default({}) }}" + _cc_disaster_recovery: "{{ cloud_compose_runtime.disaster_recovery | default({}) }}" _cc_rollout_service: "{{ cloud_compose_default_rollout_service | combine(cloud_compose_runtime.rollout | default({}), recursive=True) }}" _cc_extra_env: "{{ cloud_compose_runtime.extra_env | default(cloud_compose_extra_env) }}" @@ -65,6 +66,7 @@ - _cc_sitectl is mapping - _cc_managed is mapping - _cc_vault is mapping + - _cc_disaster_recovery is mapping - (cloud_compose_runtime.rollout | default({})) is mapping - (_cc_compose.ingress | default({})) is mapping - (_cc_compose.projects | default({})) is mapping @@ -79,7 +81,12 @@ - (_cc_managed.internal_services_auto_update | default(cloud_compose_internal_services_auto_update)) is boolean - (_cc_vault.agent_enabled | default(false)) is boolean - not (_cc_vault.agent_enabled | default(false)) - fail_msg: "runtime sections must be maps, runtime feature switches must be booleans, template must name a supported app, and Vault Agent is currently supported only by Terraform providers (set vault.agent_enabled=false for Ansible)." + - (_cc_disaster_recovery.required | default(false)) is boolean + - (_cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver')) is string + - (_cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver')) is match('^/[A-Za-z0-9._/+:-]+$') + - "'//' not in (_cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver'))" + - (_cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver') | regex_search('(^|/)\\.\\.?(/|$)')) is none + fail_msg: "runtime sections must be maps, runtime feature switches must be booleans, template must name a supported app, Vault Agent is currently supported only by Terraform providers (set vault.agent_enabled=false for Ansible), and disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments." - name: Require safe rollout service settings ansible.builtin.assert: @@ -136,6 +143,8 @@ _cc_down_commands: "{{ _cc_compose.down | default(cloud_compose_default_down) }}" _cc_rollout_commands: "{{ _cc_compose.rollout | default(cloud_compose_default_rollout) }}" _cc_internal_services_enabled: "{{ _cc_managed.internal_services_enabled | default(cloud_compose_internal_services_enabled) }}" + _cc_offhost_backup_required: "{{ _cc_disaster_recovery.required | default(false) }}" + _cc_offhost_backup_driver: "{{ _cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver') }}" - name: Reject GCP-only internal services on an on-prem host ansible.builtin.assert: @@ -398,6 +407,8 @@ CLOUD_COMPOSE_INSTANCE_NAME: "{{ cloud_compose_name }}" CLOUD_COMPOSE_APPS: "{{ _cc_compose_projects.keys() | list | join(' ') }}" CLOUD_COMPOSE_PRIMARY_APP: "{{ _cc_primary_key }}" + CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED: "{{ _cc_offhost_backup_required | ternary('true', 'false') }}" + CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER: "{{ _cc_offhost_backup_driver }}" COMPOSE_PROJECTS_FILE: "{{ cloud_compose_home }}/compose-projects.json" COMPOSE_PROJECT_NAME: "{{ _cc_primary_project.compose_project_name }}" COMPOSE_BIND_PORT: "{{ _cc_primary_project.ingress_port }}" diff --git a/ci/config-management-input-contract.sh b/ci/config-management-input-contract.sh index 8b3283b..7781958 100755 --- a/ci/config-management-input-contract.sh +++ b/ci/config-management-input-contract.sh @@ -226,6 +226,9 @@ for marker in ( "item.value.sitectl_packages | default(_cc_sitectl_packages)", "(_cc_managed.enabled | default(cloud_compose_managed_runtime_enabled)) is boolean", "(_cc_vault.agent_enabled | default(false)) is boolean", + "(_cc_disaster_recovery.required | default(false)) is boolean", + "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED", + "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER", ): if marker not in ansible_tasks: fail(f"Ansible template package-version parity marker is missing: {marker!r}") @@ -247,6 +250,9 @@ for marker in ( "all_packages = sitectl_packages | list", "'managed_runtime.enabled': managed_runtime_enabled", "'vault.agent_enabled': vault.get('agent_enabled', False)", + "runtime_sections.disaster_recovery", + "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED", + "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER", "run_bootstrap is sameas true", ): if marker not in salt_state: @@ -359,6 +365,7 @@ adapter_fixture_markers = { "invalid-lifecycle": "- 42", "invalid-project-lifecycle": "docker_compose_up: not-a-list", "invalid-internal-services": "internal_services_enabled: true", + "invalid-disaster-recovery": "driver_path: /usr/local/libexec/cloud-compose/../untrusted", } for fixture_name, marker in adapter_fixture_markers.items(): for relative_path in ( diff --git a/ci/config-management-smoke-inner.sh b/ci/config-management-smoke-inner.sh index 611e5db..00ce871 100755 --- a/ci/config-management-smoke-inner.sh +++ b/ci/config-management-smoke-inner.sh @@ -141,6 +141,7 @@ ansible_invalid_cases=( invalid-primary invalid-template invalid-vault + invalid-disaster-recovery invalid-package invalid-host-ack invalid-project-dir-root @@ -206,6 +207,7 @@ run_invalid_ansible_case invalid-internal-services 'internal-services stack is G run_invalid_ansible_case invalid-primary 'compose.primary must match' run_invalid_ansible_case invalid-template 'template must name a supported app' run_invalid_ansible_case invalid-vault 'Vault Agent is currently supported only by Terraform' +run_invalid_ansible_case invalid-disaster-recovery 'driver_path must be a safe absolute path' run_invalid_ansible_case invalid-package 'sitectl packages must use valid release package names' run_invalid_ansible_case invalid-host-ack 'dedicated_host_acknowledged=true' run_invalid_ansible_case invalid-project-dir-root 'project_dir must be a normalized non-root absolute path' @@ -257,6 +259,8 @@ projects = json.loads(Path("/home/cloud-compose/compose-projects.json").read_tex project = projects["isle-prod"] assert env["CLOUD_COMPOSE_PROVIDER"] == "onprem" +assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false" +assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"] == "/usr/local/libexec/cloud-compose/ansible-offhost" assert env["DOCKER_COMPOSE_DIR"] == "/mnt/disks/data/libops/isle/isle-prod" assert env["DOCKER_COMPOSE_REPO"] == "https://github.com/libops/isle" assert env["SITECTL_PLUGIN"] == "isle" @@ -447,6 +451,8 @@ assert project["ingress"]["domain"] == expected_domain assert Path(expected_project_dir).is_dir() if expected_name == "wp-prod": + assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false" + assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"] == "/usr/local/libexec/cloud-compose/salt-offhost" assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { "sitectl": "v1.0.0", "sitectl-wp": "v1.0.0", @@ -581,6 +587,7 @@ run_invalid_salt_case invalid-internal-services 'internal-services stack is GCP- run_invalid_salt_case invalid-primary 'compose.primary must match' run_invalid_salt_case invalid-template 'template must name a supported cloud-compose app' run_invalid_salt_case invalid-vault 'Vault Agent is currently supported only by Terraform' +run_invalid_salt_case invalid-disaster-recovery 'driver_path must be a safe absolute path' run_invalid_salt_case invalid-package 'invalid installed package' run_invalid_salt_case invalid-host-ack 'dedicated_host_acknowledged=true' run_invalid_salt_case invalid-project-dir-root 'project_dir must be a normalized non-root absolute path' diff --git a/ci/disaster-recovery-contract.sh b/ci/disaster-recovery-contract.sh new file mode 100644 index 0000000..bfdfa23 --- /dev/null +++ b/ci/disaster-recovery-contract.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +backup_script="$repo_root/rootfs/home/cloud-compose/offhost-backup.sh" +restore_script="$repo_root/rootfs/home/cloud-compose/restore-test.sh" +tmp="$(mktemp -d "$repo_root/.disaster-recovery-contract.XXXXXX")" +trap 'rm -rf -- "$tmp"' EXIT + +fail() { + echo "disaster recovery contract: $*" >&2 + exit 1 +} + +mkdir -p "$tmp/bin" "$tmp/data/projects/alpha/files" "$tmp/data/backups/mariadb/alpha" "$tmp/drivers" +chmod 0755 "$tmp" "$tmp/bin" "$tmp/data" "$tmp/data/projects" "$tmp/data/projects/alpha" "$tmp/data/projects/alpha/files" "$tmp/drivers" + +cat >"$tmp/profile.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +export PATH="${TEST_BIN:?}:/usr/bin:/bin" +acquire_cloud_compose_lifecycle_lock() { + printf '%s\n' "$1" >>"${LOCK_LOG:?}" +} +EOF + +cat >"$tmp/compose-apps.sh" <<'EOF' +#!/usr/bin/env bash +compose_app_names_array() { + local -n result="$1" + result=(alpha) +} +source_compose_app_env() { + DOCKER_COMPOSE_DIR="${TEST_DATA_ROOT:?}/projects/$1" + export DOCKER_COMPOSE_DIR +} +validate_compose_project_dir() { + [[ "$1" == "${TEST_DATA_ROOT:?}/projects/"* ]] +} +EOF + +cat >"$tmp/bin/stat" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +output="$(/usr/bin/stat "$@")" +if [[ "$*" == *"%u:"* ]]; then + printf '0:%s\n' "${output#*:}" +else + printf '%s\n' "$output" +fi +EOF + +cat >"$tmp/bin/install" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +args=() +while (($# > 0)); do + case "$1" in + -o|-g) + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac +done +exec /usr/bin/install "${args[@]}" +EOF + +cat >"$tmp/bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +[[ "$*" == "compose config --format json" ]] +jq -cn --arg root "${TEST_DATA_ROOT:?}" '{ + services: { + web: { + volumes: [ + {type: "bind", source: ($root + "/projects/alpha/files"), target: "/srv/files", read_only: false}, + {type: "volume", source: "alpha_data", target: "/var/lib/app", read_only: false}, + {type: "tmpfs", source: "", target: "/run/app", read_only: false} + ] + } + }, + volumes: {alpha_data: {name: "alpha_data"}} +}' +EOF + +cat >"$tmp/drivers/good-driver" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +[[ -z "${SHOULD_NOT_REACH_DRIVER+x}" ]] || exit 90 +printf '%s\n' "$1" >>"${0}.calls" +operation="$1" +shift +declare -A args=() +while (($# > 0)); do + args["$1"]="$2" + shift 2 +done +case "$operation" in + backup) + jq -cn \ + --arg operation_id "${args[--operation-id]}" \ + --arg manifest_sha256 "${args[--manifest-sha256]}" '{ + schema_version: 1, + kind: "cloud-compose.offhost-backup-receipt", + operation_id: $operation_id, + completed_at: "2026-08-07T12:00:00Z", + manifest_sha256: $manifest_sha256, + encrypted: true, + off_host: true, + status: "succeeded", + remote_id: "contract/backup-1", + coverage: {database: true, application_files: true, volume_topology: true} + }' >"${args[--receipt]}" + ;; + restore-test) + jq -cn \ + --arg test_id "${args[--test-id]}" \ + --arg manifest_sha256 "${args[--source-manifest-sha256]}" \ + --arg receipt_sha256 "${args[--source-receipt-sha256]}" '{ + schema_version: 1, + kind: "cloud-compose.restore-test-proof", + test_id: $test_id, + completed_at: "2026-08-07T13:00:00Z", + source_manifest_sha256: $manifest_sha256, + source_receipt_sha256: $receipt_sha256, + source_encrypted: true, + status: "succeeded", + recovery_id: "contract/recovery-1", + disposable_recovery: true, + recovery_destroyed: true, + integrity_verified: true, + coverage: {database: true, application_files: true, volume_topology: true} + }' >"${args[--proof]}" + ;; + *) exit 64 ;; +esac +EOF + +cat >"$tmp/drivers/incomplete-driver" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +operation="$1" +shift +declare -A args=() +while (($# > 0)); do + args["$1"]="$2" + shift 2 +done +[[ "$operation" == "backup" ]] +jq -cn \ + --arg operation_id "${args[--operation-id]}" \ + --arg manifest_sha256 "${args[--manifest-sha256]}" '{ + schema_version: 1, + kind: "cloud-compose.offhost-backup-receipt", + operation_id: $operation_id, + completed_at: "2026-08-07T12:00:00Z", + manifest_sha256: $manifest_sha256, + encrypted: true, + off_host: true, + status: "succeeded", + remote_id: "contract/incomplete", + coverage: {database: true, application_files: true, volume_topology: false} + }' >"${args[--receipt]}" +EOF + +chmod 0755 "$tmp/bin/stat" "$tmp/bin/install" "$tmp/bin/docker" "$tmp/drivers/good-driver" "$tmp/drivers/incomplete-driver" +printf 'logical database\n' | gzip -c >"$tmp/data/backups/mariadb/alpha/$(date -u +%Y%m%d)-alpha.sql.gz" + +export TEST_BIN="$tmp/bin" +export TEST_DATA_ROOT="$tmp/data" +export LOCK_LOG="$tmp/lock.log" +export CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh" +export CLOUD_COMPOSE_COMPOSE_APPS_PATH="$tmp/compose-apps.sh" +export CLOUD_COMPOSE_DR_LIBRARY_PATH="$repo_root/rootfs/home/cloud-compose/disaster-recovery-lib.sh" +export CLOUD_COMPOSE_DR_STATE_ROOT="$tmp/data/dr" +export CLOUD_COMPOSE_DATA_ROOT="$tmp/data" +export CLOUD_COMPOSE_VOLUMES_ROOT="$tmp/data/volumes" +export MARIADB_BACKUP_ROOT="$tmp/data/backups/mariadb" +export CLOUD_COMPOSE_INSTANCE_NAME="contract-site" +export CLOUD_COMPOSE_PROVIDER="contract" +export CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED="true" +export CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="$tmp/drivers/good-driver" +export SHOULD_NOT_REACH_DRIVER="terraform-secret" +: >"$LOCK_LOG" + +bash "$backup_script" +receipt="$(find "$CLOUD_COMPOSE_DR_STATE_ROOT/backup-receipts" -maxdepth 1 -type f -name '*.json' -print -quit)" +manifest="$(find "$CLOUD_COMPOSE_DR_STATE_ROOT/manifests" -maxdepth 1 -type f -name '*.json' -print -quit)" +[[ -n "$receipt" && -n "$manifest" ]] || fail "successful driver did not publish its atomic manifest and receipt" +jq -e ' + .required_coverage == ["database", "application_files", "volume_topology"] and + (.applications | length == 1) and + (.applications[0].databases | length == 1) and + .applications[0].application_files.roots == [env.TEST_DATA_ROOT + "/projects/alpha"] and + .applications[0].volume_topology.declared_named_volumes == ["alpha_data"] and + (.applications[0].volume_topology.service_mounts | length == 3) +' "$manifest" >/dev/null || fail "coverage manifest omitted database, application files, or volume topology" + +# A second attempt sees the same already-existing daily dump but must still +# invoke off-host transfer, allowing a failed first handoff to be retried. +bash "$backup_script" +[[ "$(grep -c '^backup$' "$tmp/drivers/good-driver.calls")" == "2" ]] || \ + fail "nightly flow skipped off-host transfer when the daily dump already existed" + +published_receipt_sha="$(sha256sum "$receipt" | awk '{print $1}')" +export CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="$tmp/drivers/incomplete-driver" +if bash "$backup_script" >/dev/null 2>&1; then + fail "driver receipt without complete volume coverage was accepted" +fi +[[ "$(sha256sum "$receipt" | awk '{print $1}')" == "$published_receipt_sha" ]] || \ + fail "invalid driver output replaced the last validated receipt" + +export CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="$tmp/drivers/good-driver" +bash "$restore_script" +proof="$(find "$CLOUD_COMPOSE_DR_STATE_ROOT/restore-proofs" -maxdepth 1 -type f -name '*.json' -print -quit)" +[[ -n "$proof" ]] || fail "scheduled disposable restore test did not publish proof" +jq -e ' + .disposable_recovery == true and + .recovery_destroyed == true and + .integrity_verified == true and + .coverage == {database: true, application_files: true, volume_topology: true} +' "$proof" >/dev/null || fail "restore proof omitted required recovery evidence" + +rm -f -- "$tmp/data/backups/mariadb/alpha/$(date -u +%Y%m%d)-alpha.sql.gz" +calls_before="$(wc -l <"$tmp/drivers/good-driver.calls")" +if bash "$backup_script" >/dev/null 2>&1; then + fail "required DR coverage succeeded without a database artifact" +fi +[[ "$(wc -l <"$tmp/drivers/good-driver.calls")" == "$calls_before" ]] || \ + fail "driver ran before required local coverage was validated" + +CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED=false bash "$backup_script" >/dev/null +echo "Disaster recovery contract passed" diff --git a/ci/systemd-contract.sh b/ci/systemd-contract.sh index 07e38a4..dc00466 100755 --- a/ci/systemd-contract.sh +++ b/ci/systemd-contract.sh @@ -27,6 +27,9 @@ assert_contains() { [[ -f "$unit_dir/cloud-compose-bootstrap.service" ]] || fail "retryable bootstrap unit is missing" [[ -f "$unit_dir/cloud-compose-internal-services.service" && -f "$unit_dir/cloud-compose-internal-services.timer" ]] || \ fail "namespaced internal-services units are missing" +[[ -f "$unit_dir/cloud-compose-offhost-backup.service" ]] || fail "off-host backup service is missing" +[[ -f "$unit_dir/cloud-compose-restore-test.service" && -f "$unit_dir/cloud-compose-restore-test.timer" ]] || \ + fail "scheduled restore-test units are missing" assert_contains "$unit_dir/cloud-compose.service" 'Requires=docker.service cloud-compose-metadata-firewall.service' assert_contains "$unit_dir/cloud-compose.service" 'RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes' @@ -50,6 +53,18 @@ assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'LogRateLimitBurst=1 assert_contains "$unit_dir/cloud-compose-internal-services.service" 'Requires=cloud-compose.service cloud-compose-metadata-firewall.service' assert_contains "$unit_dir/cloud-compose.service" 'TimeoutStartSec=1h' assert_contains "$unit_dir/cloud-compose-mariadb-backup.service" 'TimeoutStartSec=12h' +assert_contains "$unit_dir/cloud-compose-mariadb-backup.service" 'User=cloud-compose' +assert_contains "$unit_dir/cloud-compose-mariadb-backup.timer" 'Unit=cloud-compose-offhost-backup.service' +assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'Requires=cloud-compose-mariadb-backup.service' +assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'After=cloud-compose-mariadb-backup.service network-online.target' +assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'User=root' +assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'UMask=0077' +assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'TimeoutStartSec=24h' +assert_contains "$unit_dir/cloud-compose-restore-test.service" 'User=root' +assert_contains "$unit_dir/cloud-compose-restore-test.service" 'UMask=0077' +assert_contains "$unit_dir/cloud-compose-restore-test.service" 'TimeoutStartSec=24h' +assert_contains "$unit_dir/cloud-compose-restore-test.timer" 'OnCalendar=Sun *-*-* 03:00:00' +assert_contains "$unit_dir/cloud-compose-restore-test.timer" 'Persistent=true' if grep -Fq 'Wants=cloud-compose.service' "$unit_dir/cloud-compose-mariadb-backup.service"; then fail "backup service starts an intentionally inactive application" fi @@ -98,6 +113,9 @@ assert_contains "$run_script" 'systemctl disable --now libops-managed-runtime.ti assert_contains "$run_script" 'systemctl enable --now cloud-compose-docker-prune.timer' assert_contains "$run_script" 'systemctl disable --now cloud-compose-docker-prune.timer cloud-compose-docker-prune.service' assert_contains "$run_script" 'systemctl enable --now cloud-compose-mariadb-backup.timer' +assert_contains "$run_script" 'runtime_enabled "${CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED:-false}"' +assert_contains "$run_script" 'systemctl enable --now cloud-compose-restore-test.timer' +assert_contains "$run_script" 'systemctl disable --now cloud-compose-restore-test.timer cloud-compose-restore-test.service' migration_script="$repo_root/rootfs/home/cloud-compose/migrate-legacy-systemd-units.sh" migration_tmp="$(mktemp -d)" diff --git a/docs/disaster-recovery.md b/docs/disaster-recovery.md new file mode 100644 index 0000000..5ddcf3f --- /dev/null +++ b/docs/disaster-recovery.md @@ -0,0 +1,177 @@ +# Disaster recovery driver + +Cloud Compose keeps a same-disk logical MariaDB dump for each application and +can hand a complete recovery manifest to an operator-owned off-host backup +driver. The interface is provider-neutral: cloud-compose does not choose an +object store, account, encryption service, retention policy, or credential +mechanism. + +Local dumps and provider boot-disk snapshots are recovery aids, not disaster +recovery. A site has DR coverage only after the nightly unit publishes a valid +receipt proving encrypted off-host coverage of every database, application-file +root, bind mount, named volume, and service-mount topology in the manifest. + +## Enable the contract + +Install the reviewed driver and all of its configuration out of band. The file +and every directory in its path must be root-owned, must not be a symlink or be +group/world writable, and the executable must have exactly one hard link. +Then enable the provider-neutral runtime input: + +```hcl +runtime = { + disaster_recovery = { + required = true + driver_path = "/usr/local/libexec/cloud-compose/offhost-backup-driver" + } +} +``` + +Terraform, cloud-init, `.env`, plans, state, and application lifecycle commands +must not contain a storage endpoint, bucket credential, encryption key, or +access token. The driver owns those details. A root-only configuration file, +host workload identity, or an operator-managed Vault integration are suitable +implementation choices. Do not use `runtime.extra_env` for driver credentials. + +Ansible and Salt accept the same nested `runtime.disaster_recovery` object. +Changing `required` to `true` before the driver is installed intentionally +makes the next nightly handoff and weekly restore test fail. + +## Backup invocation + +The nightly `cloud-compose-mariadb-backup.timer` starts +`cloud-compose-offhost-backup.service`. Systemd first runs the existing +unprivileged MariaDB dump service. The root service then creates a deterministic +manifest and invokes: + +```text +DRIVER backup \ + --manifest PATH \ + --manifest-sha256 SHA256 \ + --operation-id YYYYMMDD-SITE \ + --receipt PATH +``` + +The driver receives a clean environment containing only `HOME=/root` and a +fixed system `PATH`. Its stdout and stderr are suppressed so an accidental SDK +or credential diagnostic cannot enter the system journal. The driver must load +its own operator-managed configuration and write its receipt to the requested +path. It must not modify the manifest. + +For every app, the manifest includes: + +- a root-only staged copy of the validated `sql.gz` logical dump, including its + SHA-256 and byte count; +- the application checkout root and persistent bind-mount sources; +- every declared named volume; and +- every resolved Compose service mount, including target, type, source, and + read-only state. + +Cloud Compose extracts only this topology from `docker compose config`; the +rendered Compose model, which can contain application environment values, stays +in a mode-0600 staging directory and is deleted. Persistent bind mounts outside +`/mnt/disks/data` and `/mnt/disks/volumes` fail closed instead of extending the +privileged backup boundary. + +The driver must finish encrypting and durably transferring every referenced +component before it writes this exact receipt shape: + +```json +{ + "schema_version": 1, + "kind": "cloud-compose.offhost-backup-receipt", + "operation_id": "20260807-example-site", + "completed_at": "2026-08-07T12:00:00Z", + "manifest_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "encrypted": true, + "off_host": true, + "status": "succeeded", + "remote_id": "operator-safe-opaque-reference", + "coverage": { + "database": true, + "application_files": true, + "volume_topology": true + } +} +``` + +`remote_id` is deliberately constrained to a short opaque identifier; it must +not contain a signed URL, token, query string, or credential. Unknown fields, +missing coverage, false encryption/off-host claims, a mismatched operation or +manifest digest, unsafe ownership, links, oversized JSON, and malformed values +are rejected. The validated manifest and receipt are atomically published under +`/mnt/disks/data/.cloud-compose-disaster-recovery/`. Driver failure leaves the +last valid receipt untouched. + +The off-host handoff runs even when the day's valid local dump already exists. +That is what lets the timer retry an earlier transfer failure without rewriting +the database artifact. The local dump remains subject to its independent +14-day retention policy and must never be reported as DR coverage. + +## Scheduled restore proof + +When DR is required, bootstrap enables `cloud-compose-restore-test.timer`. It +runs weekly on Sunday with a stable randomized delay of up to six hours. The +service selects the newest validated backup receipt, creates a cryptographically +random one-time test ID, and invokes: + +```text +DRIVER restore-test \ + --manifest PATH \ + --backup-receipt PATH \ + --source-manifest-sha256 SHA256 \ + --source-receipt-sha256 SHA256 \ + --test-id ONE_TIME_ID \ + --proof PATH +``` + +The driver must restore from off-host encrypted storage into a disposable +recovery environment, verify the database plus representative application-file +and volume data, and destroy that environment. Only then may it emit: + +```json +{ + "schema_version": 1, + "kind": "cloud-compose.restore-test-proof", + "test_id": "20260807T130000Z-random-challenge", + "completed_at": "2026-08-07T13:00:00Z", + "source_manifest_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "source_receipt_sha256": "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + "source_encrypted": true, + "status": "succeeded", + "recovery_id": "operator-safe-opaque-reference", + "disposable_recovery": true, + "recovery_destroyed": true, + "integrity_verified": true, + "coverage": { + "database": true, + "application_files": true, + "volume_topology": true + } +} +``` + +The one-time ID and both source digests prevent a stale proof from satisfying a +new run. Proofs use the same root-owned, bounded, exact-schema, atomic +publication rules as backup receipts. + +## Operator checks + +Treat either unit failure as loss of the managed recovery claim and alert on it: + +```bash +systemctl status cloud-compose-mariadb-backup.service \ + cloud-compose-offhost-backup.service \ + cloud-compose-restore-test.service +journalctl -u cloud-compose-offhost-backup.service \ + -u cloud-compose-restore-test.service +``` + +The journal intentionally contains only cloud-compose's generic status and +validation errors. Driver-specific diagnostics must go to an operator-owned +root-only sink that applies its own secret redaction. + +Before approving a destructive Terraform plan, verify that the newest backup +receipt covers the current topology and that a recent restore proof exists. +Provider snapshots can shorten recovery time, but they do not replace this +independent encrypted copy and disposable restore evidence. diff --git a/docs/index.md b/docs/index.md index 58bafa5..75f5439 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,7 +21,8 @@ the same lifecycle path used by later rollouts. - Optional GCP power management through Cloud Run and lightsout - Provider-neutral runtime contracts for DigitalOcean and Linode - Existing-host deployment through Ansible or Salt -- Nightly MariaDB backups through systemd timers +- Nightly local MariaDB recovery dumps plus an optional provider-neutral, + encrypted off-host DR driver and scheduled restore proofs ## Who owns what @@ -32,7 +33,7 @@ the same lifecycle path used by later rollouts. | App source revision on an existing VM | Authenticated rollout endpoint or operator-run `/home/cloud-compose/rollout` | `sitectl deploy` against an explicit ref and manifest app key | | App Compose behavior and health verification | sitectl plugin/component definitions | Versioned plugin release and normal lifecycle commands | | Secrets and private forge credentials | Vault/operator secret delivery | Short-lived files rendered outside Terraform state | -| Logical backup retention and off-host disaster recovery | cloud-compose timer plus operator-owned storage policy | Local dumps are pruned after 14 days; independent copies and restore tests remain an operator responsibility | +| Local logical backup and off-host disaster recovery | cloud-compose timers plus operator-owned DR driver | Local dumps are pruned after 14 days; a strict receipt proves encrypted independent coverage and a scheduled disposable restore | Any cloud-init byte can change the GCP boot-disk identity and replace the VM; cloud-init is bootstrap configuration, not the day-2 app update channel. Keep @@ -41,6 +42,8 @@ routine source deployments in rollout and application behavior in sitectl. ## Start here - [Runtime contracts](runtime-contracts.md) explains the VM/app contract. +- [Disaster recovery](disaster-recovery.md) defines the off-host driver, + receipts, restore proofs, and operator checks. - [GCP foundation and application states](runtime-contracts.md#gcp-foundation-and-application-states) explains singleton API/IAM ownership, Shared VPC, and Direct VPC egress. - [Managed runtime](managed-runtime.md) covers host tools and internal services. - [Rollout API](rollout.md) covers authenticated deploy triggers. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index f879f50..f59c73b 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -63,6 +63,7 @@ extra_javascript: nav: - Home: index.md - Runtime contracts: runtime-contracts.md + - Disaster recovery: disaster-recovery.md - Managed runtime: managed-runtime.md - Rollout API: rollout.md - DigitalOcean and Linode: non-gcp-providers.md diff --git a/docs/non-gcp-providers.md b/docs/non-gcp-providers.md index 24321af..f22bce6 100644 --- a/docs/non-gcp-providers.md +++ b/docs/non-gcp-providers.md @@ -102,13 +102,14 @@ nightly MariaDB dumps under `/mnt/disks/data/backups` are useful for logical recovery, but they live on the same data volume and are not disaster recovery. Also, `terraform destroy` intentionally deletes both managed volumes. -Before production, establish an independently owned offsite policy for both -volumes (provider volume snapshots where available, or encrypted export to -separate object storage/account), define retention separately from this -application state, and test a restore into disposable new volumes. A restore is -complete only after the Compose projects start, `sitectl healthcheck` passes, -and representative files plus database records are verified. Do not enable a -boot-disk backup toggle and record the application as protected. +Before production, install an operator-owned driver and set +`runtime.disaster_recovery.required = true`. The provider-neutral handoff +requires encrypted off-host coverage for the logical database, application +files, and complete Compose volume topology, then schedules restore tests into +a disposable recovery environment. The exact driver and proof contract is in +[Disaster recovery](disaster-recovery.md). Retention and credentials remain +operator-owned. Do not enable a boot-disk backup toggle and record the +application as protected. Fedora CoreOS should use the CoreOS installer path. Debian and Ubuntu should use the apt installer path. Both paths install the same minimum runtime surface: diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index a94d779..52c1791 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -661,9 +661,10 @@ a pinned checksum intentionally fail as soon as the branch content changes. ## Backups -`cloud-compose-mariadb-backup.timer` runs nightly between 9pm and 7am EST. It -uses a fixed randomized delay so deployments spread out across that window while -keeping a stable schedule on each VM. The timer executes: +`cloud-compose-mariadb-backup.timer` runs the local-backup and off-host handoff +flow nightly between 9pm and 7am EST. It uses a fixed randomized delay so +deployments spread out across that window while keeping a stable schedule on +each VM. The unprivileged local phase executes: ```bash sitectl mariadb backup --context "$SITECTL_CONTEXT_NAME" --gzip --output "$path" @@ -684,13 +685,24 @@ service exits non-zero after attempting all of them. Dumps older than `MARIADB_BACKUP_RETENTION_DAYS` (14 by default) are pruned from each validated app directory so they cannot fill the shared data disk indefinitely. -Local dumps remain on the same failure-domain disk as application data. -Downstream operators must still define reviewed encrypted off-host retention -and restore tests. GCP production enables crash-consistent scheduled disk -snapshots by default; `guest_flush = false` is deliberate because the logical -dump supplies the application-consistent recovery artifact. DigitalOcean and -Linode boot-disk backup toggles do not include attached volumes; see the -provider guide before claiming disaster-recovery coverage. +Local dumps remain on the same failure-domain disk as application data and are +never disaster recovery. Set `runtime.disaster_recovery.required = true` only +after installing the operator-owned root driver. The root handoff runs after +the local service, including when the daily dump already exists, and requires a +strict atomic receipt proving encrypted off-host coverage of each app's logical +database, checkout/bind files, named volumes, and resolved service-mount +topology. A weekly timer requires a challenge-bound proof that the driver +restored all three coverage classes into a disposable recovery environment, +verified integrity, and destroyed that environment. Storage-vendor settings +and credentials stay behind the driver and never enter Terraform, cloud-init, +the host environment, or logs. The complete interface and receipt schemas are +in [Disaster recovery](disaster-recovery.md). + +GCP production enables crash-consistent scheduled disk snapshots by default; +`guest_flush = false` is deliberate because the logical dump supplies the +application-consistent database artifact. DigitalOcean and Linode boot-disk +backup toggles do not include attached volumes. None of those provider-local +copies replace the independent driver receipt and restore proof. Terraform owns the attached data and Docker-volume disks. A normal `terraform destroy` deletes them; GCP production snapshots are retained, but diff --git a/internal/contracttest/disaster_recovery_test.go b/internal/contracttest/disaster_recovery_test.go new file mode 100644 index 0000000..81d65de --- /dev/null +++ b/internal/contracttest/disaster_recovery_test.go @@ -0,0 +1,96 @@ +package contracttest + +import ( + "strings" + "testing" +) + +func TestDisasterRecoveryInputParity(t *testing.T) { + root := repositoryRoot(t) + + for _, relativePath := range []string{ + "variables.tf", + "providers/gcp/variables.tf", + "providers/do/variables.tf", + "providers/linode/variables.tf", + "modules/digitalocean/variables.tf", + "modules/linode/variables.tf", + } { + content := readRepositoryFile(t, root, relativePath) + requireContains(t, content, "disaster_recovery = optional(object({", relativePath+" disaster-recovery object") + requireContains(t, content, "required = optional(bool, false)", relativePath+" required switch") + requireContains(t, content, `driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver")`, relativePath+" driver default") + requireContains(t, content, "runtime.disaster_recovery.driver_path must be a safe absolute path", relativePath+" path validation") + } + + for _, relativePath := range []string{ + "main.tf", + "providers/gcp/main.tf", + "modules/digitalocean/main.tf", + "modules/linode/main.tf", + } { + content := readRepositoryFile(t, root, relativePath) + requireContains(t, content, "offhost_backup_required", relativePath+" required forwarding") + requireContains(t, content, "offhost_backup_driver_path", relativePath+" driver forwarding") + } + + for _, relativePath := range []string{ + "modules/gcp/main.tf", + "modules/linux-vm-runtime/main.tf", + "ansible/roles/cloud_compose/tasks/main.yml", + "salt/cloud-compose/init.sls", + } { + content := readRepositoryFile(t, root, relativePath) + requireContains(t, content, "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED", relativePath+" required host control") + requireContains(t, content, "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER", relativePath+" driver host control") + } +} + +func TestDisasterRecoveryReceiptAndRestoreContracts(t *testing.T) { + root := repositoryRoot(t) + library := readRepositoryFile(t, root, "rootfs/home/cloud-compose/disaster-recovery-lib.sh") + backup := readRepositoryFile(t, root, "rootfs/home/cloud-compose/offhost-backup.sh") + restore := readRepositoryFile(t, root, "rootfs/home/cloud-compose/restore-test.sh") + + for marker, label := range map[string]string{ + `env -i HOME=/root`: "clean driver environment", + `>/dev/null 2>&1`: "suppressed driver output", + `cloud-compose.offhost-backup-receipt`: "backup receipt kind", + `.encrypted == true`: "encrypted coverage", + `.off_host == true`: "off-host coverage", + `.database == true`: "database coverage", + `.application_files == true`: "application-file coverage", + `.volume_topology == true`: "volume-topology coverage", + `cloud-compose.restore-test-proof`: "restore proof kind", + `.disposable_recovery == true`: "disposable recovery proof", + `.recovery_destroyed == true`: "recovery cleanup proof", + `.source_receipt_sha256 == $receipt_sha256`: "source receipt binding", + } { + requireContains(t, library, marker, label) + } + + for marker, label := range map[string]string{ + `docker compose config --format json`: "resolved Compose topology", + `local_recovery_artifact`: "logical database artifact", + `application_files`: "application files", + `volume_topology`: "volume topology", + `cloud_compose_dr_run_driver`: "provider-neutral backup handoff", + } { + requireContains(t, backup, marker, label) + } + + for marker, label := range map[string]string{ + `/dev/urandom`: "one-time restore challenge", + `--backup-receipt`: "remote receipt input", + `cloud_compose_dr_validate_restore_proof`: "strict restore proof validation", + `mv -- "$staged_proof" "$proof_path"`: "atomic restore proof publication", + } { + requireContains(t, restore, marker, label) + } + + for _, forbidden := range []string{"AWS_ACCESS_KEY", "GOOGLE_APPLICATION_CREDENTIALS", "AZURE_STORAGE_KEY"} { + if strings.Contains(library+backup+restore, forbidden) { + t.Errorf("disaster-recovery runtime hard-codes credential channel %q", forbidden) + } + } +} diff --git a/main.tf b/main.tf index 49cfea3..d11f765 100644 --- a/main.tf +++ b/main.tf @@ -161,13 +161,15 @@ module "gcp" { overlay_source_instance = local.gcp_overlay.source_instance volume_names = local.gcp_overlay.volume_names - users = local.runtime.users - rootfs = local.runtime.rootfs - rootfs_archive_url = local.runtime.rootfs_archive_url - rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 - extra_env = local.runtime.extra_env - runcmd = local.gcp_cloud_init.runcmd - initcmd = local.gcp_cloud_init.initcmd + users = local.runtime.users + rootfs = local.runtime.rootfs + rootfs_archive_url = local.runtime.rootfs_archive_url + rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 + offhost_backup_required = local.runtime.disaster_recovery.required + offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path + extra_env = local.runtime.extra_env + runcmd = local.gcp_cloud_init.runcmd + initcmd = local.gcp_cloud_init.initcmd artifact_registry_repository = local.gcp_artifact_registry.repository artifact_registry_location = local.gcp_artifact_registry.location diff --git a/modules/digitalocean/main.tf b/modules/digitalocean/main.tf index 8cd72b9..bf78b55 100644 --- a/modules/digitalocean/main.tf +++ b/modules/digitalocean/main.tf @@ -13,33 +13,35 @@ locals { module "runtime" { source = "../linux-vm-runtime" - name = var.name - provider_name = "digitalocean" - region = local.do.region - data_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.data_volume_name}" - volumes_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.docker_volumes_volume_name}" - ssh_users = merge(local.runtime.users, local.do.ssh.users) - cloud_compose_ssh_keys = local.do.ssh.cloud_compose_keys - rootfs = local.runtime.rootfs - rootfs_archive_url = local.runtime.rootfs_archive_url - rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 - ingress_port = local.compose.ingress_port - primary_compose_project = local.compose.primary - sitectl_ingress = local.compose.ingress - docker_compose_repo = local.compose.repo - docker_compose_branch = local.compose.branch - compose_projects = local.compose.projects - docker_compose_init = local.compose.init - docker_compose_up = local.compose.up - docker_compose_down = local.compose.down - docker_compose_rollout = local.compose.rollout - rollout_enabled = local.do.rollout.enabled - rollout_release_url = local.do.rollout.release_url - rollout_release_sha256 = local.do.rollout.release_sha256 - rollout_port = local.do.rollout.port - rollout_jwks_uri = local.do.rollout.jwks_uri - rollout_jwt_audience = local.do.rollout.jwt_audience - rollout_custom_claims = local.do.rollout.custom_claims + name = var.name + provider_name = "digitalocean" + region = local.do.region + data_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.data_volume_name}" + volumes_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.docker_volumes_volume_name}" + ssh_users = merge(local.runtime.users, local.do.ssh.users) + cloud_compose_ssh_keys = local.do.ssh.cloud_compose_keys + rootfs = local.runtime.rootfs + rootfs_archive_url = local.runtime.rootfs_archive_url + rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 + offhost_backup_required = local.runtime.disaster_recovery.required + offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path + ingress_port = local.compose.ingress_port + primary_compose_project = local.compose.primary + sitectl_ingress = local.compose.ingress + docker_compose_repo = local.compose.repo + docker_compose_branch = local.compose.branch + compose_projects = local.compose.projects + docker_compose_init = local.compose.init + docker_compose_up = local.compose.up + docker_compose_down = local.compose.down + docker_compose_rollout = local.compose.rollout + rollout_enabled = local.do.rollout.enabled + rollout_release_url = local.do.rollout.release_url + rollout_release_sha256 = local.do.rollout.release_sha256 + rollout_port = local.do.rollout.port + rollout_jwks_uri = local.do.rollout.jwks_uri + rollout_jwt_audience = local.do.rollout.jwt_audience + rollout_custom_claims = local.do.rollout.custom_claims sitectl_packages = local.sitectl.packages sitectl_version = local.sitectl.version diff --git a/modules/digitalocean/variables.tf b/modules/digitalocean/variables.tf index 9f65b80..eef7bb9 100644 --- a/modules/digitalocean/variables.tf +++ b/modules/digitalocean/variables.tf @@ -76,6 +76,11 @@ variable "runtime" { rootfs_archive_sha256 = optional(string, "") users = optional(map(list(string)), {}) + disaster_recovery = optional(object({ + required = optional(bool, false) + driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + }), {}) + compose = optional(object({ primary = optional(string, "") ingress_port = optional(number, 80) @@ -184,6 +189,15 @@ variable "runtime" { error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together." } + validation { + condition = ( + can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) && + !strcontains(var.runtime.disaster_recovery.driver_path, "//") && + length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0 + ) + error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments." + } + validation { condition = alltrue([ for name in keys(var.runtime.extra_env) : diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index db483be..e7b03ba 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -362,47 +362,49 @@ fresh_filesystem_env = { CLOUD_COMPOSE_FRESH_FILESYSTEM_IDENTITY = "v1:gcp-disk-id:${google_compute_disk.data.disk_id}" } host_env = merge({ - HOME = "/home/cloud-compose" - GCP_PROJECT = var.project_id - GCP_PROJECT_NUMBER = local.project_number - GCP_INSTANCE_NAME = var.name - CLOUD_COMPOSE_INSTANCE_NAME = var.name - GCP_REGION = var.region - GCP_ZONE = var.zone - CLOUD_COMPOSE_PROVIDER = "gcp" - CLOUD_COMPOSE_APPS = join(" ", keys(local.compose_projects)) - CLOUD_COMPOSE_PRIMARY_APP = local.primary_compose_project_key - COMPOSE_PROJECTS_FILE = "/home/cloud-compose/compose-projects.json" - COMPOSE_PROJECT_NAME = local.primary_compose_project.compose_project_name - COMPOSE_BIND_PORT = tostring(local.primary_compose_project.ingress_port) - DOCKER_COMPOSE_DIR = local.primary_compose_project.project_dir - DOCKER_COMPOSE_REPO = local.primary_compose_project.docker_compose_repo - DOCKER_COMPOSE_BRANCH = local.primary_compose_project.docker_compose_branch - DOCKER_COMPOSE_VERSION = var.docker_compose_version - DOCKER_BUILDX_VERSION = var.docker_buildx_version - DOCKER_VOLUME_OVERLAYS = join(" ", var.volume_names) - SITECTL_PACKAGES = join(" ", module.sitectl_runtime.packages) - SITECTL_VERSION = var.sitectl_version - SITECTL_PACKAGE_VERSIONS = jsonencode(module.sitectl_runtime.package_versions) - SITECTL_CONTEXT_NAME = local.primary_compose_project.sitectl_context_name - SITECTL_PLUGIN = local.primary_compose_project.sitectl_plugin - SITECTL_ENVIRONMENT = local.primary_compose_project.sitectl_environment - PRODUCTION = tostring(var.production) - SITECTL_VERIFY_ARGS = join(" ", local.primary_compose_project.sitectl_verify_args) - GCP_APP_SERVICE_ACCOUNT_EMAIL = local.app_service_account_email - GCP_APP_SERVICE_ACCOUNT_MANAGED = tostring(local.app_service_account_managed) - GCP_APP_CREDENTIALS_ENABLED = tostring(local.app_credentials_enabled) - POWER_MANAGEMENT_ENABLED = tostring(var.power_management_enabled) - COMPOSE_PROFILES = local.internal_services_compose_profiles - VAULT_ADDR = trimspace(var.vault_addr) - VAULT_NAMESPACE = trimspace(var.vault_namespace) - VAULT_ROLE = trimspace(var.vault_role) - VAULT_AGENT_ENABLED = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? "true" : "false" - VAULT_AUTH_METHOD = var.vault_auth_method - VAULT_AGENT_TOKEN_PATH = var.vault_agent_token_path - LIBOPS_MANAGED_RUNTIME_ENABLED = tostring(var.libops_managed_runtime_enabled) - LIBOPS_INTERNAL_SERVICES_ENABLED = tostring(local.internal_services_enabled) - LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE = tostring(local.internal_services_enabled && var.libops_internal_services_auto_update) + HOME = "/home/cloud-compose" + GCP_PROJECT = var.project_id + GCP_PROJECT_NUMBER = local.project_number + GCP_INSTANCE_NAME = var.name + CLOUD_COMPOSE_INSTANCE_NAME = var.name + GCP_REGION = var.region + GCP_ZONE = var.zone + CLOUD_COMPOSE_PROVIDER = "gcp" + CLOUD_COMPOSE_APPS = join(" ", keys(local.compose_projects)) + CLOUD_COMPOSE_PRIMARY_APP = local.primary_compose_project_key + CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED = tostring(var.offhost_backup_required) + CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER = var.offhost_backup_driver_path + COMPOSE_PROJECTS_FILE = "/home/cloud-compose/compose-projects.json" + COMPOSE_PROJECT_NAME = local.primary_compose_project.compose_project_name + COMPOSE_BIND_PORT = tostring(local.primary_compose_project.ingress_port) + DOCKER_COMPOSE_DIR = local.primary_compose_project.project_dir + DOCKER_COMPOSE_REPO = local.primary_compose_project.docker_compose_repo + DOCKER_COMPOSE_BRANCH = local.primary_compose_project.docker_compose_branch + DOCKER_COMPOSE_VERSION = var.docker_compose_version + DOCKER_BUILDX_VERSION = var.docker_buildx_version + DOCKER_VOLUME_OVERLAYS = join(" ", var.volume_names) + SITECTL_PACKAGES = join(" ", module.sitectl_runtime.packages) + SITECTL_VERSION = var.sitectl_version + SITECTL_PACKAGE_VERSIONS = jsonencode(module.sitectl_runtime.package_versions) + SITECTL_CONTEXT_NAME = local.primary_compose_project.sitectl_context_name + SITECTL_PLUGIN = local.primary_compose_project.sitectl_plugin + SITECTL_ENVIRONMENT = local.primary_compose_project.sitectl_environment + PRODUCTION = tostring(var.production) + SITECTL_VERIFY_ARGS = join(" ", local.primary_compose_project.sitectl_verify_args) + GCP_APP_SERVICE_ACCOUNT_EMAIL = local.app_service_account_email + GCP_APP_SERVICE_ACCOUNT_MANAGED = tostring(local.app_service_account_managed) + GCP_APP_CREDENTIALS_ENABLED = tostring(local.app_credentials_enabled) + POWER_MANAGEMENT_ENABLED = tostring(var.power_management_enabled) + COMPOSE_PROFILES = local.internal_services_compose_profiles + VAULT_ADDR = trimspace(var.vault_addr) + VAULT_NAMESPACE = trimspace(var.vault_namespace) + VAULT_ROLE = trimspace(var.vault_role) + VAULT_AGENT_ENABLED = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? "true" : "false" + VAULT_AUTH_METHOD = var.vault_auth_method + VAULT_AGENT_TOKEN_PATH = var.vault_agent_token_path + LIBOPS_MANAGED_RUNTIME_ENABLED = tostring(var.libops_managed_runtime_enabled) + LIBOPS_INTERNAL_SERVICES_ENABLED = tostring(local.internal_services_enabled) + LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE = tostring(local.internal_services_enabled && var.libops_internal_services_auto_update) }, local.fresh_filesystem_env, local.rollout_env) env_file_content = <<-EOT - path: "/home/cloud-compose/.env" diff --git a/modules/gcp/variables.tf b/modules/gcp/variables.tf index 9e1b969..e5b9643 100644 --- a/modules/gcp/variables.tf +++ b/modules/gcp/variables.tf @@ -608,6 +608,27 @@ variable "rootfs_archive_sha256" { description = "Required 64-character SHA-256 checksum when rootfs_archive_url is set." } +variable "offhost_backup_required" { + type = bool + default = false + description = "Require nightly encrypted off-host coverage and scheduled disposable restore proofs from an operator-owned driver. Same-disk MariaDB dumps are retained but are not disaster recovery." +} + +variable "offhost_backup_driver_path" { + type = string + default = "/usr/local/libexec/cloud-compose/offhost-backup-driver" + description = "Absolute path to the operator-supplied, root-owned provider-neutral DR driver. The driver owns its credentials; do not pass them through Terraform." + + validation { + condition = ( + can(regex("^/[A-Za-z0-9._/+:-]+$", var.offhost_backup_driver_path)) && + !strcontains(var.offhost_backup_driver_path, "//") && + length(regexall("(^|/)\\.\\.?(/|$)", var.offhost_backup_driver_path)) == 0 + ) + error_message = "offhost_backup_driver_path must be a safe absolute path without whitespace or dot segments." + } +} + variable "extra_env" { type = map(string) default = {} diff --git a/modules/linode/main.tf b/modules/linode/main.tf index 979dd4e..eed23d8 100644 --- a/modules/linode/main.tf +++ b/modules/linode/main.tf @@ -16,33 +16,35 @@ locals { module "runtime" { source = "../linux-vm-runtime" - name = var.name - provider_name = "linode" - region = local.linode.region - data_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.data_volume_label}" - volumes_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.docker_volumes_volume_label}" - ssh_users = merge(local.runtime.users, local.linode.ssh.users) - cloud_compose_ssh_keys = local.linode.ssh.cloud_compose_keys - rootfs = local.runtime.rootfs - rootfs_archive_url = local.runtime.rootfs_archive_url - rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 - ingress_port = local.compose.ingress_port - primary_compose_project = local.compose.primary - sitectl_ingress = local.compose.ingress - docker_compose_repo = local.compose.repo - docker_compose_branch = local.compose.branch - compose_projects = local.compose.projects - docker_compose_init = local.compose.init - docker_compose_up = local.compose.up - docker_compose_down = local.compose.down - docker_compose_rollout = local.compose.rollout - rollout_enabled = local.linode.rollout.enabled - rollout_release_url = local.linode.rollout.release_url - rollout_release_sha256 = local.linode.rollout.release_sha256 - rollout_port = local.linode.rollout.port - rollout_jwks_uri = local.linode.rollout.jwks_uri - rollout_jwt_audience = local.linode.rollout.jwt_audience - rollout_custom_claims = local.linode.rollout.custom_claims + name = var.name + provider_name = "linode" + region = local.linode.region + data_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.data_volume_label}" + volumes_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.docker_volumes_volume_label}" + ssh_users = merge(local.runtime.users, local.linode.ssh.users) + cloud_compose_ssh_keys = local.linode.ssh.cloud_compose_keys + rootfs = local.runtime.rootfs + rootfs_archive_url = local.runtime.rootfs_archive_url + rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 + offhost_backup_required = local.runtime.disaster_recovery.required + offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path + ingress_port = local.compose.ingress_port + primary_compose_project = local.compose.primary + sitectl_ingress = local.compose.ingress + docker_compose_repo = local.compose.repo + docker_compose_branch = local.compose.branch + compose_projects = local.compose.projects + docker_compose_init = local.compose.init + docker_compose_up = local.compose.up + docker_compose_down = local.compose.down + docker_compose_rollout = local.compose.rollout + rollout_enabled = local.linode.rollout.enabled + rollout_release_url = local.linode.rollout.release_url + rollout_release_sha256 = local.linode.rollout.release_sha256 + rollout_port = local.linode.rollout.port + rollout_jwks_uri = local.linode.rollout.jwks_uri + rollout_jwt_audience = local.linode.rollout.jwt_audience + rollout_custom_claims = local.linode.rollout.custom_claims sitectl_packages = local.sitectl.packages sitectl_version = local.sitectl.version diff --git a/modules/linode/variables.tf b/modules/linode/variables.tf index 6f3dae8..45cec74 100644 --- a/modules/linode/variables.tf +++ b/modules/linode/variables.tf @@ -90,6 +90,11 @@ variable "runtime" { rootfs_archive_sha256 = optional(string, "") users = optional(map(list(string)), {}) + disaster_recovery = optional(object({ + required = optional(bool, false) + driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + }), {}) + compose = optional(object({ primary = optional(string, "") ingress_port = optional(number, 80) @@ -198,6 +203,15 @@ variable "runtime" { error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together." } + validation { + condition = ( + can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) && + !strcontains(var.runtime.disaster_recovery.driver_path, "//") && + length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0 + ) + error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments." + } + validation { condition = alltrue([ for name in keys(var.runtime.extra_env) : diff --git a/modules/linux-vm-runtime/disaster_recovery.tftest.hcl b/modules/linux-vm-runtime/disaster_recovery.tftest.hcl new file mode 100644 index 0000000..d4a3414 --- /dev/null +++ b/modules/linux-vm-runtime/disaster_recovery.tftest.hcl @@ -0,0 +1,48 @@ +run "renders_provider_neutral_disaster_recovery_controls" { + command = plan + + variables { + name = "contract-test" + provider_name = "linode" + region = "us-east" + data_device = "/dev/test-data" + volumes_device = "/dev/test-volumes" + docker_compose_repo = "https://github.com/libops/wp.git" + offhost_backup_required = true + offhost_backup_driver_path = "/usr/local/libexec/cloud-compose/acme-offhost" + } + + assert { + condition = ( + local.host_env.CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED == "true" && + local.host_env.CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER == "/usr/local/libexec/cloud-compose/acme-offhost" + ) + error_message = "The Linux VM runtime must render only the required switch and operator-owned driver path." + } + + assert { + condition = ( + !strcontains(module.runtime_env.content, "ACCESS_KEY") && + !strcontains(module.runtime_env.content, "SECRET_KEY") && + !strcontains(module.runtime_env.content, "BUCKET") + ) + error_message = "The provider-neutral DR contract must not render storage credentials or a storage-vendor destination." + } +} + +run "rejects_unsafe_disaster_recovery_driver_path" { + command = plan + + variables { + name = "contract-test" + provider_name = "linode" + region = "us-east" + data_device = "/dev/test-data" + volumes_device = "/dev/test-volumes" + docker_compose_repo = "https://github.com/libops/wp.git" + offhost_backup_required = true + offhost_backup_driver_path = "/usr/local/libexec/cloud-compose/../untrusted" + } + + expect_failures = [var.offhost_backup_driver_path] +} diff --git a/modules/linux-vm-runtime/main.tf b/modules/linux-vm-runtime/main.tf index c2bb357..8632ee9 100644 --- a/modules/linux-vm-runtime/main.tf +++ b/modules/linux-vm-runtime/main.tf @@ -199,55 +199,57 @@ vault_agent_files_raw = <<-EOT vault_agent_files = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? local.vault_agent_files_raw : "" host_env = { - HOME = "/home/cloud-compose" - CLOUD_COMPOSE_PROVIDER = var.provider_name - CLOUD_COMPOSE_INSTANCE_NAME = var.name - CLOUD_COMPOSE_APPS = join(" ", keys(local.compose_projects)) - CLOUD_COMPOSE_PRIMARY_APP = local.primary_compose_project_key - COMPOSE_PROJECTS_FILE = "/home/cloud-compose/compose-projects.json" - COMPOSE_PROJECT_NAME = local.primary_compose_project.compose_project_name - COMPOSE_BIND_PORT = tostring(local.primary_compose_project.ingress_port) - DOCKER_COMPOSE_DIR = local.primary_compose_project.project_dir - DOCKER_COMPOSE_REPO = local.primary_compose_project.docker_compose_repo - DOCKER_COMPOSE_BRANCH = local.primary_compose_project.docker_compose_branch - DOCKER_COMPOSE_VERSION = var.docker_compose_version - DOCKER_BUILDX_VERSION = var.docker_buildx_version - GCP_PROJECT = "" - GCP_PROJECT_NUMBER = "" - GCP_INSTANCE_NAME = var.name - GCP_REGION = var.region - GCP_ZONE = var.zone != "" ? var.zone : var.region - GCP_APP_SERVICE_ACCOUNT_EMAIL = "" - GCP_APP_CREDENTIALS_ENABLED = "false" - SITECTL_PACKAGES = join(" ", module.sitectl_runtime.packages) - SITECTL_VERSION = var.sitectl_version - SITECTL_PACKAGE_VERSIONS = jsonencode(module.sitectl_runtime.package_versions) - SITECTL_CONTEXT_NAME = local.primary_compose_project.sitectl_context_name - SITECTL_PLUGIN = local.primary_compose_project.sitectl_plugin - SITECTL_ENVIRONMENT = local.primary_compose_project.sitectl_environment - SITECTL_VERIFY_ARGS = join(" ", local.primary_compose_project.sitectl_verify_args) - POWER_MANAGEMENT_ENABLED = "false" - COMPOSE_PROFILES = "" - VAULT_ADDR = trimspace(var.vault_addr) - VAULT_NAMESPACE = trimspace(var.vault_namespace) - VAULT_ROLE = trimspace(var.vault_role) - VAULT_AGENT_ENABLED = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? "true" : "false" - VAULT_AUTH_METHOD = var.vault_auth_method - ROLLOUT_ENABLED = tostring(var.rollout_enabled) - ROLLOUT_DOWNLOAD_URL = trimspace(var.rollout_release_url) - ROLLOUT_DOWNLOAD_SHA256 = trimspace(var.rollout_release_sha256) - ROLLOUT_PORT = tostring(var.rollout_port) - ROLLOUT_JWKS_URI = trimspace(var.rollout_jwks_uri) - ROLLOUT_JWT_AUD = trimspace(var.rollout_jwt_audience) - ROLLOUT_CUSTOM_CLAIMS = trimspace(var.rollout_custom_claims) - ROLLOUT_CMD = "/bin/bash" - ROLLOUT_ARGS = "/home/cloud-compose/rollout" - ROLLOUT_LOCK_FILE = "/mnt/disks/data/rollout.lock" - VAULT_AGENT_TOKEN_PATH = var.vault_agent_token_path - LIBOPS_MANAGED_RUNTIME_ENABLED = tostring(var.libops_managed_runtime_enabled) - LIBOPS_INTERNAL_SERVICES_ENABLED = tostring(var.libops_internal_services_enabled) - LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE = tostring(var.libops_internal_services_auto_update) - INTERNAL_SERVICES_COMPOSE_PROFILES = "" + HOME = "/home/cloud-compose" + CLOUD_COMPOSE_PROVIDER = var.provider_name + CLOUD_COMPOSE_INSTANCE_NAME = var.name + CLOUD_COMPOSE_APPS = join(" ", keys(local.compose_projects)) + CLOUD_COMPOSE_PRIMARY_APP = local.primary_compose_project_key + CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED = tostring(var.offhost_backup_required) + CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER = var.offhost_backup_driver_path + COMPOSE_PROJECTS_FILE = "/home/cloud-compose/compose-projects.json" + COMPOSE_PROJECT_NAME = local.primary_compose_project.compose_project_name + COMPOSE_BIND_PORT = tostring(local.primary_compose_project.ingress_port) + DOCKER_COMPOSE_DIR = local.primary_compose_project.project_dir + DOCKER_COMPOSE_REPO = local.primary_compose_project.docker_compose_repo + DOCKER_COMPOSE_BRANCH = local.primary_compose_project.docker_compose_branch + DOCKER_COMPOSE_VERSION = var.docker_compose_version + DOCKER_BUILDX_VERSION = var.docker_buildx_version + GCP_PROJECT = "" + GCP_PROJECT_NUMBER = "" + GCP_INSTANCE_NAME = var.name + GCP_REGION = var.region + GCP_ZONE = var.zone != "" ? var.zone : var.region + GCP_APP_SERVICE_ACCOUNT_EMAIL = "" + GCP_APP_CREDENTIALS_ENABLED = "false" + SITECTL_PACKAGES = join(" ", module.sitectl_runtime.packages) + SITECTL_VERSION = var.sitectl_version + SITECTL_PACKAGE_VERSIONS = jsonencode(module.sitectl_runtime.package_versions) + SITECTL_CONTEXT_NAME = local.primary_compose_project.sitectl_context_name + SITECTL_PLUGIN = local.primary_compose_project.sitectl_plugin + SITECTL_ENVIRONMENT = local.primary_compose_project.sitectl_environment + SITECTL_VERIFY_ARGS = join(" ", local.primary_compose_project.sitectl_verify_args) + POWER_MANAGEMENT_ENABLED = "false" + COMPOSE_PROFILES = "" + VAULT_ADDR = trimspace(var.vault_addr) + VAULT_NAMESPACE = trimspace(var.vault_namespace) + VAULT_ROLE = trimspace(var.vault_role) + VAULT_AGENT_ENABLED = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? "true" : "false" + VAULT_AUTH_METHOD = var.vault_auth_method + ROLLOUT_ENABLED = tostring(var.rollout_enabled) + ROLLOUT_DOWNLOAD_URL = trimspace(var.rollout_release_url) + ROLLOUT_DOWNLOAD_SHA256 = trimspace(var.rollout_release_sha256) + ROLLOUT_PORT = tostring(var.rollout_port) + ROLLOUT_JWKS_URI = trimspace(var.rollout_jwks_uri) + ROLLOUT_JWT_AUD = trimspace(var.rollout_jwt_audience) + ROLLOUT_CUSTOM_CLAIMS = trimspace(var.rollout_custom_claims) + ROLLOUT_CMD = "/bin/bash" + ROLLOUT_ARGS = "/home/cloud-compose/rollout" + ROLLOUT_LOCK_FILE = "/mnt/disks/data/rollout.lock" + VAULT_AGENT_TOKEN_PATH = var.vault_agent_token_path + LIBOPS_MANAGED_RUNTIME_ENABLED = tostring(var.libops_managed_runtime_enabled) + LIBOPS_INTERNAL_SERVICES_ENABLED = tostring(var.libops_internal_services_enabled) + LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE = tostring(var.libops_internal_services_auto_update) + INTERNAL_SERVICES_COMPOSE_PROFILES = "" } env_file_content = <<-EOT diff --git a/modules/linux-vm-runtime/variables.tf b/modules/linux-vm-runtime/variables.tf index 968cf42..2d8d0f9 100644 --- a/modules/linux-vm-runtime/variables.tf +++ b/modules/linux-vm-runtime/variables.tf @@ -96,6 +96,27 @@ variable "rootfs_archive_sha256" { description = "Required 64-character SHA-256 checksum when rootfs_archive_url is set." } +variable "offhost_backup_required" { + type = bool + default = false + description = "Require nightly encrypted off-host coverage and scheduled disposable restore proofs from an operator-owned driver. Same-disk MariaDB dumps are retained but are not disaster recovery." +} + +variable "offhost_backup_driver_path" { + type = string + default = "/usr/local/libexec/cloud-compose/offhost-backup-driver" + description = "Absolute path to the operator-supplied, root-owned provider-neutral DR driver. The driver owns its credentials; do not pass them through Terraform." + + validation { + condition = ( + can(regex("^/[A-Za-z0-9._/+:-]+$", var.offhost_backup_driver_path)) && + !strcontains(var.offhost_backup_driver_path, "//") && + length(regexall("(^|/)\\.\\.?(/|$)", var.offhost_backup_driver_path)) == 0 + ) + error_message = "offhost_backup_driver_path must be a safe absolute path without whitespace or dot segments." + } +} + variable "ingress_port" { type = number default = 80 diff --git a/providers/do/variables.tf b/providers/do/variables.tf index 571ca4c..aad28f9 100644 --- a/providers/do/variables.tf +++ b/providers/do/variables.tf @@ -87,6 +87,11 @@ variable "runtime" { rootfs_archive_sha256 = optional(string, "") users = optional(map(list(string)), {}) + disaster_recovery = optional(object({ + required = optional(bool, false) + driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + }), {}) + compose = optional(object({ primary = optional(string, "") ingress_port = optional(number, 80) @@ -195,6 +200,15 @@ variable "runtime" { error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together." } + validation { + condition = ( + can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) && + !strcontains(var.runtime.disaster_recovery.driver_path, "//") && + length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0 + ) + error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments." + } + validation { condition = alltrue([ for name in keys(var.runtime.extra_env) : diff --git a/providers/gcp/main.tf b/providers/gcp/main.tf index 80a74c0..0ff9757 100644 --- a/providers/gcp/main.tf +++ b/providers/gcp/main.tf @@ -145,13 +145,15 @@ module "gcp" { overlay_source_instance = local.gcp_overlay.source_instance volume_names = local.gcp_overlay.volume_names - users = local.runtime.users - rootfs = local.runtime.rootfs - rootfs_archive_url = local.runtime.rootfs_archive_url - rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 - extra_env = local.runtime.extra_env - runcmd = local.gcp_cloud_init.runcmd - initcmd = local.gcp_cloud_init.initcmd + users = local.runtime.users + rootfs = local.runtime.rootfs + rootfs_archive_url = local.runtime.rootfs_archive_url + rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 + offhost_backup_required = local.runtime.disaster_recovery.required + offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path + extra_env = local.runtime.extra_env + runcmd = local.gcp_cloud_init.runcmd + initcmd = local.gcp_cloud_init.initcmd artifact_registry_repository = local.gcp_artifact_registry.repository artifact_registry_location = local.gcp_artifact_registry.location diff --git a/providers/gcp/variables.tf b/providers/gcp/variables.tf index 1bfadee..80a8a4a 100644 --- a/providers/gcp/variables.tf +++ b/providers/gcp/variables.tf @@ -199,6 +199,11 @@ variable "runtime" { rootfs_archive_sha256 = optional(string, "") users = optional(map(list(string)), {}) + disaster_recovery = optional(object({ + required = optional(bool, false) + driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + }), {}) + compose = optional(object({ primary = optional(string, "") ingress_port = optional(number, 80) @@ -308,6 +313,15 @@ variable "runtime" { error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together." } + validation { + condition = ( + can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) && + !strcontains(var.runtime.disaster_recovery.driver_path, "//") && + length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0 + ) + error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments." + } + validation { condition = alltrue([ for name in keys(var.runtime.extra_env) : diff --git a/providers/linode/variables.tf b/providers/linode/variables.tf index 725ce0b..4b29dc0 100644 --- a/providers/linode/variables.tf +++ b/providers/linode/variables.tf @@ -102,6 +102,11 @@ variable "runtime" { rootfs_archive_sha256 = optional(string, "") users = optional(map(list(string)), {}) + disaster_recovery = optional(object({ + required = optional(bool, false) + driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + }), {}) + compose = optional(object({ primary = optional(string, "") ingress_port = optional(number, 80) @@ -210,6 +215,15 @@ variable "runtime" { error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together." } + validation { + condition = ( + can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) && + !strcontains(var.runtime.disaster_recovery.driver_path, "//") && + length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0 + ) + error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments." + } + validation { condition = alltrue([ for name in keys(var.runtime.extra_env) : diff --git a/rootfs/etc/systemd/system/cloud-compose-mariadb-backup.timer b/rootfs/etc/systemd/system/cloud-compose-mariadb-backup.timer index ef56532..09362b8 100644 --- a/rootfs/etc/systemd/system/cloud-compose-mariadb-backup.timer +++ b/rootfs/etc/systemd/system/cloud-compose-mariadb-backup.timer @@ -1,5 +1,5 @@ [Unit] -Description=Run Cloud Compose MariaDB backups nightly +Description=Run Cloud Compose local backups and off-host DR handoff nightly [Timer] # 9pm-7am EST, splayed per deployment by systemd's stable random delay. @@ -8,7 +8,7 @@ Persistent=true RandomizedDelaySec=10h FixedRandomDelay=true AccuracySec=1min -Unit=cloud-compose-mariadb-backup.service +Unit=cloud-compose-offhost-backup.service [Install] WantedBy=timers.target diff --git a/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service b/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service new file mode 100644 index 0000000..9ae3aa2 --- /dev/null +++ b/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service @@ -0,0 +1,17 @@ +[Unit] +Description=Cloud Compose encrypted off-host disaster-recovery handoff +Requires=cloud-compose-mariadb-backup.service +After=cloud-compose-mariadb-backup.service network-online.target +Wants=network-online.target +RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes + +[Service] +Type=oneshot +User=root +Group=root +UMask=0077 +ExecStart=/bin/bash /home/cloud-compose/offhost-backup.sh +TimeoutStartSec=24h +StandardOutput=journal +StandardError=journal +SyslogIdentifier=cloud-compose-offhost-backup diff --git a/rootfs/etc/systemd/system/cloud-compose-restore-test.service b/rootfs/etc/systemd/system/cloud-compose-restore-test.service new file mode 100644 index 0000000..98dd9b1 --- /dev/null +++ b/rootfs/etc/systemd/system/cloud-compose-restore-test.service @@ -0,0 +1,16 @@ +[Unit] +Description=Cloud Compose disposable disaster-recovery restore test +After=network-online.target docker.service +Wants=network-online.target +RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes + +[Service] +Type=oneshot +User=root +Group=root +UMask=0077 +ExecStart=/bin/bash /home/cloud-compose/restore-test.sh +TimeoutStartSec=24h +StandardOutput=journal +StandardError=journal +SyslogIdentifier=cloud-compose-restore-test diff --git a/rootfs/etc/systemd/system/cloud-compose-restore-test.timer b/rootfs/etc/systemd/system/cloud-compose-restore-test.timer new file mode 100644 index 0000000..831f449 --- /dev/null +++ b/rootfs/etc/systemd/system/cloud-compose-restore-test.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Test Cloud Compose off-host recovery weekly + +[Timer] +OnCalendar=Sun *-*-* 03:00:00 +Persistent=true +RandomizedDelaySec=6h +FixedRandomDelay=true +AccuracySec=1min +Unit=cloud-compose-restore-test.service + +[Install] +WantedBy=timers.target diff --git a/rootfs/etc/tmpfiles.d/cloud-compose.conf b/rootfs/etc/tmpfiles.d/cloud-compose.conf index 9c5eb15..596c446 100644 --- a/rootfs/etc/tmpfiles.d/cloud-compose.conf +++ b/rootfs/etc/tmpfiles.d/cloud-compose.conf @@ -1,2 +1,3 @@ d /run/lock/cloud-compose 0750 root cloud-compose - f /run/lock/cloud-compose/lifecycle.lock 0660 root cloud-compose - +d /mnt/disks/data/.cloud-compose-disaster-recovery 0700 root root - diff --git a/rootfs/home/cloud-compose/disaster-recovery-lib.sh b/rootfs/home/cloud-compose/disaster-recovery-lib.sh new file mode 100644 index 0000000..f976e08 --- /dev/null +++ b/rootfs/home/cloud-compose/disaster-recovery-lib.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash + +# Shared validation for the provider-neutral disaster-recovery driver contract. +# The caller must enable `set -euo pipefail` before sourcing this file. + +CLOUD_COMPOSE_DR_STATE_ROOT="${CLOUD_COMPOSE_DR_STATE_ROOT:-/mnt/disks/data/.cloud-compose-disaster-recovery}" +CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="${CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER:-/usr/local/libexec/cloud-compose/offhost-backup-driver}" + +cloud_compose_dr_is_required() { + case "${CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED:-false}" in + true) return 0 ;; + false) return 1 ;; + *) + echo "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED must be true or false" >&2 + return 2 + ;; + esac +} + +cloud_compose_dr_validate_safe_absolute_path() { + local path="$1" label="$2" + + if [[ ! "$path" =~ ^/[A-Za-z0-9._/+:-]+$ || "$path" == *"//"* || + "$path" =~ (^|/)\.\.?(/|$) ]]; then + echo "$label must be a safe absolute path without whitespace or dot segments" >&2 + return 1 + fi +} + +cloud_compose_dr_validate_utc_timestamp() { + local value="$1" label="$2" + + if [[ ! "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then + echo "$label must be an RFC 3339 UTC timestamp with whole-second precision" >&2 + return 1 + fi +} + +cloud_compose_dr_validate_remote_id() { + local value="$1" label="$2" + + if [[ ! "$value" =~ ^[A-Za-z0-9][A-Za-z0-9._:/-]{0,511}$ ]]; then + echo "$label contains unsupported characters or exceeds 512 bytes" >&2 + return 1 + fi +} + +cloud_compose_dr_validate_driver() { + local driver="${1:-$CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER}" + local current component metadata owner mode kind links resolved + local -a components + + cloud_compose_dr_validate_safe_absolute_path "$driver" "Off-host backup driver path" || return 1 + if [[ -L "$driver" || ! -f "$driver" || ! -x "$driver" ]]; then + echo "Off-host backup driver must be a non-symlink executable file: $driver" >&2 + return 1 + fi + + resolved="$(realpath -e -- "$driver")" || return 1 + if [[ "$resolved" != "$driver" ]]; then + echo "Off-host backup driver path must not traverse symbolic links: $driver" >&2 + return 1 + fi + + IFS='/' read -r -a components <<<"${driver#/}" + current="/" + for component in "${components[@]:0:${#components[@]}-1}"; do + current="${current%/}/${component}" + if [[ -L "$current" || ! -d "$current" ]]; then + echo "Off-host backup driver parent must be a real directory: $current" >&2 + return 1 + fi + metadata="$(stat -c '%u:%a:%F' -- "$current")" || return 1 + IFS=: read -r owner mode kind <<<"$metadata" + if [[ "$owner" != "0" || "$kind" != "directory" || ! "$mode" =~ ^[0-7]{3,4}$ || + $((8#$mode & 0022)) -ne 0 ]]; then + echo "Off-host backup driver parents must be root-owned and not group/world writable: $current" >&2 + return 1 + fi + done + + metadata="$(stat -c '%u:%a:%h:%F' -- "$driver")" || return 1 + IFS=: read -r owner mode links kind <<<"$metadata" + if [[ "$owner" != "0" || "$links" != "1" || "$kind" != "regular file" || + ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then + echo "Off-host backup driver must be a single-link, root-owned executable that is not group/world writable: $driver" >&2 + return 1 + fi +} + +cloud_compose_dr_validate_json_file() { + local path="$1" label="$2" metadata owner mode links kind size + + if [[ -L "$path" || ! -f "$path" ]]; then + echo "$label is missing or unsafe" >&2 + return 1 + fi + metadata="$(stat -c '%u:%a:%h:%F' -- "$path")" || return 1 + IFS=: read -r owner mode links kind <<<"$metadata" + if [[ "$owner" != "0" || "$links" != "1" || "$kind" != "regular file" || + ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then + echo "$label must be a single-link, root-owned regular file that is not group/world writable" >&2 + return 1 + fi + size="$(wc -c <"$path")" || return 1 + if ((size < 2 || size > 65536)); then + echo "$label must contain between 2 and 65536 bytes" >&2 + return 1 + fi +} + +cloud_compose_dr_prepare_state_directory() { + local path="$1" metadata owner mode kind + + cloud_compose_dr_validate_safe_absolute_path "$path" "Disaster-recovery state path" || return 1 + if [[ -L "$path" || ( -e "$path" && ! -d "$path" ) ]]; then + echo "Disaster-recovery state path is unsafe: $path" >&2 + return 1 + fi + install -d -m 0700 -o root -g root -- "$path" || return 1 + metadata="$(stat -c '%u:%a:%F' -- "$path")" || return 1 + IFS=: read -r owner mode kind <<<"$metadata" + if [[ "$owner" != "0" || "$mode" != "700" || "$kind" != "directory" || -L "$path" ]]; then + echo "Disaster-recovery state directories must be real root-owned mode-0700 directories: $path" >&2 + return 1 + fi +} + +cloud_compose_dr_validate_backup_receipt() { + local path="$1" operation_id="$2" manifest_sha256="$3" completed_at remote_id + + cloud_compose_dr_validate_json_file "$path" "Off-host backup receipt" || return 1 + jq -e \ + --arg operation_id "$operation_id" \ + --arg manifest_sha256 "$manifest_sha256" ' + type == "object" and length == 10 and + .schema_version == 1 and + .kind == "cloud-compose.offhost-backup-receipt" and + .operation_id == $operation_id and + .manifest_sha256 == $manifest_sha256 and + .status == "succeeded" and + .encrypted == true and + .off_host == true and + (.completed_at | type == "string" and length == 20 and + (explode | all(.[]; . >= 32 and . != 127))) and + (.remote_id | type == "string" and length >= 1 and length <= 512 and + (explode | all(.[]; . >= 32 and . != 127))) and + (.coverage | type == "object" and length == 3 and + .database == true and + .application_files == true and + .volume_topology == true) + ' "$path" >/dev/null || { + echo "Off-host backup driver returned an invalid or incomplete coverage receipt" >&2 + return 1 + } + completed_at="$(jq -er '.completed_at' "$path")" || return 1 + remote_id="$(jq -er '.remote_id' "$path")" || return 1 + cloud_compose_dr_validate_utc_timestamp "$completed_at" "Off-host backup completion time" || return 1 + cloud_compose_dr_validate_remote_id "$remote_id" "Off-host backup remote id" || return 1 +} + +cloud_compose_dr_validate_restore_proof() { + local path="$1" test_id="$2" manifest_sha256="$3" receipt_sha256="$4" completed_at recovery_id + + cloud_compose_dr_validate_json_file "$path" "Restore-test proof" || return 1 + jq -e \ + --arg test_id "$test_id" \ + --arg manifest_sha256 "$manifest_sha256" \ + --arg receipt_sha256 "$receipt_sha256" ' + type == "object" and length == 13 and + .schema_version == 1 and + .kind == "cloud-compose.restore-test-proof" and + .test_id == $test_id and + .source_manifest_sha256 == $manifest_sha256 and + .source_receipt_sha256 == $receipt_sha256 and + .status == "succeeded" and + .disposable_recovery == true and + .recovery_destroyed == true and + .integrity_verified == true and + (.completed_at | type == "string" and length == 20 and + (explode | all(.[]; . >= 32 and . != 127))) and + (.recovery_id | type == "string" and length >= 1 and length <= 512 and + (explode | all(.[]; . >= 32 and . != 127))) and + (.coverage | type == "object" and length == 3 and + .database == true and + .application_files == true and + .volume_topology == true) and + (.source_encrypted == true) + ' "$path" >/dev/null || { + echo "Off-host backup driver returned an invalid restore-test proof" >&2 + return 1 + } + completed_at="$(jq -er '.completed_at' "$path")" || return 1 + recovery_id="$(jq -er '.recovery_id' "$path")" || return 1 + cloud_compose_dr_validate_utc_timestamp "$completed_at" "Restore-test completion time" || return 1 + cloud_compose_dr_validate_remote_id "$recovery_id" "Restore-test recovery id" || return 1 +} + +cloud_compose_dr_run_driver() { + local driver="$1" + shift + + # Driver credentials and configuration are installed and resolved by the + # operator-owned executable. Terraform-rendered host/application variables + # are deliberately absent, and driver output is never copied to the journal. + if ! env -i HOME=/root PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + "$driver" "$@" /dev/null 2>&1; then + echo "Off-host disaster-recovery driver failed; inspect its operator-owned diagnostics" >&2 + return 1 + fi +} diff --git a/rootfs/home/cloud-compose/offhost-backup.sh b/rootfs/home/cloud-compose/offhost-backup.sh new file mode 100644 index 0000000..704e622 --- /dev/null +++ b/rootfs/home/cloud-compose/offhost-backup.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" +compose_apps_path="${CLOUD_COMPOSE_COMPOSE_APPS_PATH:-$script_dir/compose-apps.sh}" +dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}" +# shellcheck disable=SC1090 +source "$profile_path" +# shellcheck disable=SC1090 +source "$compose_apps_path" +# shellcheck disable=SC1090 +source "$dr_library_path" + +if cloud_compose_dr_is_required; then + : +else + status=$? + if ((status == 1)); then + echo "Off-host disaster recovery is not required; local MariaDB dumps remain same-disk recovery artifacts only" + exit 0 + fi + exit "$status" +fi + +driver="$CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER" +backup_root="${MARIADB_BACKUP_ROOT:-/mnt/disks/data/backups/mariadb}" +data_root="${CLOUD_COMPOSE_DATA_ROOT:-/mnt/disks/data}" +volumes_root="${CLOUD_COMPOSE_VOLUMES_ROOT:-/mnt/disks/volumes}" +backup_date="$(date -u +%Y%m%d)" +operation_id="${backup_date}-${CLOUD_COMPOSE_INSTANCE_NAME:-cloud-compose}" +state_root="$CLOUD_COMPOSE_DR_STATE_ROOT" +manifest_dir="$state_root/manifests" +receipt_dir="$state_root/backup-receipts" +staging_root="$state_root/staging" +manifest_path="$manifest_dir/${operation_id}.json" +receipt_path="$receipt_dir/${operation_id}.json" +staging_dir="" + +cleanup() { + if [[ -n "$staging_dir" ]]; then + rm -rf -- "$staging_dir" + fi +} +trap cleanup EXIT + +cloud_compose_dr_validate_driver "$driver" +acquire_cloud_compose_lifecycle_lock offhost-backup + +for path in "$state_root" "$manifest_dir" "$receipt_dir" "$staging_root"; do + cloud_compose_dr_prepare_state_directory "$path" +done + +staging_dir="$(mktemp -d "$staging_root/.${operation_id}.XXXXXX")" +chmod 0700 "$staging_dir" +application_rows="$staging_dir/applications.jsonl" +: >"$application_rows" +chmod 0600 "$application_rows" + +apps=() +compose_app_names_array apps +if ((${#apps[@]} == 0)); then + echo "Off-host backup requires at least one compose application" >&2 + exit 1 +fi + +for app in "${apps[@]}"; do + source_compose_app_env "$app" + project_dir="$DOCKER_COMPOSE_DIR" + dump_path="${backup_root}/${app}/${backup_date}-${app}.sql.gz" + staged_dump="$staging_dir/${app}.sql.gz" + compose_config="$staging_dir/${app}.compose-config.json" + application_row="$staging_dir/${app}.coverage.json" + + validate_compose_project_dir "$project_dir" + if [[ -L "$project_dir" || ! -d "$project_dir" ]]; then + echo "Application checkout is missing or unsafe for ${app}: $project_dir" >&2 + exit 1 + fi + if [[ -L "$dump_path" || ! -f "$dump_path" || ! -s "$dump_path" ]]; then + echo "Required local MariaDB recovery artifact is missing or unsafe for ${app}" >&2 + exit 1 + fi + if [[ "$(stat -c '%h:%F' -- "$dump_path")" != "1:regular file" ]]; then + echo "Required local MariaDB recovery artifact must have one link for ${app}" >&2 + exit 1 + fi + + # Copy through an already-open descriptor into the root-only handoff. This + # prevents the privileged driver from following a later path replacement in + # the application-owned local-backup directory. + exec {dump_fd}<"$dump_path" + dump_identity="$(stat -Lc '%d:%i' -- "/proc/${BASHPID}/fd/${dump_fd}")" + if [[ "$(stat -c '%d:%i' -- "$dump_path")" != "$dump_identity" ]]; then + echo "Local MariaDB recovery artifact changed while opening for ${app}" >&2 + exit 1 + fi + cat <&"$dump_fd" >"$staged_dump" + exec {dump_fd}<&- + chmod 0400 "$staged_dump" + if [[ ! -s "$staged_dump" ]] || ! gzip -t -- "$staged_dump"; then + echo "Required local MariaDB recovery artifact is invalid for ${app}" >&2 + exit 1 + fi + dump_sha256="$(sha256sum "$staged_dump" | awk '{print $1}')" + dump_bytes="$(wc -c <"$staged_dump")" + + ( + cd -- "$project_dir" + umask 077 + docker compose config --format json >"$compose_config" + ) + if ! jq -e ' + type == "object" and + (.services | type == "object" and length > 0) and + all(.services | to_entries[]; + ((.value.volumes // []) | type == "array") and + all((.value.volumes // [])[]; + type == "object" and + (.type | type == "string") and + (.type == "volume" or .type == "bind" or .type == "tmpfs") and + ((.source // "") | type == "string") and + ((.target // "") | type == "string" and length > 0))) and + ((.volumes // {}) | type == "object") + ' "$compose_config" >/dev/null; then + echo "Docker Compose returned unsafe or unsupported volume topology for ${app}" >&2 + exit 1 + fi + if ! jq -e --arg data_root "$data_root" --arg volumes_root "$volumes_root" ' + all(.services[].volumes[]?; + .type != "bind" or + (.source | type == "string" and + (. == $data_root or startswith($data_root + "/") or + . == $volumes_root or startswith($volumes_root + "/")) and + (explode | all(.[]; . >= 32 and . != 127)) and + (contains("//") | not) and + length > 0)) + ' "$compose_config" >/dev/null; then + echo "Persistent bind topology escapes managed data roots for ${app}" >&2 + exit 1 + fi + while IFS= read -r bind_source; do + if [[ "$bind_source" =~ (^|/)\.\.?(/|$) ]]; then + echo "Persistent bind topology contains a dot segment for ${app}" >&2 + exit 1 + fi + done < <(jq -r '.services[].volumes[]? | select(.type == "bind") | .source' "$compose_config") + + jq -cS \ + --arg app "$app" \ + --arg project_dir "$project_dir" \ + --arg dump_path "$staged_dump" \ + --arg dump_sha256 "$dump_sha256" \ + --argjson dump_bytes "$dump_bytes" ' + { + name: $app, + databases: [{ + engine: "mariadb", + format: "sql.gz", + local_recovery_artifact: $dump_path, + sha256: $dump_sha256, + bytes: $dump_bytes + }], + application_files: { + roots: [$project_dir], + bind_mounts: [ + .services | to_entries[] as $service | + ($service.value.volumes // [])[] | + select(.type == "bind") | + {service: $service.key, source: .source, target: .target, read_only: (.read_only // false)} + ] | sort_by(.service, .source, .target) + }, + volume_topology: { + declared_named_volumes: ((.volumes // {}) | keys | sort), + service_mounts: [ + .services | to_entries[] as $service | + ($service.value.volumes // [])[] | + {service: $service.key, type: .type, source: (.source // ""), target: .target, read_only: (.read_only // false)} + ] | sort_by(.service, .type, .source, .target) + } + } + ' "$compose_config" >"$application_row" + cat "$application_row" >>"$application_rows" + rm -f -- "$compose_config" +done + +staged_manifest="$staging_dir/manifest.json" +jq -cS -s \ + --arg operation_id "$operation_id" \ + --arg backup_date "$backup_date" \ + --arg provider "${CLOUD_COMPOSE_PROVIDER:-unknown}" \ + --arg instance "${CLOUD_COMPOSE_INSTANCE_NAME:-cloud-compose}" ' + { + schema_version: 1, + kind: "cloud-compose.offhost-backup-manifest", + operation_id: $operation_id, + backup_date: $backup_date, + provider: $provider, + instance: $instance, + required_coverage: ["database", "application_files", "volume_topology"], + applications: (sort_by(.name)) + } +' "$application_rows" >"$staged_manifest" +chmod 0400 "$staged_manifest" + +if ! jq -e --argjson app_count "${#apps[@]}" ' + .schema_version == 1 and + .kind == "cloud-compose.offhost-backup-manifest" and + (.applications | type == "array" and length == $app_count) and + all(.applications[]; + (.name | type == "string" and length >= 1 and length <= 63 and + (explode | all(.[]; . >= 32 and . != 127))) and + (.databases | length == 1) and + (.databases[0].sha256 | type == "string" and length == 64 and + (explode | all(.[]; . >= 32 and . != 127))) and + (.databases[0].bytes | type == "number" and . > 0) and + (.application_files.roots | type == "array" and length > 0) and + (.application_files.bind_mounts | type == "array") and + (.volume_topology.declared_named_volumes | type == "array") and + (.volume_topology.service_mounts | type == "array")) +' "$staged_manifest" >/dev/null; then + echo "Generated off-host coverage manifest is incomplete" >&2 + exit 1 +fi +while IFS=$'\t' read -r manifest_app manifest_sha; do + if [[ ! "$manifest_app" =~ ^[a-z][a-z0-9-]*$ || ! "$manifest_sha" =~ ^[0-9a-f]{64}$ ]]; then + echo "Generated off-host coverage manifest contains an unsafe application name or digest" >&2 + exit 1 + fi +done < <(jq -r '.applications[] | [.name, .databases[0].sha256] | @tsv' "$staged_manifest") + +manifest_sha256="$(sha256sum "$staged_manifest" | awk '{print $1}')" +staged_receipt="$staging_dir/receipt.json" +cloud_compose_dr_run_driver "$driver" backup \ + --manifest "$staged_manifest" \ + --manifest-sha256 "$manifest_sha256" \ + --operation-id "$operation_id" \ + --receipt "$staged_receipt" +if [[ "$(sha256sum "$staged_manifest" | awk '{print $1}')" != "$manifest_sha256" ]]; then + echo "Off-host backup driver modified the immutable coverage manifest" >&2 + exit 1 +fi +cloud_compose_dr_validate_backup_receipt "$staged_receipt" "$operation_id" "$manifest_sha256" + +chmod 0640 "$staged_manifest" "$staged_receipt" +mv -f -- "$staged_manifest" "$manifest_path" +mv -f -- "$staged_receipt" "$receipt_path" +echo "Encrypted off-host disaster-recovery coverage proven for ${#apps[@]} application(s): $receipt_path" diff --git a/rootfs/home/cloud-compose/restore-test.sh b/rootfs/home/cloud-compose/restore-test.sh new file mode 100644 index 0000000..e3b5ac1 --- /dev/null +++ b/rootfs/home/cloud-compose/restore-test.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" +dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}" +# shellcheck disable=SC1090 +source "$profile_path" +# shellcheck disable=SC1090 +source "$dr_library_path" + +if cloud_compose_dr_is_required; then + : +else + status=$? + if ((status == 1)); then + echo "Off-host disaster recovery is not required; skipping scheduled restore test" + exit 0 + fi + exit "$status" +fi + +driver="$CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER" +state_root="$CLOUD_COMPOSE_DR_STATE_ROOT" +manifest_dir="$state_root/manifests" +receipt_dir="$state_root/backup-receipts" +proof_dir="$state_root/restore-proofs" +staging_root="$state_root/staging" +staging_dir="" + +cleanup() { + if [[ -n "$staging_dir" ]]; then + rm -rf -- "$staging_dir" + fi +} +trap cleanup EXIT + +cloud_compose_dr_validate_driver "$driver" +for path in "$state_root" "$manifest_dir" "$receipt_dir" "$proof_dir" "$staging_root"; do + cloud_compose_dr_prepare_state_directory "$path" +done + +latest_receipt="$(find "$receipt_dir" -xdev -maxdepth 1 -type f -name '*.json' -printf '%f\n' | sort | tail -n 1)" +if [[ -z "$latest_receipt" ]]; then + echo "Scheduled restore test requires a validated off-host backup receipt" >&2 + exit 1 +fi +operation_id="${latest_receipt%.json}" +if [[ ! "$operation_id" =~ ^[0-9]{8}-[a-z][a-z0-9-]*$ ]]; then + echo "Latest off-host backup receipt has an unsafe operation id" >&2 + exit 1 +fi +receipt_path="$receipt_dir/$latest_receipt" +manifest_path="$manifest_dir/$latest_receipt" +if [[ -L "$manifest_path" || ! -f "$manifest_path" ]]; then + echo "Scheduled restore test is missing the source coverage manifest" >&2 + exit 1 +fi +cloud_compose_dr_validate_json_file "$manifest_path" "Off-host backup manifest" +manifest_sha256="$(sha256sum "$manifest_path" | awk '{print $1}')" +cloud_compose_dr_validate_backup_receipt "$receipt_path" "$operation_id" "$manifest_sha256" +receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" + +test_id="$(date -u +%Y%m%dT%H%M%SZ)-$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" +staging_dir="$(mktemp -d "$staging_root/.restore-${test_id}.XXXXXX")" +chmod 0700 "$staging_dir" +staged_proof="$staging_dir/proof.json" + +cloud_compose_dr_run_driver "$driver" restore-test \ + --manifest "$manifest_path" \ + --backup-receipt "$receipt_path" \ + --source-manifest-sha256 "$manifest_sha256" \ + --source-receipt-sha256 "$receipt_sha256" \ + --test-id "$test_id" \ + --proof "$staged_proof" +cloud_compose_dr_validate_restore_proof \ + "$staged_proof" "$test_id" "$manifest_sha256" "$receipt_sha256" + +proof_path="$proof_dir/${test_id}.json" +chmod 0640 "$staged_proof" +mv -- "$staged_proof" "$proof_path" +echo "Disposable restore test proved database, application-file, and volume-topology recovery: $proof_path" diff --git a/rootfs/home/cloud-compose/run.sh b/rootfs/home/cloud-compose/run.sh index 470a72f..c4d5c11 100644 --- a/rootfs/home/cloud-compose/run.sh +++ b/rootfs/home/cloud-compose/run.sh @@ -113,4 +113,9 @@ else systemctl disable --now cloud-compose-docker-prune.timer cloud-compose-docker-prune.service >/dev/null 2>&1 || true fi systemctl enable --now cloud-compose-mariadb-backup.timer +if runtime_enabled "${CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED:-false}"; then + systemctl enable --now cloud-compose-restore-test.timer +else + systemctl disable --now cloud-compose-restore-test.timer cloud-compose-restore-test.service >/dev/null 2>&1 || true +fi cloud_compose_publish_marker "$durable_bootstrap_marker" diff --git a/runtime_contracts.tftest.hcl b/runtime_contracts.tftest.hcl index 1dfbf9d..7cdcf89 100644 --- a/runtime_contracts.tftest.hcl +++ b/runtime_contracts.tftest.hcl @@ -147,6 +147,56 @@ run "public_entrypoint_rejects_reserved_extra_environment" { expect_failures = [var.runtime] } +run "public_entrypoint_accepts_provider_neutral_disaster_recovery_driver" { + command = plan + + variables { + name = "root-contract" + cloud_provider = "gcp" + template = "wp" + gcp = { + project_id = "test-project" + project_number = "123456789" + } + runtime = { + disaster_recovery = { + required = true + driver_path = "/usr/local/libexec/cloud-compose/acme-offhost" + } + } + } + + assert { + condition = ( + local.runtime.disaster_recovery.required && + local.runtime.disaster_recovery.driver_path == "/usr/local/libexec/cloud-compose/acme-offhost" + ) + error_message = "The public entrypoint must preserve provider-neutral DR controls." + } +} + +run "public_entrypoint_rejects_unsafe_disaster_recovery_driver_path" { + command = plan + + variables { + name = "root-contract" + cloud_provider = "gcp" + template = "wp" + gcp = { + project_id = "test-project" + project_number = "123456789" + } + runtime = { + disaster_recovery = { + required = true + driver_path = "/tmp/driver with spaces" + } + } + } + + expect_failures = [var.runtime] +} + run "public_entrypoint_accepts_direct_cloud_run_proxy_depth" { command = plan diff --git a/salt/cloud-compose/README.md b/salt/cloud-compose/README.md index a9ebcbf..01f4614 100644 --- a/salt/cloud-compose/README.md +++ b/salt/cloud-compose/README.md @@ -65,6 +65,22 @@ empty list disables that phase and is preserved instead of restoring a default. Runtime feature switches must be YAML booleans, not quoted strings; ambiguous values are rejected before host mutation. +The provider-neutral disaster-recovery interface is shared with Terraform: + +```yaml +cloud_compose: + runtime: + disaster_recovery: + required: true + driver_path: /usr/local/libexec/cloud-compose/offhost-backup-driver +``` + +Install that executable and its credentials separately as root. The formula +renders only the boolean requirement and executable path; storage endpoints, +credentials, encryption keys, and retention policy must not be placed in +pillar. See [the driver contract](../../docs/disaster-recovery.md) for receipt +and scheduled restore-proof requirements. + Set `cloud_compose.runtime.rollout` to enable the same authenticated rollout listener used by Terraform. Supply a pinned HTTPS `release_url`, its lowercase `release_sha256`, an HTTPS `jwks_uri`, `jwt_audience`, and optional JSON-object diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls index a6ac016..6ff0a03 100644 --- a/salt/cloud-compose/init.sls +++ b/salt/cloud-compose/init.sls @@ -42,7 +42,8 @@ 'sitectl': runtime.get('sitectl', {}), 'docker': runtime.get('docker', {}), 'managed_runtime': runtime.get('managed_runtime', {}), - 'vault': runtime.get('vault', {}) + 'vault': runtime.get('vault', {}), + 'disaster_recovery': runtime.get('disaster_recovery', {}) } %} {% for section_name, section_value in runtime_sections.items() %} {% if section_value is not mapping %} @@ -54,6 +55,7 @@ {% set docker = runtime_sections.docker if runtime_sections.docker is mapping else {} %} {% set managed = runtime_sections.managed_runtime if runtime_sections.managed_runtime is mapping else {} %} {% set vault = runtime_sections.vault if runtime_sections.vault is mapping else {} %} +{% set disaster_recovery = runtime_sections.disaster_recovery if runtime_sections.disaster_recovery is mapping else {} %} {% set raw_rollout_service = runtime.get('rollout', {}) %} {% if raw_rollout_service is mapping %} {% set rollout_service = raw_rollout_service %} @@ -141,6 +143,14 @@ {% if vault.get('agent_enabled', False) %} {% set ignored = invalid_runtime_inputs.append('Vault Agent is currently supported only by Terraform providers; set vault.agent_enabled=false for Salt') %} {% endif %} +{% set offhost_backup_required = disaster_recovery.get('required', False) %} +{% set offhost_backup_driver = disaster_recovery.get('driver_path', '/usr/local/libexec/cloud-compose/offhost-backup-driver') %} +{% if offhost_backup_required is not boolean %} +{% set ignored = invalid_runtime_inputs.append('runtime.disaster_recovery.required must be a boolean') %} +{% endif %} +{% if offhost_backup_driver is not string or not (offhost_backup_driver is match('^/[A-Za-z0-9._/+:-]+$')) or '//' in offhost_backup_driver or '/./' in offhost_backup_driver or '/../' in offhost_backup_driver or offhost_backup_driver.endswith('/.') or offhost_backup_driver.endswith('/..') %} +{% set ignored = invalid_runtime_inputs.append('runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments') %} +{% endif %} {% set rollout_enabled = rollout_service.get('enabled', False) %} {% set rollout_port = rollout_service.get('port', 8081) %} {% if rollout_enabled is not boolean %} @@ -464,6 +474,8 @@ 'CLOUD_COMPOSE_INSTANCE_NAME': name, 'CLOUD_COMPOSE_APPS': compose_projects.keys() | list | join(' '), 'CLOUD_COMPOSE_PRIMARY_APP': primary_key, + 'CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED': 'true' if offhost_backup_required is sameas true else 'false', + 'CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER': offhost_backup_driver, 'COMPOSE_PROJECTS_FILE': home ~ '/compose-projects.json', 'COMPOSE_PROJECT_NAME': primary_project.get('compose_project_name', compose_project_name), 'COMPOSE_BIND_PORT': primary_project.get('ingress_port', ingress_port), diff --git a/tests/config-management/ansible/invalid-disaster-recovery.yml b/tests/config-management/ansible/invalid-disaster-recovery.yml new file mode 100644 index 0000000..896513c --- /dev/null +++ b/tests/config-management/ansible/invalid-disaster-recovery.yml @@ -0,0 +1,16 @@ +- name: Reject unsafe disaster-recovery driver settings + hosts: localhost + gather_facts: true + vars: + cloud_compose_name: invalid-disaster-recovery + cloud_compose_template: wp + cloud_compose_dedicated_host_acknowledged: true + cloud_compose_install_packages: false + cloud_compose_reload_systemd: false + cloud_compose_run_bootstrap: false + cloud_compose_runtime: + disaster_recovery: + required: true + driver_path: /usr/local/libexec/cloud-compose/../untrusted + roles: + - role: cloud_compose diff --git a/tests/config-management/ansible/smoke.yml b/tests/config-management/ansible/smoke.yml index 57813e0..6614299 100644 --- a/tests/config-management/ansible/smoke.yml +++ b/tests/config-management/ansible/smoke.yml @@ -12,6 +12,9 @@ cloud_compose_internal_services_enabled: true cloud_compose_internal_services_auto_update: true cloud_compose_runtime: + disaster_recovery: + required: false + driver_path: /usr/local/libexec/cloud-compose/ansible-offhost extra_env: BASH_ENV: /tmp/cloud-compose-ansible-untrusted-bash-env LD_PRELOAD: /tmp/cloud-compose-ansible-untrusted-preload.so @@ -94,6 +97,10 @@ that: - >- 'CLOUD_COMPOSE_PROVIDER="onprem"' in (cloud_compose_env_file.content | b64decode) + - >- + 'CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED="false"' in (cloud_compose_env_file.content | b64decode) + - >- + 'CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="/usr/local/libexec/cloud-compose/ansible-offhost"' in (cloud_compose_env_file.content | b64decode) - >- 'DOCKER_COMPOSE_DIR="/mnt/disks/data/libops/isle/isle-prod"' in (cloud_compose_env_file.content | b64decode) - >- diff --git a/tests/config-management/salt-pillar/invalid-disaster-recovery.sls b/tests/config-management/salt-pillar/invalid-disaster-recovery.sls new file mode 100644 index 0000000..a9ca01c --- /dev/null +++ b/tests/config-management/salt-pillar/invalid-disaster-recovery.sls @@ -0,0 +1,12 @@ +cloud_compose: + name: invalid-disaster-recovery + provider: onprem + template: wp + dedicated_host_acknowledged: true + install_packages: false + reload_systemd: false + run_bootstrap: false + runtime: + disaster_recovery: + required: true + driver_path: /usr/local/libexec/cloud-compose/../untrusted diff --git a/tests/config-management/salt-pillar/top.sls b/tests/config-management/salt-pillar/top.sls index e652068..72b983b 100644 --- a/tests/config-management/salt-pillar/top.sls +++ b/tests/config-management/salt-pillar/top.sls @@ -29,6 +29,8 @@ base: - invalid-template 'invalid-vault': - invalid-vault + 'invalid-disaster-recovery': + - invalid-disaster-recovery 'invalid-package': - invalid-package 'invalid-host-ack': diff --git a/tests/config-management/salt-pillar/wp-prod.sls b/tests/config-management/salt-pillar/wp-prod.sls index cc2526e..54d794f 100644 --- a/tests/config-management/salt-pillar/wp-prod.sls +++ b/tests/config-management/salt-pillar/wp-prod.sls @@ -10,6 +10,9 @@ cloud_compose: internal_services_auto_update: true template: wp runtime: + disaster_recovery: + required: false + driver_path: /usr/local/libexec/cloud-compose/salt-offhost extra_env: BASH_ENV: /tmp/cloud-compose-salt-untrusted-bash-env LD_PRELOAD: /tmp/cloud-compose-salt-untrusted-preload.so diff --git a/variables.tf b/variables.tf index 9e1c074..23f2c11 100644 --- a/variables.tf +++ b/variables.tf @@ -210,6 +210,11 @@ variable "runtime" { rootfs_archive_sha256 = optional(string, "") users = optional(map(list(string)), {}) + disaster_recovery = optional(object({ + required = optional(bool, false) + driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + }), {}) + compose = optional(object({ primary = optional(string, "") ingress_port = optional(number, 80) @@ -319,6 +324,15 @@ variable "runtime" { error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together." } + validation { + condition = ( + can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) && + !strcontains(var.runtime.disaster_recovery.driver_path, "//") && + length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0 + ) + error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments." + } + validation { condition = alltrue([ for name in keys(var.runtime.extra_env) : From 2361abf792bbac97dbcc8d7daf9ce78dfd8d6cd4 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:17:05 +0000 Subject: [PATCH 02/67] [patch] Restore required DigitalOcean ISLE smoke --- .github/workflows/cloud-smoke-cleanup.yml | 6 ++++++ .github/workflows/cloud-smoke.yml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/cloud-smoke-cleanup.yml b/.github/workflows/cloud-smoke-cleanup.yml index 4d62288..6cdb48c 100644 --- a/.github/workflows/cloud-smoke-cleanup.yml +++ b/.github/workflows/cloud-smoke-cleanup.yml @@ -19,6 +19,12 @@ jobs: fail-fast: false matrix: include: + - name: DigitalOcean ISLE + kind: app + provider: digitalocean + template: isle + environment: cloud-smoke-cleanup-digitalocean + concurrency_group: cloud-compose-smoke-digitalocean-isle - name: DigitalOcean WordPress kind: app provider: digitalocean diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml index 26a5cd6..5b38dd8 100644 --- a/.github/workflows/cloud-smoke.yml +++ b/.github/workflows/cloud-smoke.yml @@ -126,6 +126,9 @@ jobs: fail-fast: false matrix: include: + - name: DigitalOcean ISLE + provider: digitalocean + template: isle - name: DigitalOcean WordPress provider: digitalocean template: wp From 67bc19ae5171083ad1db490af46c823a17935916 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 13:02:41 +0000 Subject: [PATCH 03/67] [patch] Allow ISLE bootstrap to complete in smoke --- .github/workflows/cloud-smoke.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml index 5b38dd8..8631f66 100644 --- a/.github/workflows/cloud-smoke.yml +++ b/.github/workflows/cloud-smoke.yml @@ -129,17 +129,21 @@ jobs: - name: DigitalOcean ISLE provider: digitalocean template: isle + boot_timeout: "3600" - name: DigitalOcean WordPress provider: digitalocean template: wp + boot_timeout: "1200" - name: Linode WordPress provider: linode template: wp + boot_timeout: "1200" concurrency: group: cloud-compose-smoke-${{ matrix.provider }}-${{ matrix.template }} cancel-in-progress: false env: CLOUD_COMPOSE_SMOKE_AUTO_APPROVE: "true" + CLOUD_COMPOSE_SMOKE_BOOT_TIMEOUT: ${{ matrix.boot_timeout }} CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "1800" CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS: "true" CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }} From 909d6b415cf002106354a6a65e78beef6e3c209f Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 18:58:23 +0000 Subject: [PATCH 04/67] [patch] Make provisioning readiness observable --- .github/workflows/cloud-smoke.yml | 4 +- ci/cloud-smoke.sh | 204 ++++++++++-------- ci/gcp-upgrade-smoke-contract.sh | 35 ++- ci/systemd-contract.sh | 12 +- modules/gcp/main.tf | 1 + modules/linux-vm-runtime/main.tf | 1 + .../runtime_inputs.tftest.hcl | 11 + .../linux-vm-runtime/templates/cloud-init.yml | 19 +- .../etc/systemd/system/cloud-compose.service | 5 +- .../home/cloud-compose/smoke-healthcheck.sh | 17 ++ .../local/sbin/cloud-compose-diagnostics.sh | 200 +++++++++++++++++ templates/cloud-init.yml | 12 ++ 12 files changed, 414 insertions(+), 107 deletions(-) create mode 100755 rootfs/home/cloud-compose/smoke-healthcheck.sh create mode 100755 rootfs/usr/local/sbin/cloud-compose-diagnostics.sh diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml index 8631f66..e011fb3 100644 --- a/.github/workflows/cloud-smoke.yml +++ b/.github/workflows/cloud-smoke.yml @@ -129,7 +129,9 @@ jobs: - name: DigitalOcean ISLE provider: digitalocean template: isle - boot_timeout: "3600" + # Exceed the 90-minute application service ceiling with enough + # room to observe the terminal unit state before cleanup. + boot_timeout: "6000" - name: DigitalOcean WordPress provider: digitalocean template: wp diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh index 97a18fb..77164cc 100755 --- a/ci/cloud-smoke.sh +++ b/ci/cloud-smoke.sh @@ -3,6 +3,8 @@ set -euo pipefail repo_root="$(cd "$(dirname "$0")/.." && pwd)" +readonly diagnostics_program="/usr/local/sbin/cloud-compose-diagnostics.sh" +readonly smoke_healthcheck_program="/home/cloud-compose/smoke-healthcheck.sh" usage() { cat <<'EOF' @@ -377,42 +379,36 @@ wait_for_ssh() { return 1 } +remote_diagnostics_available() { + local home_dir="$1" key_path="$2" host="$3" port="$4" user="$5" + + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "test -x ${diagnostics_program}" >/dev/null 2>&1 +} + remote_bootstrap_state() { local home_dir="$1" key_path="$2" host="$3" port="$4" user="$5" - ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set +e -if [ -f /home/cloud-compose/.cloud-compose-bootstrap-complete ]; then - echo complete - exit 0 -fi -bootstrap_load_state=\"\$(systemctl show --property=LoadState --value -- cloud-compose-bootstrap.service 2>/dev/null)\" -if [ \"\$bootstrap_load_state\" = loaded ]; then - bootstrap_active_state=\"\$(systemctl show --property=ActiveState --value -- cloud-compose-bootstrap.service 2>/dev/null)\" - bootstrap_sub_state=\"\$(systemctl show --property=SubState --value -- cloud-compose-bootstrap.service 2>/dev/null)\" - case \"\$bootstrap_active_state:\$bootstrap_sub_state\" in - active:* | activating:* | *:auto-restart) - echo active - exit 0 - ;; - esac -elif [ \"\$bootstrap_load_state\" = not-found ] && - systemctl is-active --quiet cloud-compose; then - # Releases before retryable bootstrap have no durable unit or marker; their - # active application service remains the compatibility completion signal. - echo complete - exit 0 -fi -if systemctl is-active --quiet cloud-final.service; then - echo active - exit 0 -fi -if pgrep -f \"[/]home/cloud-compose/run[.]sh|[/]home/cloud-compose/[h]ost-conf[.]sh|[/]home/cloud-compose/[h]ost-init[.]sh|[/]home/cloud-compose/[a]pp-init[.]sh|[/]home/cloud-compose/[i]nstall-dependencies|[a]pt-get|[r]pm-ostree|[d]ocker run|[s]itectl|[g]it clone\" >/dev/null; then - echo active - exit 0 -fi -echo idle -exit 1 -'" 2>/dev/null || true + if remote_diagnostics_available "$home_dir" "$key_path" "$host" "$port" "$user"; then + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "sudo -n ${diagnostics_program} state" 2>/dev/null || true + return + fi + + # The pinned upgrade fixture predates the checked-in diagnostics program. + # Keep its compatibility probes simple and non-interactive. + if ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "test -f /home/cloud-compose/.cloud-compose-bootstrap-complete" >/dev/null 2>&1; then + echo complete + elif ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "systemctl is-active --quiet cloud-compose.service" >/dev/null 2>&1; then + echo complete + elif ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "systemctl is-active --quiet cloud-final.service" >/dev/null 2>&1; then + echo active + else + echo idle + fi } wait_for_cloud_init() { @@ -421,25 +417,47 @@ wait_for_cloud_init() { local deadline local status_output local bootstrap_state - local last_dump=0 + local diagnostics_available timeout_seconds="$(boot_timeout_seconds)" deadline=$((SECONDS + timeout_seconds)) echo "Waiting for cloud-init on ${host}" while (( SECONDS < deadline )); do + diagnostics_available=false + if remote_diagnostics_available "$home_dir" "$key_path" "$host" "$port" "$user"; then + diagnostics_available=true + fi bootstrap_state="$(remote_bootstrap_state "$home_dir" "$key_path" "$host" "$port" "$user")" if [[ "$bootstrap_state" == "complete" ]]; then return 0 fi - status_output="$( - ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ - "if command -v cloud-init >/dev/null 2>&1; then sudo cloud-init status --long 2>&1; else echo 'cloud-init not installed'; fi" 2>&1 || true - )" + if [[ "$diagnostics_available" == "true" ]]; then + status_output="$( + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "sudo -n ${diagnostics_program} status" 2>&1 || true + )" + else + status_output="$( + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "cloud-init status --long" 2>&1 || true + )" + fi printf '%s\n' "$status_output" if grep -q '^status: done' <<<"$status_output"; then + if [[ "$diagnostics_available" == "true" ]]; then + if [[ "$bootstrap_state" == "active" ]]; then + echo "cloud-init is done while cloud-compose bootstrap is still active; continuing" + sleep 30 + continue + fi + echo "cloud-init completed without the Cloud Compose readiness marker" >&2 + return 1 + fi + # The pinned upgrade fixture predates the durable readiness marker. + # Retain cloud-init completion as its final compatibility signal. return 0 fi if grep -q '^status: error' <<<"$status_output"; then @@ -451,24 +469,6 @@ wait_for_cloud_init() { return 1 fi - if (( SECONDS - last_dump >= 120 )); then - last_dump=$SECONDS - ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set +e -echo \"--- active bootstrap processes ---\" -ps -eo pid,ppid,stat,etime,args | grep -E \"cloud-init|runcmd|run.sh|host-conf|host-init|app-init|install-dependencies|apt-get|docker|sitectl|git clone\" | grep -v grep -echo \"--- cloud-compose bootstrap unit ---\" -sudo journalctl -u cloud-compose-bootstrap --no-pager -n 160 -echo \"--- legacy cloud-compose bootstrap log (when present) ---\" -if sudo test -f /home/cloud-compose/run.log && sudo test ! -L /home/cloud-compose/run.log; then - sudo tail -n 160 /home/cloud-compose/run.log -else - echo \"Legacy bootstrap log is not present\" -fi -echo \"--- /var/log/cloud-init-output.log ---\" -sudo tail -n 120 /var/log/cloud-init-output.log -'" || true - fi - sleep 30 done @@ -482,40 +482,50 @@ dump_remote_logs() { quoted_project_dir="$(shell_quote "$project_dir")" echo "Dumping smoke-test diagnostics from ${host}" >&2 - ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set +e -echo \"--- cloud-init status ---\" -sudo cloud-init status --long -echo \"--- /var/log/cloud-init-output.log ---\" -sudo tail -n 400 /var/log/cloud-init-output.log -echo \"--- /var/log/cloud-init.log ---\" -sudo tail -n 400 /var/log/cloud-init.log -echo \"--- cloud-init runcmd ---\" -sudo sed -n '1,240p' /var/lib/cloud/instance/scripts/runcmd -echo \"--- cloud-compose bootstrap unit ---\" -sudo journalctl -u cloud-compose-bootstrap --no-pager -n 400 -echo \"--- legacy cloud-compose bootstrap log (when present) ---\" -if sudo test -f /home/cloud-compose/run.log && sudo test ! -L /home/cloud-compose/run.log; then - sudo tail -n 400 /home/cloud-compose/run.log -else - echo \"Legacy bootstrap log is not present\" -fi -echo \"--- cloud-compose unit ---\" -sudo journalctl -u cloud-compose --no-pager -n 300 -echo \"--- lifecycle lock permissions ---\" -sudo stat -Lc '%A %a %U:%G %u:%g %n' /run/lock/cloud-compose /run/lock/cloud-compose/lifecycle.lock -echo \"--- docker ps ---\" -sudo docker ps -a -echo \"--- docker compose ps ---\" -if [ -d ${quoted_project_dir} ]; then - if command -v runuser >/dev/null 2>&1; then - runuser -u cloud-compose -- env HOME=/home/cloud-compose PROJECT_DIR=${quoted_project_dir} bash -lc \"source /home/cloud-compose/profile.sh && cd \\\"\$PROJECT_DIR\\\" && docker compose ps\" - else - sudo -u cloud-compose env HOME=/home/cloud-compose PROJECT_DIR=${quoted_project_dir} bash -lc \"source /home/cloud-compose/profile.sh && cd \\\"\$PROJECT_DIR\\\" && docker compose ps\" + if remote_diagnostics_available "$home_dir" "$key_path" "$host" "$port" "$user"; then + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "sudo -n ${diagnostics_program} dump" || true + return fi -else - echo \"Project directory ${project_dir} is not present yet\" -fi -'" || true + + # Compatibility diagnostics for the pinned pre-program upgrade fixture. + echo "--- legacy cloud-init status ---" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "cloud-init status --long" || true + echo "--- legacy /var/log/cloud-init-output.log ---" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "tail -n 400 /var/log/cloud-init-output.log" || true + echo "--- legacy /var/log/cloud-init.log ---" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "tail -n 400 /var/log/cloud-init.log" || true + echo "--- legacy cloud-init runcmd ---" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "sed -n 1,240p /var/lib/cloud/instance/scripts/runcmd" || true + echo "--- legacy cloud-compose bootstrap unit ---" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "systemctl status cloud-compose-bootstrap.service --no-pager" || true + echo "--- legacy cloud-compose bootstrap log ---" + if ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "test -f /home/cloud-compose/run.log" >/dev/null 2>&1 && + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "test ! -L /home/cloud-compose/run.log" >/dev/null 2>&1; then + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "tail -n 400 /home/cloud-compose/run.log" || true + else + echo "Legacy bootstrap log is not present" + fi + echo "--- legacy cloud-compose unit ---" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "sudo -n /usr/bin/systemctl status cloud-compose.service" || true + echo "--- legacy lifecycle lock permissions ---" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "stat -Lc '%A %a %U:%G %u:%g %n' /run/lock/cloud-compose /run/lock/cloud-compose/lifecycle.lock" || true + echo "--- legacy docker ps ---" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "docker ps -a" || true + echo "--- legacy docker compose ps ---" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "docker compose --project-directory ${quoted_project_dir} ps" || true } configure_sitectl_context() { @@ -564,11 +574,17 @@ run_healthcheck() { user="$(jq -r '.ssh_user' "$output_json")" quoted_context="$(shell_quote "$context")" - ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set -euo pipefail -export HOME=/home/cloud-compose -source /home/cloud-compose/profile.sh -exec sitectl healthcheck --context ${quoted_context} --persist --format table -'" + if ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "test -x ${smoke_healthcheck_program}" >/dev/null 2>&1; then + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "${smoke_healthcheck_program} ${quoted_context}" + else + # The pinned 0.10.2 upgrade fixture predates the checked-in wrapper. + # Invoke its sitectl binary directly with the environment profile's + # stable path settings, without sending an embedded shell program. + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "env HOME=/home/cloud-compose DOCKER_CONFIG=/mnt/disks/data/docker-config PATH=/home/cloud-compose/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin sitectl healthcheck --context ${quoted_context} --persist --format table" + fi return fi diff --git a/ci/gcp-upgrade-smoke-contract.sh b/ci/gcp-upgrade-smoke-contract.sh index 6d02628..adb84ec 100755 --- a/ci/gcp-upgrade-smoke-contract.sh +++ b/ci/gcp-upgrade-smoke-contract.sh @@ -47,16 +47,33 @@ grep -Fq "provider_tag_cleanup gcp-wp \"\$run_id\"" "$script" || fail "upgrade runner does not finish cleanup with the verified provider sweep" grep -Fq 'target_env gcp-wp' "$script" || fail "upgrade cleanup does not load the concrete GCP WordPress target environment" -[[ "$(grep -Fc 'sudo test -f /home/cloud-compose/run.log && sudo test ! -L /home/cloud-compose/run.log' "$shared_smoke")" -eq 2 ]] || - fail "shared smoke diagnostics do not retain guarded legacy bootstrap logs for the pinned baseline" -grep -Fq 'bootstrap_load_state=\"\$(systemctl show --property=LoadState --value -- cloud-compose-bootstrap.service' "$shared_smoke" || - fail "shared smoke readiness does not distinguish retryable bootstrap from the legacy baseline" -grep -Fq 'active:* | activating:* | *:auto-restart)' "$shared_smoke" || - fail "shared smoke readiness can abandon the bootstrap unit during its restart delay" -grep -Fq 'elif [ \"\$bootstrap_load_state\" = not-found ] &&' "$shared_smoke" || - fail "shared smoke readiness does not restrict legacy fallback to an absent bootstrap unit" -grep -Fq 'systemctl is-active --quiet cloud-compose; then' "$shared_smoke" || +grep -Fq 'readonly diagnostics_program="/usr/local/sbin/cloud-compose-diagnostics.sh"' "$shared_smoke" || + fail "shared smoke diagnostics do not use the checked-in privileged program" +for diagnostics_command in state status dump; do + grep -Fq "sudo -n \${diagnostics_program} ${diagnostics_command}" "$shared_smoke" || + fail "shared smoke diagnostics do not invoke the ${diagnostics_command} command non-interactively" +done +grep -Fq 'test -f /home/cloud-compose/.cloud-compose-bootstrap-complete' "$shared_smoke" || + fail "shared smoke readiness lost its pinned-baseline marker compatibility probe" +grep -Fq 'systemctl is-active --quiet cloud-compose.service' "$shared_smoke" || fail "shared smoke readiness lost its pre-bootstrap-unit compatibility signal" +grep -Fq 'cloud-init completed without the Cloud Compose readiness marker' "$shared_smoke" || + fail "current smoke readiness can still accept cloud-init completion without durable application readiness" +grep -Fq 'The pinned upgrade fixture predates the durable readiness marker.' "$shared_smoke" || + fail "legacy cloud-init completion is not explicitly confined to the pinned upgrade fixture" +grep -Fq 'test ! -L /home/cloud-compose/run.log' "$shared_smoke" || + fail "shared smoke diagnostics do not retain a guarded legacy bootstrap log fallback" +grep -Fq 'sudo -n /usr/bin/systemctl status cloud-compose.service' "$shared_smoke" || + fail "shared smoke diagnostics do not limit pinned-baseline sudo to its existing exact command" +grep -Fq 'readonly smoke_healthcheck_program="/home/cloud-compose/smoke-healthcheck.sh"' "$shared_smoke" || + fail "shared smoke healthcheck does not use the checked-in host wrapper" +grep -Fq '"${smoke_healthcheck_program} ${quoted_context}"' "$shared_smoke" || + fail "shared smoke healthcheck does not invoke the checked-in host wrapper directly" +grep -Fq 'env HOME=/home/cloud-compose DOCKER_CONFIG=/mnt/disks/data/docker-config PATH=/home/cloud-compose/bin:' "$shared_smoke" || + fail "shared smoke healthcheck lost its direct pinned-baseline fallback" +if sed -n '/^run_healthcheck()/,/^}/p' "$shared_smoke" | grep -Fq 'bash -lc'; then + fail "shared smoke healthcheck still sends an embedded Bash program over SSH" +fi grep -Fq 'CLOUD_COMPOSE_SMOKE_RUN_ID must match GITHUB_RUN_ID in GitHub Actions' "$script" || fail "hosted cleanup ownership is not bound to the actual GitHub run id" grep -Fq 'CLOUD_COMPOSE_SMOKE_RUN_ID must be set explicitly outside GitHub Actions' "$script" || diff --git a/ci/systemd-contract.sh b/ci/systemd-contract.sh index dc00466..da2550d 100755 --- a/ci/systemd-contract.sh +++ b/ci/systemd-contract.sh @@ -4,6 +4,8 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" unit_dir="$repo_root/rootfs/etc/systemd/system" +diagnostics_program="$repo_root/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh" +smoke_healthcheck_program="$repo_root/rootfs/home/cloud-compose/smoke-healthcheck.sh" fail() { echo "systemd contract: $*" >&2 @@ -25,6 +27,8 @@ assert_contains() { [[ -f "$unit_dir/cloud-compose-vault-agent.service" ]] || fail "namespaced Vault Agent unit is missing" [[ -f "$unit_dir/cloud-compose-overlay.service" ]] || fail "Docker overlay mount unit is missing" [[ -f "$unit_dir/cloud-compose-bootstrap.service" ]] || fail "retryable bootstrap unit is missing" +[[ -x "$diagnostics_program" ]] || fail "checked-in Cloud Compose diagnostics program is missing or not executable" +[[ -x "$smoke_healthcheck_program" ]] || fail "checked-in smoke healthcheck wrapper is missing or not executable" [[ -f "$unit_dir/cloud-compose-internal-services.service" && -f "$unit_dir/cloud-compose-internal-services.timer" ]] || \ fail "namespaced internal-services units are missing" [[ -f "$unit_dir/cloud-compose-offhost-backup.service" ]] || fail "off-host backup service is missing" @@ -51,7 +55,13 @@ assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'SyslogLevelPrefix=n assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'LogRateLimitIntervalSec=30s' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'LogRateLimitBurst=1000' assert_contains "$unit_dir/cloud-compose-internal-services.service" 'Requires=cloud-compose.service cloud-compose-metadata-firewall.service' -assert_contains "$unit_dir/cloud-compose.service" 'TimeoutStartSec=1h' +assert_contains "$unit_dir/cloud-compose.service" 'TimeoutStartSec=90min' +assert_contains "$diagnostics_program" 'usage: ${diagnostics_program} state|status|dump' +assert_contains "$diagnostics_program" 'readonly PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"' +assert_contains "$diagnostics_program" '--- Cloud Compose provisioning heartbeat ---' +assert_contains "$diagnostics_program" 'ps -p "$main_pid" -o pid=,ppid=,stat=,etime=,comm=' +assert_contains "$smoke_healthcheck_program" 'source /home/cloud-compose/profile.sh' +assert_contains "$smoke_healthcheck_program" 'exec sitectl healthcheck --context "$context" --persist --format table' assert_contains "$unit_dir/cloud-compose-mariadb-backup.service" 'TimeoutStartSec=12h' assert_contains "$unit_dir/cloud-compose-mariadb-backup.service" 'User=cloud-compose' assert_contains "$unit_dir/cloud-compose-mariadb-backup.timer" 'Unit=cloud-compose-offhost-backup.service' diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index e7b03ba..1831ff7 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -464,6 +464,7 @@ rollout_runcmd = var.rollout_enabled ? [ "bash /home/cloud-compose/deploy-rollout.sh >> /home/cloud-compose/run.log 2>&1", ] : [] cloud_init_yaml = templatefile("${path.module}/../../templates/cloud-init.yml", { + DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootFs}/usr/local/sbin/cloud-compose-diagnostics.sh"), FILESYSTEM_PREP_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/prepare-filesystem.sh"), FILESYSTEM_PERSIST_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/persist-filesystems.sh"), FRESH_FILESYSTEM_IDENTITY = "v1:gcp-disk-id:${google_compute_disk.data.disk_id}", diff --git a/modules/linux-vm-runtime/main.tf b/modules/linux-vm-runtime/main.tf index 8632ee9..b4a4502 100644 --- a/modules/linux-vm-runtime/main.tf +++ b/modules/linux-vm-runtime/main.tf @@ -333,6 +333,7 @@ rootfs_archive_install_command = local.rootfs_archive_url != "" ? local.rootfs_a cloud_init = templatefile("${path.module}/templates/cloud-init.yml", { CLOUD_COMPOSE_SSH_KEYS = var.cloud_compose_ssh_keys SSH_USERS = var.ssh_users + DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootfs}/usr/local/sbin/cloud-compose-diagnostics.sh") DATA_DEVICE = var.data_device VOLUMES_DEVICE = var.volumes_device WRITE_FILES_CONTENT = local.write_files_content diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index 26288d4..dd72833 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -25,6 +25,17 @@ run "renders_safe_ssh_values" { condition = strcontains(output.cloud_init, "name: ${jsonencode("app_operator")}") error_message = "SSH usernames must be YAML encoded as literal values." } + + assert { + condition = ( + strcontains(output.cloud_init, "path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh") && + strcontains(output.cloud_init, "NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh state") && + strcontains(output.cloud_init, "NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh status") && + strcontains(output.cloud_init, "NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh dump") && + strcontains(output.cloud_init, "install -m 0755 -o root -g root") + ) + error_message = "Cloud-init must install one root-owned diagnostics program with exact passwordless sudo commands." + } } run "normalizes_minimal_compose_project" { diff --git a/modules/linux-vm-runtime/templates/cloud-init.yml b/modules/linux-vm-runtime/templates/cloud-init.yml index 1cea770..9e34e12 100644 --- a/modules/linux-vm-runtime/templates/cloud-init.yml +++ b/modules/linux-vm-runtime/templates/cloud-init.yml @@ -10,6 +10,9 @@ users: - docker sudo: - ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service + - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh state + - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh status + - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh dump %{ if length(CLOUD_COMPOSE_SSH_KEYS) > 0 ~} ssh_authorized_keys: %{ for key in CLOUD_COMPOSE_SSH_KEYS ~} @@ -26,6 +29,11 @@ users: %{ endfor ~} write_files: +- path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh + owner: root:root + permissions: "0600" + encoding: b64 + content: ${DIAGNOSTICS_SCRIPT_B64} ${WRITE_FILES_CONTENT} ${DOCKER_COMPOSE_SCRIPTS} ${COMPOSE_PROJECTS_FILE} @@ -55,7 +63,12 @@ runcmd: chown cloud-compose:cloud-compose /home/cloud-compose/.ssh/authorized_keys chmod 0600 /home/cloud-compose/.ssh/authorized_keys install -d -m 0755 /etc/sudoers.d - printf 'cloud-compose ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service\n' >/etc/sudoers.d/90-cloud-compose + printf '%s\n' \ + 'cloud-compose ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service' \ + 'cloud-compose ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh state' \ + 'cloud-compose ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh status' \ + 'cloud-compose ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh dump' \ + >/etc/sudoers.d/90-cloud-compose chmod 0440 /etc/sudoers.d/90-cloud-compose } @@ -90,6 +103,10 @@ runcmd: cp -a /var/lib/cloud-compose/mounted-rootfs/mnt/disks/. /mnt/disks/ fi %{ endif } + install -d -m 0755 -o root -g root /usr/local/sbin + install -m 0755 -o root -g root \ + /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh \ + /usr/local/sbin/cloud-compose-diagnostics.sh chown root:cloud-compose /mnt/disks/data chmod 1775 /mnt/disks/data chown cloud-compose:cloud-compose /mnt/disks/volumes diff --git a/rootfs/etc/systemd/system/cloud-compose.service b/rootfs/etc/systemd/system/cloud-compose.service index 2cd1381..83ead84 100644 --- a/rootfs/etc/systemd/system/cloud-compose.service +++ b/rootfs/etc/systemd/system/cloud-compose.service @@ -18,7 +18,10 @@ ExecStart=/bin/bash /home/cloud-compose/up ExecStop=/bin/bash /home/cloud-compose/down Restart=on-failure RestartSec=30s -TimeoutStartSec=1h +# run.sh allows application convergence to take 75 minutes. Keep systemd's +# ceiling above that application budget so the service manager cannot kill a +# healthy first boot before the caller's bounded wait reports its own result. +TimeoutStartSec=90min TimeoutStopSec=15min [Install] diff --git a/rootfs/home/cloud-compose/smoke-healthcheck.sh b/rootfs/home/cloud-compose/smoke-healthcheck.sh new file mode 100755 index 0000000..2318fe4 --- /dev/null +++ b/rootfs/home/cloud-compose/smoke-healthcheck.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if (($# != 1)) || [[ -z "$1" ]]; then + echo "usage: smoke-healthcheck.sh CONTEXT" >&2 + exit 2 +fi + +readonly context="$1" + +# Load the same validated runtime environment and tool path used by the +# host-owned lifecycle scripts before handing control to sitectl. +export HOME=/home/cloud-compose +source /home/cloud-compose/profile.sh + +exec sitectl healthcheck --context "$context" --persist --format table diff --git a/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh b/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh new file mode 100755 index 0000000..df07749 --- /dev/null +++ b/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh @@ -0,0 +1,200 @@ +#!/bin/bash + +set -euo pipefail + +# This program is reached through an exact sudoers command. Never resolve its +# child commands from a caller-controlled tool directory. +readonly PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +export PATH + +readonly bootstrap_marker="/home/cloud-compose/.cloud-compose-bootstrap-complete" +readonly diagnostics_program="/usr/local/sbin/cloud-compose-diagnostics.sh" +readonly process_pattern='[/]home/cloud-compose/run[.]sh|[/]home/cloud-compose/[h]ost-conf[.]sh|[/]home/cloud-compose/[h]ost-init[.]sh|[/]home/cloud-compose/[a]pp-init[.]sh|[/]home/cloud-compose/[i]nstall-dependencies|[a]pt-get|[r]pm-ostree|[d]ocker run|[s]itectl|[g]it clone' + +usage() { + echo "usage: ${diagnostics_program} state|status|dump" >&2 +} + +require_root() { + if ((EUID != 0)); then + echo "Cloud Compose diagnostics must run as root" >&2 + exit 1 + fi +} + +unit_value() { + local unit="$1" property="$2" value + + value="$(systemctl show --property="$property" --value -- "$unit" 2>/dev/null || true)" + printf '%s\n' "${value:-unknown}" +} + +bootstrap_state() { + local bootstrap_load_state bootstrap_active_state bootstrap_sub_state + + if [[ -f "$bootstrap_marker" && ! -L "$bootstrap_marker" ]]; then + echo complete + return 0 + fi + + bootstrap_load_state="$(unit_value cloud-compose-bootstrap.service LoadState)" + if [[ "$bootstrap_load_state" == "loaded" ]]; then + bootstrap_active_state="$(unit_value cloud-compose-bootstrap.service ActiveState)" + bootstrap_sub_state="$(unit_value cloud-compose-bootstrap.service SubState)" + case "${bootstrap_active_state}:${bootstrap_sub_state}" in + active:* | activating:* | *:auto-restart) + echo active + return 0 + ;; + esac + elif [[ "$bootstrap_load_state" == "not-found" ]] && + systemctl is-active --quiet cloud-compose.service; then + # Compatibility with releases that predate the retryable bootstrap + # unit and durable readiness marker. + echo complete + return 0 + fi + + if systemctl is-active --quiet cloud-final.service; then + echo active + return 0 + fi + if pgrep -f -- "$process_pattern" >/dev/null; then + echo active + return 0 + fi + + echo idle + return 1 +} + +unit_heartbeat() { + local unit main_pid + + for unit in cloud-final.service cloud-compose-bootstrap.service cloud-compose.service; do + echo "--- ${unit} state ---" + systemctl show --no-pager \ + --property=LoadState \ + --property=ActiveState \ + --property=SubState \ + --property=Result \ + --property=MainPID \ + --property=ExecMainStatus \ + -- "$unit" 2>&1 || true + main_pid="$(unit_value "$unit" MainPID)" + if [[ "$main_pid" =~ ^[1-9][0-9]*$ ]]; then + ps -p "$main_pid" -o pid=,ppid=,stat=,etime=,comm= 2>/dev/null || true + fi + done + + echo "--- active bootstrap process state ---" + while IFS= read -r process_id; do + [[ "$process_id" =~ ^[1-9][0-9]*$ ]] || continue + ps -p "$process_id" -o pid=,ppid=,stat=,etime=,comm= 2>/dev/null || true + done < <(pgrep -f -- "$process_pattern" 2>/dev/null || true) +} + +diagnostic_status() { + local cloud_init_status=0 state + + echo "--- Cloud Compose provisioning heartbeat ---" + date -u '+%Y-%m-%dT%H:%M:%SZ' + echo "--- cloud-init status ---" + if command -v cloud-init >/dev/null 2>&1; then + cloud-init status --long || cloud_init_status=$? + else + echo "cloud-init not installed" + fi + unit_heartbeat + state="$(bootstrap_state 2>/dev/null || true)" + echo "bootstrap-state: ${state:-unknown}" + return "$cloud_init_status" +} + +tail_regular_file() { + local label="$1" path="$2" lines="$3" + + echo "--- ${label} ---" + if [[ -f "$path" && ! -L "$path" ]]; then + tail -n "$lines" -- "$path" 2>&1 || true + else + echo "${label} is not present as a regular file" + fi +} + +dump_compose_state() { + local manifest="/home/cloud-compose/compose-projects.json" + local docker_path encoded row app project_dir + + echo "--- docker ps ---" + docker_path="$(command -v docker || true)" + if [[ -z "$docker_path" ]]; then + echo "docker is not installed" + return 0 + fi + "$docker_path" ps -a 2>&1 || true + + echo "--- docker compose project state ---" + if [[ ! -f "$manifest" || -L "$manifest" ]] || + ! jq -e 'type == "object"' "$manifest" >/dev/null 2>&1; then + echo "Compose project manifest is unavailable or invalid" + return 0 + fi + + while IFS= read -r encoded; do + [[ -n "$encoded" ]] || continue + row="$(printf '%s' "$encoded" | base64 -d)" || continue + app="$(jq -er '.key | select(type == "string" and length > 0)' <<<"$row" 2>/dev/null || true)" + project_dir="$(jq -er '.value.project_dir | select(type == "string" and startswith("/mnt/disks/data/"))' <<<"$row" 2>/dev/null || true)" + if [[ -z "$app" || -z "$project_dir" || ! -d "$project_dir" || -L "$project_dir" ]]; then + echo "Skipping unavailable or unsafe Compose project: ${app:-unknown}" + continue + fi + echo "--- docker compose ps: ${app} ---" + runuser -u cloud-compose -- env HOME=/home/cloud-compose \ + "$docker_path" compose --project-directory "$project_dir" ps 2>&1 || true + done < <(jq -r 'to_entries[] | @base64' "$manifest") +} + +diagnostic_dump() { + diagnostic_status || true + tail_regular_file "/var/log/cloud-init-output.log" "/var/log/cloud-init-output.log" 400 + tail_regular_file "/var/log/cloud-init.log" "/var/log/cloud-init.log" 400 + echo "--- cloud-init runcmd ---" + if [[ -f /var/lib/cloud/instance/scripts/runcmd && + ! -L /var/lib/cloud/instance/scripts/runcmd ]]; then + sed -n '1,240p' /var/lib/cloud/instance/scripts/runcmd 2>&1 || true + else + echo "cloud-init runcmd is not present as a regular file" + fi + echo "--- cloud-compose bootstrap unit ---" + journalctl -u cloud-compose-bootstrap --no-pager -n 400 2>&1 || true + tail_regular_file "legacy cloud-compose bootstrap log" "/home/cloud-compose/run.log" 400 + echo "--- cloud-compose unit ---" + journalctl -u cloud-compose --no-pager -n 300 2>&1 || true + echo "--- lifecycle lock permissions ---" + stat -Lc '%A %a %U:%G %u:%g %n' \ + /run/lock/cloud-compose \ + /run/lock/cloud-compose/lifecycle.lock 2>&1 || true + dump_compose_state +} + +main() { + require_root + if [[ "$#" -ne 1 ]]; then + usage + return 2 + fi + + case "$1" in + state) bootstrap_state ;; + status) diagnostic_status ;; + dump) diagnostic_dump ;; + *) + usage + return 2 + ;; + esac +} + +main "$@" diff --git a/templates/cloud-init.yml b/templates/cloud-init.yml index 4559bee..6b68bba 100644 --- a/templates/cloud-init.yml +++ b/templates/cloud-init.yml @@ -10,6 +10,9 @@ users: - docker sudo: - ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service + - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh state + - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh status + - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh dump %{ if length(CLOUD_COMPOSE_SSH_KEYS) > 0 ~} ssh_authorized_keys: %{ for key in CLOUD_COMPOSE_SSH_KEYS ~} @@ -70,6 +73,11 @@ bootcmd: install -m 0600 /dev/null /run/cloud-compose-filesystems-ready write_files: +- path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh + owner: root:root + permissions: "0600" + encoding: b64 + content: ${DIAGNOSTICS_SCRIPT_B64} ${WRITE_FILES_CONTENT} ${DOCKER_COMPOSE_SCRIPTS} ${COMPOSE_PROJECTS_FILE} @@ -82,6 +90,10 @@ runcmd: ${ROOTFS_ARCHIVE_COMMAND} - | set -eu + install -d -m 0755 -o root -g root /usr/local/sbin + install -m 0755 -o root -g root \ + /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh \ + /usr/local/sbin/cloud-compose-diagnostics.sh test -f /run/cloud-compose-filesystems-ready || { echo "Cloud Compose filesystems were not prepared; refusing application initialization" >&2 exit 1 From df8868e912160b8c71e0bf727d4472f169535c83 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 18:58:40 +0000 Subject: [PATCH 05/67] [patch] Promote first-customer application releases --- README.md | 13 +++-- docs/examples.md | 12 ++--- docs/managed-runtime.md | 16 +++--- examples/binpack/main.tf | 6 +-- .../contracttest/template_version_test.go | 52 +++++++++++-------- providers/do/template_versions.tftest.hcl | 6 +-- providers/gcp/template_versions.tftest.hcl | 6 +-- providers/linode/template_versions.tftest.hcl | 6 +-- renovate.json5 | 26 ++++++++++ template_versions.tftest.hcl | 26 +++++----- templates/apps.json | 38 +++++++------- 11 files changed, 122 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 9fd235c..6a65f89 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,15 @@ release set. Override individual selectors with `runtime.sitectl.package_versions` when intentionally testing or promoting a different compatible release set. -All presets use their coordinated sitectl v1.0.0 core/plugin release set. The -ISLE preset selects the `libops/isle` v1.1.0 template; every other application -preset remains on its v1.0.0 template contract. Keep each preset's complete -template and package set together when promoting an override. +Renovate tracks both the Compose tags and sitectl release tags in the registry. +Its pull requests are promotion candidates: keep the complete compatible set +together and require the hosted application smoke before merging them. + +All presets pin sitectl v1.8.2 plus the latest reviewed application plugin. +ISLE selects the `libops/isle` v1.3.0 template, Drupal and WordPress select +their v1.1.0 templates, and the remaining application templates stay on +v1.0.0. Keep each preset's complete template and package set together when +promoting an override. GCP deployments have two Terraform ownership layers. Apply [`modules/gcp-foundation`](modules/gcp-foundation/README.md) once per service diff --git a/docs/examples.md b/docs/examples.md index e128c5c..dbad7ea 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -129,12 +129,12 @@ healthcheck settings: runtime = { sitectl = { package_versions = { - sitectl = "v1.0.0" - sitectl-wp = "v1.0.0" + sitectl = "v1.8.2" + sitectl-wp = "v2.0.0" } } compose = { - branch = "v1.0.0" + branch = "v1.1.0" ingress = { letsencrypt = true bot_mitigation = true @@ -175,9 +175,9 @@ runtime = { } sitectl = { package_versions = { - sitectl = "v1.0.0" - sitectl-wp = "v1.0.0" - sitectl-drupal = "v1.0.0" + sitectl = "v1.8.2" + sitectl-wp = "v2.0.0" + sitectl-drupal = "v1.3.0" } } } diff --git a/docs/managed-runtime.md b/docs/managed-runtime.md index 630426f..bc10f56 100644 --- a/docs/managed-runtime.md +++ b/docs/managed-runtime.md @@ -26,11 +26,11 @@ core and plugins independently. The provider-neutral backward-compatible fallback only for an installed package with neither a template selector nor an explicit per-package selector. -All presets select coordinated sitectl v1 release packages. The ISLE preset -uses the `libops/isle` v1.1.0 template with sitectl, sitectl-drupal, and -sitectl-isle v1.0.0. Every other application preset remains on its v1.0.0 -template contract. Override the template or package selectors only as one -reviewed, compatible release set. +All presets pin sitectl v1.8.2 with a reviewed application plugin. ISLE uses +the `libops/isle` v1.3.0 template with sitectl-drupal v1.3.0 and sitectl-isle +v1.5.0. Drupal and WordPress use their v1.1.0 templates; the other application +templates remain on v1.0.0. Override the template or package selectors only as +one reviewed, compatible release set. Omitting `runtime.sitectl.packages` selects the template's package set. An explicit list replaces that set; `packages = []` or `packages = ["sitectl"]` @@ -51,9 +51,9 @@ runtime = { "sitectl-isle", ] package_versions = { - sitectl = "v1.0.0" - sitectl-drupal = "v1.0.0" - sitectl-isle = "v1.0.0" + sitectl = "v1.8.2" + sitectl-drupal = "v1.3.0" + sitectl-isle = "v1.5.0" } plugin = "isle" } diff --git a/examples/binpack/main.tf b/examples/binpack/main.tf index 98f06b4..eefa061 100644 --- a/examples/binpack/main.tf +++ b/examples/binpack/main.tf @@ -29,9 +29,9 @@ module "apps" { } sitectl = { package_versions = { - sitectl = "v1.0.0" - sitectl-wp = "v1.0.0" - sitectl-drupal = "v1.0.0" + sitectl = "v1.8.2" + sitectl-wp = "v2.0.0" + sitectl-drupal = "v1.3.0" } } } diff --git a/internal/contracttest/template_version_test.go b/internal/contracttest/template_version_test.go index ba1b401..1b531bb 100644 --- a/internal/contracttest/template_version_test.go +++ b/internal/contracttest/template_version_test.go @@ -30,38 +30,47 @@ func TestTemplateVersionContract(t *testing.T) { expectedVersions := map[string]map[string]string{ "default": { - "sitectl": "v1.0.0", + "sitectl": "v1.8.2", }, "archivesspace": { - "sitectl": "v1.0.0", - "sitectl-archivesspace": "v1.0.0", + "sitectl": "v1.8.2", + "sitectl-archivesspace": "v2.0.0", }, "drupal": { - "sitectl": "v1.0.0", - "sitectl-drupal": "v1.0.0", + "sitectl": "v1.8.2", + "sitectl-drupal": "v1.3.0", }, "isle": { - "sitectl": "v1.0.0", - "sitectl-drupal": "v1.0.0", - "sitectl-isle": "v1.0.0", + "sitectl": "v1.8.2", + "sitectl-drupal": "v1.3.0", + "sitectl-isle": "v1.5.0", }, "ojs": { - "sitectl": "v1.0.0", - "sitectl-ojs": "v1.0.0", + "sitectl": "v1.8.2", + "sitectl-ojs": "v1.2.0", }, "omeka-classic": { - "sitectl": "v1.0.0", - "sitectl-omeka-classic": "v1.0.0", + "sitectl": "v1.8.2", + "sitectl-omeka-classic": "v1.2.0", }, "omeka-s": { - "sitectl": "v1.0.0", - "sitectl-omeka-s": "v1.0.0", + "sitectl": "v1.8.2", + "sitectl-omeka-s": "v1.2.0", }, "wp": { - "sitectl": "v1.0.0", - "sitectl-wp": "v1.0.0", + "sitectl": "v1.8.2", + "sitectl-wp": "v2.0.0", }, } + expectedBranches := map[string]string{ + "archivesspace": "v1.0.0", + "drupal": "v1.1.0", + "isle": "v1.3.0", + "ojs": "v1.0.0", + "omeka-classic": "v1.0.0", + "omeka-s": "v1.0.0", + "wp": "v1.1.0", + } if !maps.Equal(registry.Default.PackageVersions, expectedVersions["default"]) { t.Errorf("template %q package versions diverged:\nexpected %s\nactual %s", "default", prettyJSON(t, expectedVersions["default"]), prettyJSON(t, registry.Default.PackageVersions)) @@ -80,10 +89,7 @@ func TestTemplateVersionContract(t *testing.T) { if !maps.Equal(definition.PackageVersions, expected) { t.Errorf("template %q package versions diverged:\nexpected %s\nactual %s", name, prettyJSON(t, expected), prettyJSON(t, definition.PackageVersions)) } - expectedBranch := "v1.0.0" - if name == "isle" { - expectedBranch = "v1.1.0" - } + expectedBranch := expectedBranches[name] if definition.Branch != expectedBranch { t.Errorf("template %q branch = %q, want stable contract %s", name, definition.Branch, expectedBranch) } @@ -140,8 +146,8 @@ func TestTemplateVersionContract(t *testing.T) { for _, packageName := range []string{"sitectl", "sitectl-wp", "sitectl-drupal"} { requireContains(t, content, packageName, relativePath+" bin-pack package") } - requireContains(t, content, `sitectl = "v1.0.0"`, relativePath+" bin-pack core selector") - requireContains(t, content, `sitectl-wp = "v1.0.0"`, relativePath+" bin-pack WordPress selector") - requireContains(t, content, `sitectl-drupal = "v1.0.0"`, relativePath+" bin-pack Drupal selector") + requireContains(t, content, `sitectl = "v1.8.2"`, relativePath+" bin-pack core selector") + requireContains(t, content, `sitectl-wp = "v2.0.0"`, relativePath+" bin-pack WordPress selector") + requireContains(t, content, `sitectl-drupal = "v1.3.0"`, relativePath+" bin-pack Drupal selector") } } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index 781c11c..3fc8276 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -28,8 +28,8 @@ run "custom_package_set_merges_only_applicable_template_versions" { } assert { - condition = local.runtime.compose.branch == "v1.1.0" - error_message = "The DigitalOcean entrypoint must inherit the ISLE v1.1.0 template branch when no override is supplied." + condition = local.runtime.compose.branch == "v1.3.0" + error_message = "The DigitalOcean entrypoint must inherit the ISLE v1.3.0 template branch when no override is supplied." } assert { @@ -55,7 +55,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == { - sitectl = "v1.0.0" + sitectl = "v1.8.2" } error_message = "The DigitalOcean entrypoint must preserve an explicit core-only package set." } diff --git a/providers/gcp/template_versions.tftest.hcl b/providers/gcp/template_versions.tftest.hcl index 958be67..4b7ff77 100644 --- a/providers/gcp/template_versions.tftest.hcl +++ b/providers/gcp/template_versions.tftest.hcl @@ -38,8 +38,8 @@ run "custom_package_set_merges_only_applicable_template_versions" { } assert { - condition = local.runtime.compose.branch == "v1.1.0" - error_message = "The GCP entrypoint must inherit the ISLE v1.1.0 template branch when no override is supplied." + condition = local.runtime.compose.branch == "v1.3.0" + error_message = "The GCP entrypoint must inherit the ISLE v1.3.0 template branch when no override is supplied." } assert { @@ -67,7 +67,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == { - sitectl = "v1.0.0" + sitectl = "v1.8.2" } error_message = "The GCP entrypoint must preserve an explicit core-only package set." } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index 460b3a2..14be098 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -33,8 +33,8 @@ run "custom_package_set_merges_only_applicable_template_versions" { } assert { - condition = local.runtime.compose.branch == "v1.1.0" - error_message = "The Linode entrypoint must inherit the ISLE v1.1.0 template branch when no override is supplied." + condition = local.runtime.compose.branch == "v1.3.0" + error_message = "The Linode entrypoint must inherit the ISLE v1.3.0 template branch when no override is supplied." } assert { @@ -65,7 +65,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == { - sitectl = "v1.0.0" + sitectl = "v1.8.2" } error_message = "The Linode entrypoint must preserve an explicit core-only package set." } diff --git a/renovate.json5 b/renovate.json5 index 169e5fc..eabf862 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -4,6 +4,32 @@ 'github>libops/renovate-config:default.json5', ], customManagers: [ + { + customType: 'regex', + description: 'Propose reviewed sitectl package promotions for application presets', + managerFilePatterns: [ + '/^templates/apps\.json$/', + ], + matchStrings: [ + '"(?sitectl(?:-[a-z0-9-]+)?)"\\s*:\\s*"(?v[0-9]+\\.[0-9]+\\.[0-9]+)"', + ], + datasourceTemplate: 'github-releases', + packageNameTemplate: 'libops/{{{depName}}}', + versioningTemplate: 'semver', + }, + { + customType: 'regex', + description: 'Propose reviewed Compose-template promotions for application presets', + managerFilePatterns: [ + '/^templates/apps\.json$/', + ], + matchStrings: [ + '"repo"\\s*:\\s*"https://github\\.com/libops/(?[a-z0-9-]+)(?:\\.git)?"\\s*,\\s*"branch"\\s*:\\s*"(?v[0-9]+\\.[0-9]+\\.[0-9]+)"', + ], + datasourceTemplate: 'github-tags', + packageNameTemplate: 'libops/{{{depName}}}', + versioningTemplate: 'semver', + }, { customType: 'regex', description: 'Update Docker Compose and Buildx Terraform defaults', diff --git a/template_versions.tftest.hcl b/template_versions.tftest.hcl index b33564c..4b992a8 100644 --- a/template_versions.tftest.hcl +++ b/template_versions.tftest.hcl @@ -29,7 +29,7 @@ run "default_template_uses_v1_core" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.0.0" + sitectl = "v1.8.2" } error_message = "The default template must select the released sitectl v1 core." } @@ -59,15 +59,15 @@ run "non_isle_template_uses_v1_release_set" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.0.0" - sitectl-wp = "v1.0.0" + sitectl = "v1.8.2" + sitectl-wp = "v2.0.0" } error_message = "Non-ISLE templates must select their coordinated sitectl v1 release set." } assert { - condition = local.compose.branch == "v1.0.0" - error_message = "Non-ISLE templates must retain their stable v1.0.0 template contract." + condition = local.compose.branch == "v1.1.0" + error_message = "The WordPress preset must select its stable v1.1.0 template contract." } } @@ -90,16 +90,16 @@ run "isle_template_uses_v1_release_set" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.0.0" - sitectl-drupal = "v1.0.0" - sitectl-isle = "v1.0.0" + sitectl = "v1.8.2" + sitectl-drupal = "v1.3.0" + sitectl-isle = "v1.5.0" } error_message = "The ISLE template must select its coordinated sitectl v1 release set by default." } assert { - condition = local.compose.branch == "v1.1.0" - error_message = "The ISLE preset must select the stable v1.1.0 template contract." + condition = local.compose.branch == "v1.3.0" + error_message = "The ISLE preset must select the stable v1.3.0 template contract." } assert { @@ -168,7 +168,7 @@ run "explicit_package_versions_override_template_defaults" { assert { condition = local.sitectl.package_versions == { sitectl = "v0.40.1" - sitectl-drupal = "v1.0.0" + sitectl-drupal = "v1.3.0" sitectl-isle = "v0.19.1" } error_message = "Explicit per-package selectors must override only their matching template defaults." @@ -200,7 +200,7 @@ run "custom_package_set_filters_template_versions" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.0.0" + sitectl = "v1.8.2" sitectl-wp = "v0.6.1" } error_message = "Template selectors for packages omitted by a custom package set must not reach the runtime." @@ -229,7 +229,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.sitectl.packages == tolist(["sitectl"]) && local.sitectl.package_versions == { - sitectl = "v1.0.0" + sitectl = "v1.8.2" } error_message = "An explicit core-only package set must not be mistaken for an omitted template package selection." } diff --git a/templates/apps.json b/templates/apps.json index 5a8600d..ac347ac 100644 --- a/templates/apps.json +++ b/templates/apps.json @@ -6,7 +6,7 @@ "packages": ["sitectl"], "extra_env": {}, "package_versions": { - "sitectl": "v1.0.0" + "sitectl": "v1.8.2" } }, "templates": { @@ -17,8 +17,8 @@ "packages": ["sitectl", "sitectl-archivesspace"], "extra_env": {}, "package_versions": { - "sitectl": "v1.0.0", - "sitectl-archivesspace": "v1.0.0" + "sitectl": "v1.8.2", + "sitectl-archivesspace": "v2.0.0" } }, "ojs": { @@ -28,44 +28,44 @@ "packages": ["sitectl", "sitectl-ojs"], "extra_env": {}, "package_versions": { - "sitectl": "v1.0.0", - "sitectl-ojs": "v1.0.0" + "sitectl": "v1.8.2", + "sitectl-ojs": "v1.2.0" } }, "isle": { "repo": "https://github.com/libops/isle", - "branch": "v1.1.0", + "branch": "v1.3.0", "plugin": "isle", "packages": ["sitectl", "sitectl-drupal", "sitectl-isle"], "extra_env": { "ISLANDORA_TAG": "6.3.19" }, "package_versions": { - "sitectl": "v1.0.0", - "sitectl-drupal": "v1.0.0", - "sitectl-isle": "v1.0.0" + "sitectl": "v1.8.2", + "sitectl-drupal": "v1.3.0", + "sitectl-isle": "v1.5.0" } }, "drupal": { "repo": "https://github.com/libops/drupal.git", - "branch": "v1.0.0", + "branch": "v1.1.0", "plugin": "drupal", "packages": ["sitectl", "sitectl-drupal"], "extra_env": {}, "package_versions": { - "sitectl": "v1.0.0", - "sitectl-drupal": "v1.0.0" + "sitectl": "v1.8.2", + "sitectl-drupal": "v1.3.0" } }, "wp": { "repo": "https://github.com/libops/wp.git", - "branch": "v1.0.0", + "branch": "v1.1.0", "plugin": "wp", "packages": ["sitectl", "sitectl-wp"], "extra_env": {}, "package_versions": { - "sitectl": "v1.0.0", - "sitectl-wp": "v1.0.0" + "sitectl": "v1.8.2", + "sitectl-wp": "v2.0.0" } }, "omeka-s": { @@ -75,8 +75,8 @@ "packages": ["sitectl", "sitectl-omeka-s"], "extra_env": {}, "package_versions": { - "sitectl": "v1.0.0", - "sitectl-omeka-s": "v1.0.0" + "sitectl": "v1.8.2", + "sitectl-omeka-s": "v1.2.0" } }, "omeka-classic": { @@ -86,8 +86,8 @@ "packages": ["sitectl", "sitectl-omeka-classic"], "extra_env": {}, "package_versions": { - "sitectl": "v1.0.0", - "sitectl-omeka-classic": "v1.0.0" + "sitectl": "v1.8.2", + "sitectl-omeka-classic": "v1.2.0" } } } From d7379656fa1d6c198d760b024db1e219f67c39c0 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 19:04:41 +0000 Subject: [PATCH 06/67] [patch] Align config management release fixtures --- ansible/README.md | 2 +- salt/cloud-compose/README.md | 2 +- tests/config-management/ansible/smoke.yml | 8 ++++---- tests/config-management/salt-pillar/wp-prod.sls | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/README.md b/ansible/README.md index 665539c..8dc3181 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -115,7 +115,7 @@ all: sitectl: environment: production package_versions: - sitectl-isle: v1.0.0 + sitectl-isle: v1.5.0 wp-prod.example.edu: ansible_user: debian cloud_compose_name: wp-prod diff --git a/salt/cloud-compose/README.md b/salt/cloud-compose/README.md index 01f4614..b554c61 100644 --- a/salt/cloud-compose/README.md +++ b/salt/cloud-compose/README.md @@ -137,7 +137,7 @@ cloud_compose: sitectl: environment: production package_versions: - sitectl-isle: v1.0.0 + sitectl-isle: v1.5.0 ``` Apply: diff --git a/tests/config-management/ansible/smoke.yml b/tests/config-management/ansible/smoke.yml index 6614299..96ed382 100644 --- a/tests/config-management/ansible/smoke.yml +++ b/tests/config-management/ansible/smoke.yml @@ -48,9 +48,9 @@ sitectl: environment: production package_versions: - sitectl: v1.0.0 - sitectl-drupal: v1.0.0 - sitectl-isle: v1.0.0 + sitectl: v1.8.2 + sitectl-drupal: v1.3.0 + sitectl-isle: v1.5.0 managed_runtime: enabled: false internal_services_enabled: false @@ -124,7 +124,7 @@ - '(cloud_compose_application_env_file.content | b64decode | from_json).PORT == "9999"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].docker_compose_repo == "https://github.com/libops/isle"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].project_dir == "/mnt/disks/data/libops/isle/isle-prod"' - - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].compose_project_name == "libops-isle-v1-1-0"' + - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].compose_project_name == "libops-isle-v1-3-0"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].ingress.domain == "isle.example.edu"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].sitectl_plugin == "isle"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].init_commands == []' diff --git a/tests/config-management/salt-pillar/wp-prod.sls b/tests/config-management/salt-pillar/wp-prod.sls index 54d794f..1c910c4 100644 --- a/tests/config-management/salt-pillar/wp-prod.sls +++ b/tests/config-management/salt-pillar/wp-prod.sls @@ -46,8 +46,8 @@ cloud_compose: sitectl: environment: production package_versions: - sitectl: v1.0.0 - sitectl-wp: v1.0.0 + sitectl: v1.8.2 + sitectl-wp: v2.0.0 managed_runtime: enabled: false internal_services_enabled: false From 6137153c9f758f70a6c7b34de3e5ddeca91ffd90 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 19:16:35 +0000 Subject: [PATCH 07/67] [patch] Check in config management assertions --- ci/config-management-smoke-assert.py | 282 +++++++++++++++++++++++++++ ci/config-management-smoke-inner.sh | 236 ++-------------------- 2 files changed, 294 insertions(+), 224 deletions(-) create mode 100644 ci/config-management-smoke-assert.py diff --git a/ci/config-management-smoke-assert.py b/ci/config-management-smoke-assert.py new file mode 100644 index 0000000..b2ea3f0 --- /dev/null +++ b/ci/config-management-smoke-assert.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 + +"""Assertions for the containerized Ansible and Salt smoke tests.""" + +import argparse +import grp +import json +import os +import stat +import subprocess +from pathlib import Path + + +RUNTIME_HOME = Path("/home/cloud-compose") + + +def load_runtime_env(path: Path) -> dict[str, str]: + result = subprocess.run( + [ + "env", + "-i", + "PATH=/usr/bin:/bin", + f"CLOUD_COMPOSE_ENV_FILE={path}", + "bash", + "--noprofile", + "--norc", + "-c", + "source /home/cloud-compose/profile.sh; env -0", + ], + check=True, + stdout=subprocess.PIPE, + ) + return { + entry.split(b"=", 1)[0].decode(): entry.split(b"=", 1)[1].decode() + for entry in result.stdout.split(b"\0") + if b"=" in entry + } + + +def assert_runtime_files() -> None: + for path in [ + RUNTIME_HOME / "init", + RUNTIME_HOME / "up", + RUNTIME_HOME / "down", + RUNTIME_HOME / "rollout", + RUNTIME_HOME / "run.sh", + RUNTIME_HOME / "start-cloud-compose-bootstrap.sh", + ]: + assert path.exists(), path + assert os.access(path, os.X_OK), path + + cloud_compose_gid = grp.getgrnam("cloud-compose").gr_gid + for path, expected_mode in { + RUNTIME_HOME / "init": 0o750, + RUNTIME_HOME / "up": 0o750, + RUNTIME_HOME / "down": 0o750, + RUNTIME_HOME / "rollout": 0o750, + RUNTIME_HOME / ".env": 0o640, + RUNTIME_HOME / "application-env.json": 0o640, + RUNTIME_HOME / "compose-projects.json": 0o640, + RUNTIME_HOME / "managed-runtime-artifacts.tsv": 0o640, + }.items(): + metadata = path.stat() + assert metadata.st_uid == 0, (path, metadata.st_uid) + assert metadata.st_gid == cloud_compose_gid, (path, metadata.st_gid) + assert stat.S_IMODE(metadata.st_mode) == expected_mode, ( + path, + oct(stat.S_IMODE(metadata.st_mode)), + ) + + +def assert_ansible_runtime() -> None: + env = load_runtime_env(RUNTIME_HOME / ".env") + application_env = json.loads( + (RUNTIME_HOME / "application-env.json").read_text() + ) + projects = json.loads((RUNTIME_HOME / "compose-projects.json").read_text()) + project = projects["isle-prod"] + + assert env["CLOUD_COMPOSE_PROVIDER"] == "onprem" + assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false" + assert ( + env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"] + == "/usr/local/libexec/cloud-compose/ansible-offhost" + ) + assert env["DOCKER_COMPOSE_DIR"] == "/mnt/disks/data/libops/isle/isle-prod" + assert env["DOCKER_COMPOSE_REPO"] == "https://github.com/libops/isle" + assert env["SITECTL_PLUGIN"] == "isle" + assert "sitectl-isle" in env["SITECTL_PACKAGES"].split() + assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { + "sitectl": "v1.8.2", + "sitectl-drupal": "v1.3.0", + "sitectl-isle": "v1.5.0", + } + assert project["docker_compose_repo"] == "https://github.com/libops/isle" + assert project["project_dir"] == "/mnt/disks/data/libops/isle/isle-prod" + assert project["compose_project_name"] == "libops-isle-v1-3-0" + assert project["ingress"]["domain"] == "isle.example.edu" + assert project["sitectl_plugin"] == "isle" + assert project["init_commands"] == [] + assert project["up_commands"] == [] + assert project["down_commands"] == ["app-down-command"] + assert project["rollout_commands"] == ["global-rollout-command"] + assert Path("/mnt/disks/data/libops/isle/isle-prod").is_dir() + assert "BASH_ENV" not in env + assert "LD_PRELOAD" not in env + assert "PORT" not in env + assert application_env["BASH_ENV"] == ( + "/tmp/cloud-compose-ansible-untrusted-bash-env" + ) + assert application_env["LD_PRELOAD"] == ( + "/tmp/cloud-compose-ansible-untrusted-preload.so" + ) + assert application_env["PORT"] == "9999" + assert application_env["CONTRACT_BACKTICKS"] == ( + "`touch /tmp/cloud-compose-ansible-backtick-injection`" + ) + assert application_env["CONTRACT_BACKSLASH"] == "a\\path\\ends\\" + assert application_env["CONTRACT_COMMAND_SUB"] == ( + "$(touch /tmp/cloud-compose-ansible-command-injection)" + ) + assert application_env["CONTRACT_DOLLARS"] == "$HOME ${HOME}" + assert application_env["CONTRACT_QUOTES"] == ( + 'a "double" and a single quote: O\'Reilly' + ) + assert application_env["CONTRACT_WHITESPACE"] == " leading and trailing " + assert application_env["CONTRACT_MULTILINE"] == "line one\nline two" + assert not Path("/tmp/cloud-compose-ansible-backtick-injection").exists() + assert not Path("/tmp/cloud-compose-ansible-command-injection").exists() + + assert_runtime_files() + + +def assert_salt_noop() -> None: + states = json.loads(Path("/tmp/cloud-compose-salt-noop.json").read_text())[ + "local" + ] + lock_states = [ + state + for state_id, state in states.items() + if "|-cloud-compose-lifecycle-lock_|-" in state_id + ] + assert len(lock_states) == 1, lock_states + assert lock_states[0]["result"] is True, lock_states[0] + assert lock_states[0]["changes"] == {}, lock_states[0] + + +def assert_salt_runtime( + expected_name: str, + expected_repo: str, + expected_plugin: str, + expected_package: str, + expected_domain: str, + expected_project_dir: str, + expected_compose_project_name: str, +) -> None: + env = load_runtime_env(RUNTIME_HOME / ".env") + application_env = json.loads( + (RUNTIME_HOME / "application-env.json").read_text() + ) + projects = json.loads((RUNTIME_HOME / "compose-projects.json").read_text()) + project = projects[expected_name] + artifact_manifest = (RUNTIME_HOME / "managed-runtime-artifacts.tsv").read_text() + + assert env["CLOUD_COMPOSE_PROVIDER"] == "onprem" + assert env["CLOUD_COMPOSE_APPS"] == expected_name + assert env["CLOUD_COMPOSE_PRIMARY_APP"] == expected_name + assert env["DOCKER_COMPOSE_DIR"] == expected_project_dir + assert env["DOCKER_COMPOSE_REPO"] == expected_repo + assert env["COMPOSE_PROJECT_NAME"] == expected_compose_project_name, ( + env["COMPOSE_PROJECT_NAME"], + expected_compose_project_name, + ) + assert env["SITECTL_PLUGIN"] == expected_plugin + assert expected_package in env["SITECTL_PACKAGES"].split() + assert project["docker_compose_repo"] == expected_repo + assert project["project_dir"] == expected_project_dir + assert project["compose_project_name"] == expected_compose_project_name, ( + project["compose_project_name"], + expected_compose_project_name, + ) + assert project["sitectl_plugin"] == expected_plugin + assert project["ingress"]["domain"] == expected_domain + assert Path(expected_project_dir).is_dir() + + if expected_name == "wp-prod": + assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false" + assert ( + env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"] + == "/usr/local/libexec/cloud-compose/salt-offhost" + ) + assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { + "sitectl": "v1.8.2", + "sitectl-wp": "v2.0.0", + } + assert "BASH_ENV" not in env + assert "LD_PRELOAD" not in env + assert "PORT" not in env + assert application_env["BASH_ENV"] == ( + "/tmp/cloud-compose-salt-untrusted-bash-env" + ) + assert application_env["LD_PRELOAD"] == ( + "/tmp/cloud-compose-salt-untrusted-preload.so" + ) + assert application_env["PORT"] == "9999" + assert application_env["CONTRACT_BACKTICKS"] == ( + "`touch /tmp/cloud-compose-salt-backtick-injection`" + ) + assert application_env["CONTRACT_BACKSLASH"] == "a\\path\\ends\\" + assert application_env["CONTRACT_COMMAND_SUB"] == ( + "$(touch /tmp/cloud-compose-salt-command-injection)" + ) + assert application_env["CONTRACT_DOLLARS"] == "$HOME ${HOME}" + assert application_env["CONTRACT_QUOTES"] == ( + 'a "double" and a single quote: O\'Reilly' + ) + assert application_env["CONTRACT_WHITESPACE"] == " leading and trailing " + assert application_env["CONTRACT_MULTILINE"] == "line one\nline two" + assert env["LIBOPS_MANAGED_RUNTIME_ENABLED"] == "false" + assert env["LIBOPS_INTERNAL_SERVICES_ENABLED"] == "false" + assert env["LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE"] == "false" + assert project["init_commands"] == [] + assert project["up_commands"] == [] + assert project["down_commands"] == ["app-down-command"] + assert project["rollout_commands"] == ["global-rollout-command"] + assert ( + "contract-agent\thttps://example.invalid/contract-agent\t" + in artifact_manifest + ) + assert ( + "\t/usr/local/bin/contract-agent\t0750\troot\troot\t" + "cloud-compose.service" + in artifact_manifest + ) + assert not Path("/tmp/cloud-compose-salt-backtick-injection").exists() + assert not Path("/tmp/cloud-compose-salt-command-injection").exists() + elif expected_name == "drupal-prod": + assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { + "sitectl": "v1.8.2", + "sitectl-drupal": "v1.3.0", + } + + assert_runtime_files() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("ansible-runtime") + subparsers.add_parser("salt-noop") + + salt_runtime = subparsers.add_parser("salt-runtime") + salt_runtime.add_argument("expected_name") + salt_runtime.add_argument("expected_repo") + salt_runtime.add_argument("expected_plugin") + salt_runtime.add_argument("expected_package") + salt_runtime.add_argument("expected_domain") + salt_runtime.add_argument("expected_project_dir") + salt_runtime.add_argument("expected_compose_project_name") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.command == "ansible-runtime": + assert_ansible_runtime() + elif args.command == "salt-noop": + assert_salt_noop() + else: + assert_salt_runtime( + args.expected_name, + args.expected_repo, + args.expected_plugin, + args.expected_package, + args.expected_domain, + args.expected_project_dir, + args.expected_compose_project_name, + ) + + +if __name__ == "__main__": + main() diff --git a/ci/config-management-smoke-inner.sh b/ci/config-management-smoke-inner.sh index 00ce871..fbd89e0 100755 --- a/ci/config-management-smoke-inner.sh +++ b/ci/config-management-smoke-inner.sh @@ -229,99 +229,7 @@ ansible-playbook \ verify_lifecycle_lock_contract -python - <<'PY' -import json -import grp -import os -import stat -import subprocess -from pathlib import Path - -def load_runtime_env(path): - result = subprocess.run( - [ - "env", "-i", "PATH=/usr/bin:/bin", f"CLOUD_COMPOSE_ENV_FILE={path}", - "bash", "--noprofile", "--norc", "-c", - "source /home/cloud-compose/profile.sh; env -0", - ], - check=True, - stdout=subprocess.PIPE, - ) - return { - entry.split(b"=", 1)[0].decode(): entry.split(b"=", 1)[1].decode() - for entry in result.stdout.split(b"\0") if b"=" in entry - } - -env = load_runtime_env(Path("/home/cloud-compose/.env")) -application_env = json.loads(Path("/home/cloud-compose/application-env.json").read_text()) - -projects = json.loads(Path("/home/cloud-compose/compose-projects.json").read_text()) -project = projects["isle-prod"] - -assert env["CLOUD_COMPOSE_PROVIDER"] == "onprem" -assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false" -assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"] == "/usr/local/libexec/cloud-compose/ansible-offhost" -assert env["DOCKER_COMPOSE_DIR"] == "/mnt/disks/data/libops/isle/isle-prod" -assert env["DOCKER_COMPOSE_REPO"] == "https://github.com/libops/isle" -assert env["SITECTL_PLUGIN"] == "isle" -assert "sitectl-isle" in env["SITECTL_PACKAGES"].split() -assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { - "sitectl": "v1.0.0", - "sitectl-drupal": "v1.0.0", - "sitectl-isle": "v1.0.0", -} -assert project["docker_compose_repo"] == "https://github.com/libops/isle" -assert project["project_dir"] == "/mnt/disks/data/libops/isle/isle-prod" -assert project["ingress"]["domain"] == "isle.example.edu" -assert project["sitectl_plugin"] == "isle" -assert project["init_commands"] == [] -assert project["up_commands"] == [] -assert project["down_commands"] == ["app-down-command"] -assert project["rollout_commands"] == ["global-rollout-command"] -assert Path("/mnt/disks/data/libops/isle/isle-prod").is_dir() -assert "BASH_ENV" not in env -assert "LD_PRELOAD" not in env -assert "PORT" not in env -assert application_env["BASH_ENV"] == "/tmp/cloud-compose-ansible-untrusted-bash-env" -assert application_env["LD_PRELOAD"] == "/tmp/cloud-compose-ansible-untrusted-preload.so" -assert application_env["PORT"] == "9999" -assert application_env["CONTRACT_BACKTICKS"] == "`touch /tmp/cloud-compose-ansible-backtick-injection`" -assert application_env["CONTRACT_BACKSLASH"] == "a\\path\\ends\\" -assert application_env["CONTRACT_COMMAND_SUB"] == "$(touch /tmp/cloud-compose-ansible-command-injection)" -assert application_env["CONTRACT_DOLLARS"] == "$HOME ${HOME}" -assert application_env["CONTRACT_QUOTES"] == 'a "double" and a single quote: O\'Reilly' -assert application_env["CONTRACT_WHITESPACE"] == " leading and trailing " -assert application_env["CONTRACT_MULTILINE"] == "line one\nline two" -assert not Path("/tmp/cloud-compose-ansible-backtick-injection").exists() -assert not Path("/tmp/cloud-compose-ansible-command-injection").exists() - -for path in [ - "/home/cloud-compose/init", - "/home/cloud-compose/up", - "/home/cloud-compose/down", - "/home/cloud-compose/rollout", - "/home/cloud-compose/run.sh", - "/home/cloud-compose/start-cloud-compose-bootstrap.sh", -]: - assert Path(path).exists(), path - assert os.access(path, os.X_OK), path - -cloud_compose_gid = grp.getgrnam("cloud-compose").gr_gid -for path, expected_mode in { - "/home/cloud-compose/init": 0o750, - "/home/cloud-compose/up": 0o750, - "/home/cloud-compose/down": 0o750, - "/home/cloud-compose/rollout": 0o750, - "/home/cloud-compose/.env": 0o640, - "/home/cloud-compose/application-env.json": 0o640, - "/home/cloud-compose/compose-projects.json": 0o640, - "/home/cloud-compose/managed-runtime-artifacts.tsv": 0o640, -}.items(): - metadata = Path(path).stat() - assert metadata.st_uid == 0, (path, metadata.st_uid) - assert metadata.st_gid == cloud_compose_gid, (path, metadata.st_gid) - assert stat.S_IMODE(metadata.st_mode) == expected_mode, (path, oct(stat.S_IMODE(metadata.st_mode))) -PY +python /work/ci/config-management-smoke-assert.py ansible-runtime rm -rf /home/cloud-compose /mnt/disks @@ -383,137 +291,17 @@ run_salt_case() { --config-dir=/tmp/salt/etc \ --out=json \ state.apply cloud-compose >/tmp/cloud-compose-salt-noop.json - python - <<'PY' -import json -from pathlib import Path - -states = json.loads(Path("/tmp/cloud-compose-salt-noop.json").read_text())["local"] -lock_states = [ - state for state_id, state in states.items() - if "|-cloud-compose-lifecycle-lock_|-" in state_id -] -assert len(lock_states) == 1, lock_states -assert lock_states[0]["result"] is True, lock_states[0] -assert lock_states[0]["changes"] == {}, lock_states[0] -PY + python /work/ci/config-management-smoke-assert.py salt-noop python -m json.tool /home/cloud-compose/compose-projects.json >/dev/null - python - "$expected_name" "$expected_repo" "$expected_plugin" "$expected_package" "$expected_domain" "$expected_project_dir" "$expected_compose_project_name" <<'PY' -import json -import grp -import os -import stat -import subprocess -import sys -from pathlib import Path - -expected_name, expected_repo, expected_plugin, expected_package, expected_domain, expected_project_dir, expected_compose_project_name = sys.argv[1:] - -def load_runtime_env(path): - result = subprocess.run( - [ - "env", "-i", "PATH=/usr/bin:/bin", f"CLOUD_COMPOSE_ENV_FILE={path}", - "bash", "--noprofile", "--norc", "-c", - "source /home/cloud-compose/profile.sh; env -0", - ], - check=True, - stdout=subprocess.PIPE, - ) - return { - entry.split(b"=", 1)[0].decode(): entry.split(b"=", 1)[1].decode() - for entry in result.stdout.split(b"\0") if b"=" in entry - } - -env = load_runtime_env(Path("/home/cloud-compose/.env")) -application_env = json.loads(Path("/home/cloud-compose/application-env.json").read_text()) - -projects = json.loads(Path("/home/cloud-compose/compose-projects.json").read_text()) -project = projects[expected_name] -artifact_manifest = Path("/home/cloud-compose/managed-runtime-artifacts.tsv").read_text() - -assert env["CLOUD_COMPOSE_PROVIDER"] == "onprem" -assert env["CLOUD_COMPOSE_APPS"] == expected_name -assert env["CLOUD_COMPOSE_PRIMARY_APP"] == expected_name -assert env["DOCKER_COMPOSE_DIR"] == expected_project_dir -assert env["DOCKER_COMPOSE_REPO"] == expected_repo -assert env["COMPOSE_PROJECT_NAME"] == expected_compose_project_name, ( - env["COMPOSE_PROJECT_NAME"], expected_compose_project_name -) -assert env["SITECTL_PLUGIN"] == expected_plugin -assert expected_package in env["SITECTL_PACKAGES"].split() -assert project["docker_compose_repo"] == expected_repo -assert project["project_dir"] == expected_project_dir -assert project["compose_project_name"] == expected_compose_project_name, ( - project["compose_project_name"], expected_compose_project_name -) -assert project["sitectl_plugin"] == expected_plugin -assert project["ingress"]["domain"] == expected_domain -assert Path(expected_project_dir).is_dir() - -if expected_name == "wp-prod": - assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false" - assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"] == "/usr/local/libexec/cloud-compose/salt-offhost" - assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { - "sitectl": "v1.0.0", - "sitectl-wp": "v1.0.0", - } - assert "BASH_ENV" not in env - assert "LD_PRELOAD" not in env - assert "PORT" not in env - assert application_env["BASH_ENV"] == "/tmp/cloud-compose-salt-untrusted-bash-env" - assert application_env["LD_PRELOAD"] == "/tmp/cloud-compose-salt-untrusted-preload.so" - assert application_env["PORT"] == "9999" - assert application_env["CONTRACT_BACKTICKS"] == "`touch /tmp/cloud-compose-salt-backtick-injection`" - assert application_env["CONTRACT_BACKSLASH"] == "a\\path\\ends\\" - assert application_env["CONTRACT_COMMAND_SUB"] == "$(touch /tmp/cloud-compose-salt-command-injection)" - assert application_env["CONTRACT_DOLLARS"] == "$HOME ${HOME}" - assert application_env["CONTRACT_QUOTES"] == 'a "double" and a single quote: O\'Reilly' - assert application_env["CONTRACT_WHITESPACE"] == " leading and trailing " - assert application_env["CONTRACT_MULTILINE"] == "line one\nline two" - assert env["LIBOPS_MANAGED_RUNTIME_ENABLED"] == "false" - assert env["LIBOPS_INTERNAL_SERVICES_ENABLED"] == "false" - assert env["LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE"] == "false" - assert project["init_commands"] == [] - assert project["up_commands"] == [] - assert project["down_commands"] == ["app-down-command"] - assert project["rollout_commands"] == ["global-rollout-command"] - assert "contract-agent\thttps://example.invalid/contract-agent\t" in artifact_manifest - assert "\t/usr/local/bin/contract-agent\t0750\troot\troot\tcloud-compose.service" in artifact_manifest - assert not Path("/tmp/cloud-compose-salt-backtick-injection").exists() - assert not Path("/tmp/cloud-compose-salt-command-injection").exists() -elif expected_name == "drupal-prod": - assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { - "sitectl": "v1.0.0", - "sitectl-drupal": "v1.0.0", - } - -for path in [ - "/home/cloud-compose/init", - "/home/cloud-compose/up", - "/home/cloud-compose/down", - "/home/cloud-compose/rollout", - "/home/cloud-compose/run.sh", - "/home/cloud-compose/start-cloud-compose-bootstrap.sh", -]: - assert Path(path).exists(), path - assert os.access(path, os.X_OK), path - -cloud_compose_gid = grp.getgrnam("cloud-compose").gr_gid -for path, expected_mode in { - "/home/cloud-compose/init": 0o750, - "/home/cloud-compose/up": 0o750, - "/home/cloud-compose/down": 0o750, - "/home/cloud-compose/rollout": 0o750, - "/home/cloud-compose/.env": 0o640, - "/home/cloud-compose/application-env.json": 0o640, - "/home/cloud-compose/compose-projects.json": 0o640, - "/home/cloud-compose/managed-runtime-artifacts.tsv": 0o640, -}.items(): - metadata = Path(path).stat() - assert metadata.st_uid == 0, (path, metadata.st_uid) - assert metadata.st_gid == cloud_compose_gid, (path, metadata.st_gid) - assert stat.S_IMODE(metadata.st_mode) == expected_mode, (path, oct(stat.S_IMODE(metadata.st_mode))) -PY + python /work/ci/config-management-smoke-assert.py salt-runtime \ + "$expected_name" \ + "$expected_repo" \ + "$expected_plugin" \ + "$expected_package" \ + "$expected_domain" \ + "$expected_project_dir" \ + "$expected_compose_project_name" } run_salt_case \ @@ -524,7 +312,7 @@ run_salt_case \ sitectl-wp \ wp.example.edu \ /mnt/disks/data/libops/wp.git/wp-prod \ - libops-wp-v1-0-0 + libops-wp-v1-1-0 run_salt_case \ drupal-prod \ @@ -534,7 +322,7 @@ run_salt_case \ sitectl-drupal \ drupal.example.edu \ /mnt/disks/data/libops/drupal.git/drupal-prod \ - libops-drupal-v1-0-0 + libops-drupal-v1-1-0 run_invalid_salt_case() { local invalid_case="$1" expected="$2" From 8591a506e2d321474279c422518d33957570eac4 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 20:38:24 +0000 Subject: [PATCH 08/67] [minor] Harden first-customer runtime recovery --- ansible/roles/cloud_compose/tasks/main.yml | 31 ++- ci/bootstrap-recovery-contract.sh | 51 +++-- ci/config-management-cloud-smoke-inner.sh | 6 + ci/config-management-input-contract.sh | 50 ++++- ci/config-management-smoke-assert.py | 16 ++ ci/cos-jq-portability-contract.sh | 10 +- ci/disaster-recovery-contract.sh | 185 ++---------------- ci/filesystem-prep-contract.sh | 6 +- ci/gcp-upgrade-smoke-contract.sh | 2 +- ci/host-runtime-security.sh | 16 ++ ci/rootfs-package-contract.sh | 6 + ci/sitectl-version-contract.sh | 36 ++++ ci/systemd-contract.sh | 40 +++- .../assert-coverage-manifest.jq | 6 + .../disaster-recovery/assert-restore-proof.jq | 4 + ci/testdata/disaster-recovery/compose-apps.sh | 15 ++ .../disaster-recovery/fake-compose-config.jq | 12 ++ ci/testdata/disaster-recovery/fake-docker.sh | 8 + ci/testdata/disaster-recovery/fake-install.sh | 17 ++ ci/testdata/disaster-recovery/fake-stat.sh | 10 + .../disaster-recovery/good-backup-receipt.jq | 12 ++ ci/testdata/disaster-recovery/good-driver.sh | 30 +++ .../disaster-recovery/good-restore-proof.jq | 15 ++ .../incomplete-backup-receipt.jq | 12 ++ .../disaster-recovery/incomplete-driver.sh | 17 ++ ci/testdata/disaster-recovery/profile.sh | 9 + docs/runtime-contracts.md | 16 +- .../contracttest/cloud_smoke_cleanup_test.go | 16 +- .../testdata/cloud-smoke-lifecycle/ssh.sh | 9 + modules/gcp/variables.tf | 2 +- .../linux-vm-runtime/templates/cloud-init.yml | 4 +- .../system/cloud-compose-bootstrap.service | 4 +- .../system/cloud-compose-docker-prune.service | 2 +- .../system/cloud-compose-key-rotation.service | 2 +- ...loud-compose-metadata-firewall-pre.service | 2 +- .../cloud-compose-metadata-firewall.service | 2 +- .../cloud-compose-offhost-backup.service | 2 +- .../system/cloud-compose-overlay.service | 2 +- .../system/cloud-compose-restore-test.service | 2 +- .../system/cloud-compose-vault-agent.service | 6 +- .../system/libops-managed-runtime.service | 2 +- rootfs/etc/tmpfiles.d/cloud-compose.conf | 2 + .../cloud-compose/assert-app-initialized.sh | 2 +- .../home/cloud-compose/bootstrap-helpers.sh | 31 ++- .../cloud-compose/disaster-recovery-lib.sh | 54 ++--- rootfs/home/cloud-compose/host-init.sh | 24 ++- .../cloud-compose/libops-managed-runtime.sh | 132 ++++++++++++- rootfs/home/cloud-compose/offhost-backup.sh | 104 +++------- rootfs/home/cloud-compose/restore-test.sh | 2 + rootfs/home/cloud-compose/run.sh | 2 +- .../start-cloud-compose-bootstrap.sh | 2 +- .../cloud-compose/bootstrap-required.sh | 19 ++ .../cloud-compose/bootstrap-security.sh | 123 ++++++++++++ .../cloud-compose/require-bootstrap-ready.sh | 12 ++ .../libexec/cloud-compose/run-bootstrap.sh | 9 + .../libexec/cloud-compose/run-root-program.sh | 27 +++ .../start-cloud-compose-bootstrap.sh | 9 + .../local/sbin/cloud-compose-diagnostics.sh | 26 ++- .../cloud-compose/jq/diagnostics-entry-app.jq | 1 + .../jq/diagnostics-entry-project-dir.jq | 1 + .../jq/diagnostics-project-entries.jq | 1 + .../diagnostics-validate-compose-projects.jq | 1 + .../jq/dr-backup-completed-at.jq | 1 + .../cloud-compose/jq/dr-backup-remote-id.jq | 1 + .../jq/dr-restore-completed-at.jq | 1 + .../jq/dr-restore-recovery-id.jq | 1 + .../jq/dr-validate-backup-receipt.jq | 16 ++ .../jq/dr-validate-restore-proof.jq | 19 ++ .../cloud-compose/jq/offhost-bind-sources.jq | 1 + .../jq/offhost-build-application-coverage.jq | 27 +++ .../jq/offhost-build-manifest.jq | 10 + .../jq/offhost-manifest-app-digests.jq | 1 + .../jq/offhost-validate-bind-roots.jq | 8 + .../jq/offhost-validate-compose-config.jq | 11 ++ .../jq/offhost-validate-manifest.jq | 14 ++ salt/cloud-compose/init.sls | 27 ++- templates/cloud-init.yml | 9 +- 77 files changed, 1034 insertions(+), 392 deletions(-) create mode 100644 ci/testdata/disaster-recovery/assert-coverage-manifest.jq create mode 100644 ci/testdata/disaster-recovery/assert-restore-proof.jq create mode 100644 ci/testdata/disaster-recovery/compose-apps.sh create mode 100644 ci/testdata/disaster-recovery/fake-compose-config.jq create mode 100644 ci/testdata/disaster-recovery/fake-docker.sh create mode 100644 ci/testdata/disaster-recovery/fake-install.sh create mode 100644 ci/testdata/disaster-recovery/fake-stat.sh create mode 100644 ci/testdata/disaster-recovery/good-backup-receipt.jq create mode 100644 ci/testdata/disaster-recovery/good-driver.sh create mode 100644 ci/testdata/disaster-recovery/good-restore-proof.jq create mode 100644 ci/testdata/disaster-recovery/incomplete-backup-receipt.jq create mode 100644 ci/testdata/disaster-recovery/incomplete-driver.sh create mode 100644 ci/testdata/disaster-recovery/profile.sh create mode 100644 internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh create mode 100644 rootfs/usr/local/libexec/cloud-compose/bootstrap-required.sh create mode 100644 rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh create mode 100644 rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh create mode 100644 rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh create mode 100644 rootfs/usr/local/libexec/cloud-compose/run-root-program.sh create mode 100644 rootfs/usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh create mode 100644 rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-app.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-project-dir.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/diagnostics-project-entries.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/diagnostics-validate-compose-projects.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/dr-backup-completed-at.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/dr-backup-remote-id.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/dr-restore-completed-at.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/dr-restore-recovery-id.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/dr-validate-backup-receipt.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/dr-validate-restore-proof.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/offhost-bind-sources.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/offhost-build-application-coverage.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/offhost-build-manifest.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/offhost-manifest-app-digests.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/offhost-validate-bind-roots.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/offhost-validate-compose-config.jq create mode 100644 rootfs/usr/local/share/cloud-compose/jq/offhost-validate-manifest.jq diff --git a/ansible/roles/cloud_compose/tasks/main.yml b/ansible/roles/cloud_compose/tasks/main.yml index 1dd1912..8dcb9e2 100644 --- a/ansible/roles/cloud_compose/tasks/main.yml +++ b/ansible/roles/cloud_compose/tasks/main.yml @@ -543,17 +543,21 @@ (cloud_compose_lifecycle_lock_file.stat.mode | default('')) == '0660' ) -- name: Find cloud-compose shell scripts +- name: Find top-level cloud-compose control scripts ansible.builtin.find: - paths: "{{ cloud_compose_home }}" + paths: + - "{{ cloud_compose_home }}" + - /usr/local/libexec/cloud-compose patterns: "*.sh" - recurse: true + recurse: false file_type: file register: cloud_compose_shell_scripts -- name: Make cloud-compose shell scripts executable +- name: Make cloud-compose control scripts root-owned and executable ansible.builtin.file: path: "{{ item.path }}" + owner: root + group: root mode: "0755" loop: "{{ cloud_compose_shell_scripts.files }}" @@ -617,19 +621,28 @@ - name: Install and start authenticated rollout service ansible.builtin.command: - cmd: bash "{{ cloud_compose_home }}/deploy-rollout.sh" + cmd: bash /usr/local/libexec/cloud-compose/run-root-program.sh deploy-rollout.sh when: _cc_rollout_service.enabled | bool - name: Clear cloud-compose bootstrap marker ansible.builtin.file: - path: "{{ cloud_compose_home }}/.cloud-compose-bootstrap-complete" + path: /var/lib/cloud-compose/bootstrap-complete state: absent when: cloud_compose_force_bootstrap | bool +- name: Inspect validated cloud-compose bootstrap readiness + ansible.builtin.command: + cmd: bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh + register: cloud_compose_bootstrap_readiness + changed_when: false + failed_when: false + when: cloud_compose_run_bootstrap | bool + - name: Bootstrap cloud-compose runtime ansible.builtin.command: - cmd: bash "{{ cloud_compose_home }}/start-cloud-compose-bootstrap.sh" - creates: "{{ cloud_compose_home }}/.cloud-compose-bootstrap-complete" + cmd: bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh async: "{{ cloud_compose_bootstrap_timeout }}" poll: "{{ cloud_compose_bootstrap_poll_interval }}" - when: cloud_compose_run_bootstrap | bool + when: + - cloud_compose_run_bootstrap | bool + - (cloud_compose_bootstrap_readiness.rc | default(1)) != 0 diff --git a/ci/bootstrap-recovery-contract.sh b/ci/bootstrap-recovery-contract.sh index e37e1ba..bc20151 100644 --- a/ci/bootstrap-recovery-contract.sh +++ b/ci/bootstrap-recovery-contract.sh @@ -9,6 +9,11 @@ start_bootstrap="$repo_root/rootfs/home/cloud-compose/start-cloud-compose-bootst run_script="$repo_root/rootfs/home/cloud-compose/run.sh" app_init="$repo_root/rootfs/home/cloud-compose/app-init.sh" run_bootstrap="$repo_root/rootfs/home/cloud-compose/run-bootstrap.sh" +bootstrap_security="$repo_root/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh" +bootstrap_required="$repo_root/rootfs/usr/local/libexec/cloud-compose/bootstrap-required.sh" +bootstrap_entrypoint="$repo_root/rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh" +bootstrap_start_entrypoint="$repo_root/rootfs/usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh" +bootstrap_readiness_gate="$repo_root/rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh" bootstrap_unit="$repo_root/rootfs/etc/systemd/system/cloud-compose-bootstrap.service" tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT @@ -182,10 +187,14 @@ fi assert_contains "$app_init" 'acquire_cloud_compose_lifecycle_lock "init"' assert_contains "$app_init" 'trap release_cloud_compose_lifecycle_lock EXIT' assert_contains "$run_script" 'if cloud_compose_should_run_app_init' +assert_contains "$run_script" 'durable_bootstrap_marker="/var/lib/cloud-compose/bootstrap-complete"' assert_contains "$run_script" 'cloud_compose_publish_marker "$current_boot_app_init_marker"' assert_contains "$run_script" 'cloud_compose_start_and_wait_for_oneshot cloud-compose.service "$app_wait_seconds"' assert_contains "$run_script" '"$fresh_filesystem_marker" "$fresh_filesystem_identity"' assert_contains "$run_script" 'cloud_compose_publish_marker "$durable_bootstrap_marker"' +assert_contains "$helpers" 'if ((EUID == 0)) && ! chown 0:0 "$tmp_marker"; then' +assert_contains "$helpers" '0:0:644:1:regular file' +assert_contains "$helpers" '0:0:755:directory' [[ "$(grep -Fc 'cloud_compose_consume_fresh_filesystem_marker \' "$run_script")" == "1" ]] || fail "bootstrap must have one fresh-filesystem authority consumption boundary" rotation_line="$(grep -nF 'bash /home/cloud-compose/rotate-keys-daily.sh' "$run_script" | cut -d: -f1)" @@ -209,6 +218,16 @@ durable_marker_line="$(grep -nF 'cloud_compose_publish_marker "$durable_bootstra assert_contains "$run_bootstrap" 'if ((EUID != 0)); then' assert_contains "$run_bootstrap" 'exec bash /home/cloud-compose/run.sh' +assert_contains "$bootstrap_entrypoint" 'cloud_compose_secure_runtime_home' +assert_contains "$bootstrap_entrypoint" 'exec /bin/bash /home/cloud-compose/run.sh' +assert_contains "$bootstrap_start_entrypoint" 'cloud_compose_secure_runtime_home' +assert_contains "$bootstrap_required" 'cloud_compose_bootstrap_marker_ready' +assert_contains "$bootstrap_readiness_gate" 'cloud_compose_bootstrap_marker_ready' +assert_contains "$bootstrap_security" '0:0:644:1:regular file' +assert_contains "$bootstrap_security" '"$marker_size" == "6"' +assert_contains "$bootstrap_security" '"$payload" == "ready"' +assert_contains "$bootstrap_security" 'Cloud Compose control input is not root-controlled' +assert_contains "$bootstrap_security" 'Cloud Compose lifecycle dispatcher is not root-controlled' if rg -n 'bootstrap\\.log|exec (>>|>)[^[:space:]]' "$run_bootstrap" >/dev/null; then fail "bootstrap wrapper writes an independently unbounded log file" fi @@ -245,6 +264,7 @@ done fail "published sitectl is not executable after first-boot installation" assert_contains "$start_bootstrap" 'CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS:-10800' +assert_contains "$start_bootstrap" 'CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/var/lib/cloud-compose/bootstrap-complete' assert_contains "$start_bootstrap" 'if cloud_compose_marker_exists "$durable_marker"; then' assert_contains "$start_bootstrap" 'systemctl daemon-reload' assert_contains "$start_bootstrap" 'systemctl stop -- "$bootstrap_unit"' @@ -257,7 +277,7 @@ fi for cloud_init_template in \ "$repo_root/templates/cloud-init.yml" \ "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do - assert_contains "$cloud_init_template" 'bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' + assert_contains "$cloud_init_template" 'bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh' if grep -Fq 'bash /home/cloud-compose/run.sh > /home/cloud-compose/run.log 2>&1' \ "$cloud_init_template"; then fail "cloud-init bypasses the retryable bootstrap unit" @@ -304,6 +324,8 @@ rm -f -- "$durable_marker" integration_root="$tmp/integration" integration_home="$integration_root/home/cloud-compose" integration_run="$integration_root/run" +integration_state="$integration_root/var/lib/cloud-compose" +integration_durable_marker="$integration_state/bootstrap-complete" integration_bin="$integration_root/bin" integration_log="$integration_root/systemctl.log" app_init_count="$integration_root/app-init-count" @@ -311,7 +333,7 @@ fresh_data_root="$integration_root/data" fresh_marker="$fresh_data_root/.cloud-compose/fresh-filesystem" first_output="$integration_root/first-attempt.log" retry_output="$integration_root/retry-attempt.log" -mkdir -p "$integration_home" "$integration_run" "$integration_bin" "$(dirname -- "$fresh_marker")" +mkdir -p "$integration_home" "$integration_run" "$integration_state" "$integration_bin" "$(dirname -- "$fresh_marker")" : >"$integration_log" printf '0\n' >"$app_init_count" printf 'fresh\n' >"$fresh_marker" @@ -319,6 +341,7 @@ printf 'fresh\n' >"$fresh_marker" sed \ -e "s#/home/cloud-compose#$integration_home#g" \ -e "s#/run/cloud-compose-app-init-complete#$integration_run/app-init-complete#g" \ + -e "s#/var/lib/cloud-compose#$integration_state#g" \ "$run_script" >"$integration_home/run.sh" cp "$helpers" "$integration_home/bootstrap-helpers.sh" cat >"$integration_home/profile.sh" <<'EOF' @@ -453,7 +476,7 @@ fi fail "first full bootstrap attempt did not complete app-init exactly once" [[ -f "$integration_run/app-init-complete" ]] || fail "failed first attempt did not retain current-boot app-init readiness" -[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] || +[[ ! -e "$integration_durable_marker" ]] || fail "failed first attempt published durable bootstrap readiness" [[ ! -e "$fresh_marker" ]] || fail "failed first attempt retained fresh-filesystem authority past key convergence" @@ -462,7 +485,7 @@ env "${integration_env[@]}" RUN_ATTEMPT=recover \ bash "$integration_home/run.sh" >"$retry_output" 2>&1 [[ "$(<"$app_init_count")" == "1" ]] || fail "bootstrap retry repeated successful app-init" -[[ -f "$integration_home/.cloud-compose-bootstrap-complete" ]] || +[[ -f "$integration_durable_marker" ]] || fail "bootstrap retry did not publish durable readiness" [[ ! -e "$fresh_marker" ]] || fail "successful bootstrap retry retained fresh-filesystem reconciliation authority" @@ -474,7 +497,7 @@ grep -Fq 'recover enable -- cloud-compose.service' "$integration_log" || fail "bootstrap retry did not converge the application service" # Marker removal must be flushed before durable readiness can be republished. -rm -f -- "$integration_home/.cloud-compose-bootstrap-complete" +rm -f -- "$integration_durable_marker" printf 'fresh\n' >"$fresh_marker" if env "${integration_env[@]}" RUN_ATTEMPT=sync-fail \ bash "$integration_home/run.sh" >/dev/null 2>&1; then @@ -482,16 +505,16 @@ if env "${integration_env[@]}" RUN_ATTEMPT=sync-fail \ fi [[ ! -e "$fresh_marker" ]] || fail "post-consume durability coverage did not remove the fresh marker" -[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] || +[[ ! -e "$integration_durable_marker" ]] || fail "failed post-consume durability barrier published readiness" env "${integration_env[@]}" RUN_ATTEMPT=recover \ bash "$integration_home/run.sh" >/dev/null 2>&1 -[[ -f "$integration_home/.cloud-compose-bootstrap-complete" ]] || +[[ -f "$integration_durable_marker" ]] || fail "bootstrap retry did not flush an already-absent marker before readiness" # GCP never falls back to the generic non-GCP marker identity. This check runs # before key rotation, so missing disk identity cannot reach IAM. -rm -f -- "$integration_home/.cloud-compose-bootstrap-complete" +rm -f -- "$integration_durable_marker" printf 'fresh\n' >"$fresh_marker" if env "${integration_env[@]}" CLOUD_COMPOSE_PROVIDER=gcp RUN_ATTEMPT=recover \ bash "$integration_home/run.sh" >/dev/null 2>&1; then @@ -499,12 +522,12 @@ if env "${integration_env[@]}" CLOUD_COMPOSE_PROVIDER=gcp RUN_ATTEMPT=recover \ fi [[ -f "$fresh_marker" ]] || fail "GCP bootstrap consumed generic fresh-filesystem authority" -[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] || +[[ ! -e "$integration_durable_marker" ]] || fail "GCP bootstrap with generic authority published durable readiness" # A marker payload for another incarnation must fail before application # initialization or durable readiness. -rm -f -- "$integration_home/.cloud-compose-bootstrap-complete" +rm -f -- "$integration_durable_marker" printf 'v1:gcp-disk-id:111111111111111111\n' >"$fresh_marker" if env "${integration_env[@]}" RUN_ATTEMPT=recover \ bash "$integration_home/run.sh" >/dev/null 2>&1; then @@ -512,13 +535,13 @@ if env "${integration_env[@]}" RUN_ATTEMPT=recover \ fi [[ -f "$fresh_marker" ]] || fail "mismatched fresh-filesystem marker payload was consumed" -[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] || +[[ ! -e "$integration_durable_marker" ]] || fail "mismatched fresh-filesystem marker payload published durable readiness" # Unsafe authority must fail closed before durable readiness. The current-boot # app-init marker remains valid, so these attempts exercise only the early # marker boundary rather than repeating application initialization. -rm -f -- "$integration_home/.cloud-compose-bootstrap-complete" +rm -f -- "$integration_durable_marker" printf 'fresh\n' >"$fresh_marker" if env "${integration_env[@]}" RUN_ATTEMPT=recover \ FRESH_MARKER_IDENTITY=1000:1000:600:1 \ @@ -527,7 +550,7 @@ if env "${integration_env[@]}" RUN_ATTEMPT=recover \ fi [[ -f "$fresh_marker" ]] || fail "unsafe fresh-filesystem marker was consumed" -[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] || +[[ ! -e "$integration_durable_marker" ]] || fail "unsafe fresh-filesystem marker published durable readiness" rm -f -- "$fresh_marker" @@ -538,7 +561,7 @@ if env "${integration_env[@]}" RUN_ATTEMPT=recover \ fi [[ -L "$fresh_marker" ]] || fail "symlink fresh-filesystem marker was consumed" -[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] || +[[ ! -e "$integration_durable_marker" ]] || fail "symlink fresh-filesystem marker published durable readiness" echo "Bootstrap recovery contract passed" diff --git a/ci/config-management-cloud-smoke-inner.sh b/ci/config-management-cloud-smoke-inner.sh index f53111d..d872ff5 100755 --- a/ci/config-management-cloud-smoke-inner.sh +++ b/ci/config-management-cloud-smoke-inner.sh @@ -197,6 +197,12 @@ test -x /home/cloud-compose/down test -x /home/cloud-compose/rollout test -x /home/cloud-compose/run.sh test -x /home/cloud-compose/start-cloud-compose-bootstrap.sh +test -x /usr/local/libexec/cloud-compose/bootstrap-required.sh +test -x /usr/local/libexec/cloud-compose/bootstrap-security.sh +test -x /usr/local/libexec/cloud-compose/run-bootstrap.sh +test -x /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh +test -x /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh +test -x /usr/local/libexec/cloud-compose/run-root-program.sh python3 -m json.tool /home/cloud-compose/compose-projects.json >/dev/null python3 -m json.tool /home/cloud-compose/application-env.json >/dev/null diff --git a/ci/config-management-input-contract.sh b/ci/config-management-input-contract.sh index 7781958..a1e1d0a 100755 --- a/ci/config-management-input-contract.sh +++ b/ci/config-management-input-contract.sh @@ -11,6 +11,48 @@ cmp -s "$validator" "$salt_validator" || { exit 1 } +ansible_tasks="$repo_root/ansible/roles/cloud_compose/tasks/main.yml" +salt_state="$repo_root/salt/cloud-compose/init.sls" +root_program_runner="$repo_root/rootfs/usr/local/libexec/cloud-compose/run-root-program.sh" +rollout_runner="bash /usr/local/libexec/cloud-compose/run-root-program.sh deploy-rollout.sh" + +contract_fail() { + echo "config-management input contract: $1" >&2 + exit 1 +} + +grep -Fq "cmd: $rollout_runner" "$ansible_tasks" || \ + contract_fail "Ansible rollout does not use the trusted root-program runner" +if grep -Fq 'cmd: bash "{{ cloud_compose_home }}/' "$ansible_tasks" || \ + grep -Fq 'cmd: bash /home/cloud-compose/' "$ansible_tasks"; then + contract_fail "Ansible rollout executes a root program directly from the runtime home" +fi + +grep -Fq 'configure-metadata-firewall.sh | deploy-rollout.sh | docker-prune.sh' "$root_program_runner" || \ + contract_fail "the trusted root-program runner does not allow deploy-rollout.sh" + +salt_rollout_block="$(sed -n '/^cloud-compose-rollout-service:/,/^{% endif %}$/p' "$salt_state")" +[[ -n "$salt_rollout_block" ]] || contract_fail "Salt rollout state is missing" +if grep -Fq -- '- name: bash /home/cloud-compose/' "$salt_state"; then + contract_fail "Salt rollout executes a root program directly from the runtime home" +fi +for marker in \ + "- name: $rollout_runner" \ + '- file: cloud-compose-env' \ + '- file: cloud-compose-application-env' \ + '- file: cloud-compose-project-manifest' \ + '- file: cloud-compose-managed-runtime-artifacts' \ + '- file: cloud-compose-rootfs' \ + '- cmd: cloud-compose-lifecycle-lock' \ + '- cmd: cloud-compose-rootfs-script-modes' \ + '- file: cloud-compose-lifecycle-init' \ + '- file: cloud-compose-lifecycle-up' \ + '- file: cloud-compose-lifecycle-down' \ + '- file: cloud-compose-lifecycle-rollout'; do + grep -Fq -- "$marker" <<<"$salt_rollout_block" || \ + contract_fail "Salt rollout state is missing $marker" +done + python3 - "$repo_root" "$validator" <<'PY' import base64 import copy @@ -175,8 +217,10 @@ if "files/validate-runtime-inputs.py" not in ansible_tasks: fail("Ansible does not execute the shared host-input validator") if "--data-root" in ansible_tasks: fail("Ansible makes the production project ownership boundary configurable") -if 'cmd: bash "{{ cloud_compose_home }}/start-cloud-compose-bootstrap.sh"' not in ansible_tasks: +if "cmd: bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh" not in ansible_tasks: fail("Ansible bypasses the retryable bootstrap service") +if "cmd: bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh" not in ansible_tasks: + fail("Ansible does not validate bootstrap readiness evidence") if 'cmd: bash "{{ cloud_compose_home }}/run.sh"' in ansible_tasks: fail("Ansible still invokes the one-shot bootstrap script directly") if "cloud_compose_bootstrap_timeout: 14400" not in ansible_defaults: @@ -189,8 +233,10 @@ salt_gate = salt_state.find("cloud-compose-host-inputs-valid:") salt_first_mutation = salt_state.find("cloud-compose-packages:") if salt_gate < 0 or salt_first_mutation < 0 or salt_gate > salt_first_mutation: fail("Salt host-input validation does not precede its first host mutation") -if "home ~ '/start-cloud-compose-bootstrap.sh'" not in salt_state: +if "bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh" not in salt_state: fail("Salt bypasses the retryable bootstrap service") +if "bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh" not in salt_state: + fail("Salt does not validate bootstrap readiness evidence") if "home ~ '/run.sh'" in salt_state: fail("Salt still invokes the one-shot bootstrap script directly") gate_block = salt_state[salt_gate:salt_first_mutation] diff --git a/ci/config-management-smoke-assert.py b/ci/config-management-smoke-assert.py index b2ea3f0..247163b 100644 --- a/ci/config-management-smoke-assert.py +++ b/ci/config-management-smoke-assert.py @@ -12,6 +12,7 @@ RUNTIME_HOME = Path("/home/cloud-compose") +BOOTSTRAP_LIBEXEC = Path("/usr/local/libexec/cloud-compose") def load_runtime_env(path: Path) -> dict[str, str]: @@ -45,6 +46,12 @@ def assert_runtime_files() -> None: RUNTIME_HOME / "rollout", RUNTIME_HOME / "run.sh", RUNTIME_HOME / "start-cloud-compose-bootstrap.sh", + BOOTSTRAP_LIBEXEC / "bootstrap-required.sh", + BOOTSTRAP_LIBEXEC / "bootstrap-security.sh", + BOOTSTRAP_LIBEXEC / "run-bootstrap.sh", + BOOTSTRAP_LIBEXEC / "require-bootstrap-ready.sh", + BOOTSTRAP_LIBEXEC / "run-root-program.sh", + BOOTSTRAP_LIBEXEC / "start-cloud-compose-bootstrap.sh", ]: assert path.exists(), path assert os.access(path, os.X_OK), path @@ -68,6 +75,15 @@ def assert_runtime_files() -> None: oct(stat.S_IMODE(metadata.st_mode)), ) + for path in BOOTSTRAP_LIBEXEC.glob("*.sh"): + metadata = path.stat() + assert metadata.st_uid == 0, (path, metadata.st_uid) + assert metadata.st_gid == 0, (path, metadata.st_gid) + assert stat.S_IMODE(metadata.st_mode) == 0o755, ( + path, + oct(stat.S_IMODE(metadata.st_mode)), + ) + def assert_ansible_runtime() -> None: env = load_runtime_env(RUNTIME_HOME / ".env") diff --git a/ci/cos-jq-portability-contract.sh b/ci/cos-jq-portability-contract.sh index 9780f3c..f349150 100644 --- a/ci/cos-jq-portability-contract.sh +++ b/ci/cos-jq-portability-contract.sh @@ -3,13 +3,17 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -runtime_root="$repo_root/rootfs/home/cloud-compose" +runtime_program_roots=( + "$repo_root/rootfs/home/cloud-compose" + "$repo_root/rootfs/usr/local/share/cloud-compose/jq" +) regex_call_pattern='(^|[^[:alnum:]_])(test|match|capture|scan|splits|sub|gsub)[[:space:]]*\(' nul_contains_pattern='contains[[:space:]]*\([[:space:]]*"\\u0000"[[:space:]]*\)' failed=false # Container-Optimized OS ships jq without Oniguruma. Its runtime scripts may # use jq for JSON structure and types, but text validation belongs in Bash. +# Check both shell call sites and the deployed jq program library. # The remaining sub(/.../) and gsub(/.../) calls are awk regex literals, not # jq filters; jq has no slash-delimited regex syntax. while IFS= read -r call; do @@ -18,7 +22,7 @@ while IFS= read -r call; do fi printf 'COS runtime contains a regex-dependent jq-style call: %s\n' "$call" >&2 failed=true -done < <(grep -ERn --include='*.sh' "$regex_call_pattern" "$runtime_root" || true) +done < <(grep -ERn --include='*.sh' --include='*.jq' "$regex_call_pattern" "${runtime_program_roots[@]}" || true) # jq 1.6 treats every string as containing a NUL when contains("\u0000") is # used. COS and supported configuration-management hosts can still run jq 1.6, @@ -26,7 +30,7 @@ done < <(grep -ERn --include='*.sh' "$regex_call_pattern" "$runtime_root" || tru while IFS= read -r call; do printf 'Runtime contains jq 1.6-incompatible NUL validation: %s\n' "$call" >&2 failed=true -done < <(grep -ERn --include='*.sh' "$nul_contains_pattern" "$runtime_root" || true) +done < <(grep -ERn --include='*.sh' --include='*.jq' "$nul_contains_pattern" "${runtime_program_roots[@]}" || true) if [[ "$failed" == "true" ]]; then exit 1 diff --git a/ci/disaster-recovery-contract.sh b/ci/disaster-recovery-contract.sh index bfdfa23..bb66236 100644 --- a/ci/disaster-recovery-contract.sh +++ b/ci/disaster-recovery-contract.sh @@ -5,6 +5,7 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" backup_script="$repo_root/rootfs/home/cloud-compose/offhost-backup.sh" restore_script="$repo_root/rootfs/home/cloud-compose/restore-test.sh" +fixture_root="$repo_root/ci/testdata/disaster-recovery" tmp="$(mktemp -d "$repo_root/.disaster-recovery-contract.XXXXXX")" trap 'rm -rf -- "$tmp"' EXIT @@ -16,158 +17,19 @@ fail() { mkdir -p "$tmp/bin" "$tmp/data/projects/alpha/files" "$tmp/data/backups/mariadb/alpha" "$tmp/drivers" chmod 0755 "$tmp" "$tmp/bin" "$tmp/data" "$tmp/data/projects" "$tmp/data/projects/alpha" "$tmp/data/projects/alpha/files" "$tmp/drivers" -cat >"$tmp/profile.sh" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail -export PATH="${TEST_BIN:?}:/usr/bin:/bin" -acquire_cloud_compose_lifecycle_lock() { - printf '%s\n' "$1" >>"${LOCK_LOG:?}" -} -EOF - -cat >"$tmp/compose-apps.sh" <<'EOF' -#!/usr/bin/env bash -compose_app_names_array() { - local -n result="$1" - result=(alpha) -} -source_compose_app_env() { - DOCKER_COMPOSE_DIR="${TEST_DATA_ROOT:?}/projects/$1" - export DOCKER_COMPOSE_DIR -} -validate_compose_project_dir() { - [[ "$1" == "${TEST_DATA_ROOT:?}/projects/"* ]] -} -EOF - -cat >"$tmp/bin/stat" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail -output="$(/usr/bin/stat "$@")" -if [[ "$*" == *"%u:"* ]]; then - printf '0:%s\n' "${output#*:}" -else - printf '%s\n' "$output" -fi -EOF - -cat >"$tmp/bin/install" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail -args=() -while (($# > 0)); do - case "$1" in - -o|-g) - shift 2 - ;; - *) - args+=("$1") - shift - ;; - esac -done -exec /usr/bin/install "${args[@]}" -EOF - -cat >"$tmp/bin/docker" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail -[[ "$*" == "compose config --format json" ]] -jq -cn --arg root "${TEST_DATA_ROOT:?}" '{ - services: { - web: { - volumes: [ - {type: "bind", source: ($root + "/projects/alpha/files"), target: "/srv/files", read_only: false}, - {type: "volume", source: "alpha_data", target: "/var/lib/app", read_only: false}, - {type: "tmpfs", source: "", target: "/run/app", read_only: false} - ] - } - }, - volumes: {alpha_data: {name: "alpha_data"}} -}' -EOF - -cat >"$tmp/drivers/good-driver" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail -[[ -z "${SHOULD_NOT_REACH_DRIVER+x}" ]] || exit 90 -printf '%s\n' "$1" >>"${0}.calls" -operation="$1" -shift -declare -A args=() -while (($# > 0)); do - args["$1"]="$2" - shift 2 -done -case "$operation" in - backup) - jq -cn \ - --arg operation_id "${args[--operation-id]}" \ - --arg manifest_sha256 "${args[--manifest-sha256]}" '{ - schema_version: 1, - kind: "cloud-compose.offhost-backup-receipt", - operation_id: $operation_id, - completed_at: "2026-08-07T12:00:00Z", - manifest_sha256: $manifest_sha256, - encrypted: true, - off_host: true, - status: "succeeded", - remote_id: "contract/backup-1", - coverage: {database: true, application_files: true, volume_topology: true} - }' >"${args[--receipt]}" - ;; - restore-test) - jq -cn \ - --arg test_id "${args[--test-id]}" \ - --arg manifest_sha256 "${args[--source-manifest-sha256]}" \ - --arg receipt_sha256 "${args[--source-receipt-sha256]}" '{ - schema_version: 1, - kind: "cloud-compose.restore-test-proof", - test_id: $test_id, - completed_at: "2026-08-07T13:00:00Z", - source_manifest_sha256: $manifest_sha256, - source_receipt_sha256: $receipt_sha256, - source_encrypted: true, - status: "succeeded", - recovery_id: "contract/recovery-1", - disposable_recovery: true, - recovery_destroyed: true, - integrity_verified: true, - coverage: {database: true, application_files: true, volume_topology: true} - }' >"${args[--proof]}" - ;; - *) exit 64 ;; -esac -EOF - -cat >"$tmp/drivers/incomplete-driver" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail -operation="$1" -shift -declare -A args=() -while (($# > 0)); do - args["$1"]="$2" - shift 2 -done -[[ "$operation" == "backup" ]] -jq -cn \ - --arg operation_id "${args[--operation-id]}" \ - --arg manifest_sha256 "${args[--manifest-sha256]}" '{ - schema_version: 1, - kind: "cloud-compose.offhost-backup-receipt", - operation_id: $operation_id, - completed_at: "2026-08-07T12:00:00Z", - manifest_sha256: $manifest_sha256, - encrypted: true, - off_host: true, - status: "succeeded", - remote_id: "contract/incomplete", - coverage: {database: true, application_files: true, volume_topology: false} - }' >"${args[--receipt]}" -EOF - -chmod 0755 "$tmp/bin/stat" "$tmp/bin/install" "$tmp/bin/docker" "$tmp/drivers/good-driver" "$tmp/drivers/incomplete-driver" +/usr/bin/install -m 0644 "$fixture_root/profile.sh" "$tmp/profile.sh" +/usr/bin/install -m 0644 "$fixture_root/compose-apps.sh" "$tmp/compose-apps.sh" +/usr/bin/install -m 0755 "$fixture_root/fake-stat.sh" "$tmp/bin/stat" +/usr/bin/install -m 0755 "$fixture_root/fake-install.sh" "$tmp/bin/install" +/usr/bin/install -m 0755 "$fixture_root/fake-docker.sh" "$tmp/bin/docker" +/usr/bin/install -m 0644 "$fixture_root/fake-compose-config.jq" "$tmp/bin/fake-compose-config.jq" +/usr/bin/install -m 0755 "$fixture_root/good-driver.sh" "$tmp/drivers/good-driver" +/usr/bin/install -m 0755 "$fixture_root/incomplete-driver.sh" "$tmp/drivers/incomplete-driver" +/usr/bin/install -m 0644 \ + "$fixture_root/good-backup-receipt.jq" \ + "$fixture_root/good-restore-proof.jq" \ + "$fixture_root/incomplete-backup-receipt.jq" \ + "$tmp/drivers/" printf 'logical database\n' | gzip -c >"$tmp/data/backups/mariadb/alpha/$(date -u +%Y%m%d)-alpha.sql.gz" export TEST_BIN="$tmp/bin" @@ -177,6 +39,7 @@ export CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh" export CLOUD_COMPOSE_COMPOSE_APPS_PATH="$tmp/compose-apps.sh" export CLOUD_COMPOSE_DR_LIBRARY_PATH="$repo_root/rootfs/home/cloud-compose/disaster-recovery-lib.sh" export CLOUD_COMPOSE_DR_STATE_ROOT="$tmp/data/dr" +export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/usr/local/share/cloud-compose/jq" export CLOUD_COMPOSE_DATA_ROOT="$tmp/data" export CLOUD_COMPOSE_VOLUMES_ROOT="$tmp/data/volumes" export MARIADB_BACKUP_ROOT="$tmp/data/backups/mariadb" @@ -191,14 +54,8 @@ bash "$backup_script" receipt="$(find "$CLOUD_COMPOSE_DR_STATE_ROOT/backup-receipts" -maxdepth 1 -type f -name '*.json' -print -quit)" manifest="$(find "$CLOUD_COMPOSE_DR_STATE_ROOT/manifests" -maxdepth 1 -type f -name '*.json' -print -quit)" [[ -n "$receipt" && -n "$manifest" ]] || fail "successful driver did not publish its atomic manifest and receipt" -jq -e ' - .required_coverage == ["database", "application_files", "volume_topology"] and - (.applications | length == 1) and - (.applications[0].databases | length == 1) and - .applications[0].application_files.roots == [env.TEST_DATA_ROOT + "/projects/alpha"] and - .applications[0].volume_topology.declared_named_volumes == ["alpha_data"] and - (.applications[0].volume_topology.service_mounts | length == 3) -' "$manifest" >/dev/null || fail "coverage manifest omitted database, application files, or volume topology" +jq -e -f "$fixture_root/assert-coverage-manifest.jq" "$manifest" >/dev/null || \ + fail "coverage manifest omitted database, application files, or volume topology" # A second attempt sees the same already-existing daily dump but must still # invoke off-host transfer, allowing a failed first handoff to be retried. @@ -218,12 +75,8 @@ export CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="$tmp/drivers/good-driver" bash "$restore_script" proof="$(find "$CLOUD_COMPOSE_DR_STATE_ROOT/restore-proofs" -maxdepth 1 -type f -name '*.json' -print -quit)" [[ -n "$proof" ]] || fail "scheduled disposable restore test did not publish proof" -jq -e ' - .disposable_recovery == true and - .recovery_destroyed == true and - .integrity_verified == true and - .coverage == {database: true, application_files: true, volume_topology: true} -' "$proof" >/dev/null || fail "restore proof omitted required recovery evidence" +jq -e -f "$fixture_root/assert-restore-proof.jq" "$proof" >/dev/null || \ + fail "restore proof omitted required recovery evidence" rm -f -- "$tmp/data/backups/mariadb/alpha/$(date -u +%Y%m%d)-alpha.sql.gz" calls_before="$(wc -l <"$tmp/drivers/good-driver.calls")" diff --git a/ci/filesystem-prep-contract.sh b/ci/filesystem-prep-contract.sh index 270e730..722747d 100644 --- a/ci/filesystem-prep-contract.sh +++ b/ci/filesystem-prep-contract.sh @@ -643,11 +643,11 @@ for cloud_init_template in \ "$cloud_init_template" || fail "cloud-init does not verify every required mount before initialization" grep -Fq 'Required cloud-compose mount is unavailable:' "$cloud_init_template" || fail "cloud-init mount gate does not report the unavailable path" - grep -Fq ' bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' \ + grep -Fq ' bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh' \ "$cloud_init_template" || fail "cloud-init retryable bootstrap startup is outside the fail-closed mount block" - marker_reset_line="$(grep -nF ' rm -f /home/cloud-compose/.cloud-compose-bootstrap-complete' \ + marker_reset_line="$(grep -nF ' rm -f /var/lib/cloud-compose/bootstrap-complete' \ "$cloud_init_template" | cut -d: -f1)" - run_line="$(grep -nF ' bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' \ + run_line="$(grep -nF ' bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh' \ "$cloud_init_template" | cut -d: -f1)" [[ -n "$marker_reset_line" && -n "$run_line" && "$marker_reset_line" -lt "$run_line" ]] || fail "cloud-init does not clear stale bootstrap readiness before retryable startup" diff --git a/ci/gcp-upgrade-smoke-contract.sh b/ci/gcp-upgrade-smoke-contract.sh index adb84ec..3c887d7 100755 --- a/ci/gcp-upgrade-smoke-contract.sh +++ b/ci/gcp-upgrade-smoke-contract.sh @@ -175,7 +175,7 @@ baseline_initcmd_line="$(grep -nF 'for CMD in ADDITIONAL_INITCMD' "$tmp/baseline fail "baseline cloud-init does not execute fixture initcmd before run.sh" current_bootstrap_line="$( - grep -nF 'bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' \ + grep -nF 'bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh' \ "$repo_root/templates/cloud-init.yml" | cut -d: -f1 || true )" diff --git a/ci/host-runtime-security.sh b/ci/host-runtime-security.sh index 6a76810..e298e30 100644 --- a/ci/host-runtime-security.sh +++ b/ci/host-runtime-security.sh @@ -165,6 +165,8 @@ assert_contains "$profile_script" 'export PATH="/usr/local/sbin:/usr/local/bin:/ tmpfiles_conf="$repo_root/rootfs/etc/tmpfiles.d/cloud-compose.conf" assert_contains "$tmpfiles_conf" 'd /run/lock/cloud-compose 0750 root cloud-compose -' assert_contains "$tmpfiles_conf" 'f /run/lock/cloud-compose/lifecycle.lock 0660 root cloud-compose -' +assert_contains "$tmpfiles_conf" 'd /var/lib/cloud-compose 0755 root root -' +assert_contains "$tmpfiles_conf" 'd /home/cloud-compose 0755 root root -' host_init="$repo_root/rootfs/home/cloud-compose/host-init.sh" if grep -Eq 'chown[[:space:]]+-R[[:space:]]+cloud-compose[^[:space:]]*[[:space:]]+/home/cloud-compose' "$host_init"; then fail "host initialization gives the app account recursive ownership of root-executed code" @@ -174,11 +176,25 @@ if grep -Eq 'chown[[:space:]]+-R[[:space:]]+cloud-compose[^[:space:]]*[[:space:] fi assert_contains "$host_init" 'chown root:root /home/cloud-compose' assert_contains "$host_init" "-exec chown root:root {} +" +assert_contains "$host_init" 'Unsafe Cloud Compose lifecycle dispatcher:' +assert_contains "$repo_root/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh" \ + 'Cloud Compose control input is not root-controlled' assert_contains "$host_init" '/home/cloud-compose/.sitectl \' assert_contains "$host_init" 'install -d -m 0750 -o cloud-compose -g cloud-compose "$mutable_dir"' +assert_contains "$host_init" 'Managed command directory was not secured by the runtime installer' +if sed -n '/for mutable_dir in/,/done/p' "$host_init" | grep -Fq '/home/cloud-compose/bin'; then + fail "host initialization leaves the privileged published command directory app-writable" +fi assert_contains "$host_init" 'install -d -m 1775 -o root -g cloud-compose /mnt/disks/data' assert_contains "$host_init" 'install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/volumes' assert_contains "$host_init" 'install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops' +managed_runtime="$repo_root/rootfs/home/cloud-compose/libops-managed-runtime.sh" +assert_contains "$managed_runtime" 'prepare_managed_runtime_directory' +assert_contains "$managed_runtime" 'validate_published_bin_directory' +assert_contains "$managed_runtime" '$STATE_DIR:0755:false' +assert_contains "$managed_runtime" '$PUBLISHED_BIN_DIR:0755:true' +assert_contains "$managed_runtime" 'managed runtime updates must run as root' +assert_contains "$managed_runtime" 'production managed runtime directories require a root updater' if grep -A12 -F 'source_compose_app_env()' "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" | \ grep -Eq 'source[[:space:]]+.*COMPOSE_APPS_ENV_DIR'; then fail "privileged Compose manifest loading still sources an app-writable shell file" diff --git a/ci/rootfs-package-contract.sh b/ci/rootfs-package-contract.sh index 0c00046..47f1737 100755 --- a/ci/rootfs-package-contract.sh +++ b/ci/rootfs-package-contract.sh @@ -20,5 +20,11 @@ tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | awk ' END { exit bad } ' tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/home/cloud-compose/run.sh' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ + grep -Fx 'rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ + grep -Fx 'rootfs/usr/local/libexec/cloud-compose/run-root-program.sh' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ + grep -Fx 'rootfs/usr/local/share/cloud-compose/jq/offhost-validate-manifest.jq' >/dev/null echo "Rootfs package contract passed" diff --git a/ci/sitectl-version-contract.sh b/ci/sitectl-version-contract.sh index 0ec2f60..b93e13a 100644 --- a/ci/sitectl-version-contract.sh +++ b/ci/sitectl-version-contract.sh @@ -42,6 +42,42 @@ CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh" \ test "$(stat -c "%a" "$TMP_DIR")" = 700 test "$(stat -c "%a" "$PACKAGE_STATE_DIR")" = 700 test "$(stat -c "%a" "$ARTIFACT_STATE_DIR")" = 700 + test "$(stat -c "%a" "$PUBLISHED_BIN_DIR")" = 755 + test "$(stat -c "%u:%g" "$STATE_DIR")" = "$(id -u):$(id -g)" + test "$(stat -c "%u:%g" "$PUBLISHED_BIN_DIR")" = "$(id -u):$(id -g)" + + touch "$PUBLISHED_BIN_DIR/docker" + if mkdirs; then + echo "managed runtime accepted an unmanaged published command" >&2 + exit 1 + fi + rm -f "$PUBLISHED_BIN_DIR/docker" + + unsafe_target="$2/unsafe-target" + unsafe_state="$2/unsafe-state" + mkdir -p "$unsafe_target" + ln -s "$unsafe_target" "$unsafe_state" + STATE_DIR="$unsafe_state" + BIN_DIR="$STATE_DIR/bin" + TMP_DIR="$STATE_DIR/tmp" + PACKAGE_STATE_DIR="$STATE_DIR/packages" + ARTIFACT_STATE_DIR="$STATE_DIR/artifacts" + if mkdirs; then + echo "managed runtime accepted a redirected state directory" >&2 + exit 1 + fi + + unsafe_state="$2/unsafe-writable-state" + mkdir -m 0775 "$unsafe_state" + STATE_DIR="$unsafe_state" + BIN_DIR="$STATE_DIR/bin" + TMP_DIR="$STATE_DIR/tmp" + PACKAGE_STATE_DIR="$STATE_DIR/packages" + ARTIFACT_STATE_DIR="$STATE_DIR/artifacts" + if mkdirs; then + echo "managed runtime accepted group-writable state" >&2 + exit 1 + fi ' cloud-compose-sitectl-modes "$runtime_script" "$tmp" run_contract() { diff --git a/ci/systemd-contract.sh b/ci/systemd-contract.sh index da2550d..9ce70de 100755 --- a/ci/systemd-contract.sh +++ b/ci/systemd-contract.sh @@ -6,6 +6,8 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" unit_dir="$repo_root/rootfs/etc/systemd/system" diagnostics_program="$repo_root/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh" smoke_healthcheck_program="$repo_root/rootfs/home/cloud-compose/smoke-healthcheck.sh" +bootstrap_security="$repo_root/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh" +root_program_runner="$repo_root/rootfs/usr/local/libexec/cloud-compose/run-root-program.sh" fail() { echo "systemd contract: $*" >&2 @@ -28,6 +30,8 @@ assert_contains() { [[ -f "$unit_dir/cloud-compose-overlay.service" ]] || fail "Docker overlay mount unit is missing" [[ -f "$unit_dir/cloud-compose-bootstrap.service" ]] || fail "retryable bootstrap unit is missing" [[ -x "$diagnostics_program" ]] || fail "checked-in Cloud Compose diagnostics program is missing or not executable" +[[ -f "$bootstrap_security" ]] || fail "root-owned bootstrap security helper is missing" +[[ -f "$root_program_runner" ]] || fail "root-owned service launcher is missing" [[ -x "$smoke_healthcheck_program" ]] || fail "checked-in smoke healthcheck wrapper is missing or not executable" [[ -f "$unit_dir/cloud-compose-internal-services.service" && -f "$unit_dir/cloud-compose-internal-services.timer" ]] || \ fail "namespaced internal-services units are missing" @@ -41,8 +45,11 @@ assert_contains "$unit_dir/cloud-compose.service" 'After=network-online.target d assert_contains "$unit_dir/cloud-compose.service" 'ExecStartPre=/bin/bash /home/cloud-compose/assert-app-initialized.sh' assert_contains "$unit_dir/cloud-compose.service" 'Restart=on-failure' assert_contains "$unit_dir/cloud-compose.service" 'RestartSec=30s' -assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ConditionPathExists=!/home/cloud-compose/.cloud-compose-bootstrap-complete' -assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecStart=/bin/bash /home/cloud-compose/run-bootstrap.sh' +if grep -Fq 'ConditionPathExists=' "$unit_dir/cloud-compose-bootstrap.service"; then + fail "bootstrap still trusts an unvalidated marker path condition" +fi +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecCondition=/bin/bash /usr/local/libexec/cloud-compose/bootstrap-required.sh' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-bootstrap.sh' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'Restart=on-failure' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'RestartSec=30s' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'TimeoutStartSec=2h' @@ -58,6 +65,16 @@ assert_contains "$unit_dir/cloud-compose-internal-services.service" 'Requires=cl assert_contains "$unit_dir/cloud-compose.service" 'TimeoutStartSec=90min' assert_contains "$diagnostics_program" 'usage: ${diagnostics_program} state|status|dump' assert_contains "$diagnostics_program" 'readonly PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"' +assert_contains "$diagnostics_program" 'readonly bootstrap_marker="/var/lib/cloud-compose/bootstrap-complete"' +assert_contains "$bootstrap_security" 'cloud_compose_bootstrap_marker_ready()' +assert_contains "$bootstrap_security" '0:0:644:1:regular file' +assert_contains "$bootstrap_security" '"$marker_size" == "6"' +assert_contains "$bootstrap_security" '"$payload" == "ready"' +assert_contains "$bootstrap_security" 'cloud_compose_secure_runtime_home()' +assert_contains "$root_program_runner" 'cloud_compose_secure_runtime_home' +assert_contains "$root_program_runner" 'Unsupported Cloud Compose root program:' +[[ -f "$repo_root/rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh" ]] || \ + fail "validated bootstrap readiness gate is missing" assert_contains "$diagnostics_program" '--- Cloud Compose provisioning heartbeat ---' assert_contains "$diagnostics_program" 'ps -p "$main_pid" -o pid=,ppid=,stat=,etime=,comm=' assert_contains "$smoke_healthcheck_program" 'source /home/cloud-compose/profile.sh' @@ -99,9 +116,24 @@ assert_contains "$docker_metadata_dropin" 'Requires=cloud-compose-metadata-firew assert_contains "$docker_metadata_dropin" 'After=cloud-compose-metadata-firewall-pre.service' assert_contains "$metadata_pre_unit" 'Before=docker.service' assert_contains "$metadata_pre_unit" 'WantedBy=multi-user.target' -assert_contains "$metadata_pre_unit" 'ExecStart=/bin/bash /home/cloud-compose/configure-metadata-firewall.sh pre-docker' +assert_contains "$metadata_pre_unit" 'ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh configure-metadata-firewall.sh pre-docker' assert_contains "$unit_dir/cloud-compose-overlay.service" 'Before=docker.service cloud-compose.service' -assert_contains "$unit_dir/cloud-compose-overlay.service" 'ExecStart=/bin/bash /home/cloud-compose/mount-overlays.sh' +assert_contains "$unit_dir/cloud-compose-overlay.service" 'ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh mount-overlays.sh' +for root_home_unit in \ + cloud-compose-docker-prune.service \ + cloud-compose-key-rotation.service \ + cloud-compose-metadata-firewall-pre.service \ + cloud-compose-metadata-firewall.service \ + cloud-compose-offhost-backup.service \ + cloud-compose-overlay.service \ + cloud-compose-restore-test.service \ + cloud-compose-vault-agent.service \ + libops-managed-runtime.service; do + if grep -Eq '^Exec(Start|StartPre|StartPost|Stop|StopPost)=/bin/bash /home/cloud-compose/' "$unit_dir/$root_home_unit"; then + fail "$root_home_unit executes historically writable home code without the root-owned launcher" + fi + assert_contains "$unit_dir/$root_home_unit" '/usr/local/libexec/cloud-compose/run-root-program.sh' +done if grep -Eq '^(After|Before|BindsTo|PartOf|Requires|Requisite|Wants)=.*cloud-compose-bootstrap\\.service' \ "$unit_dir/cloud-compose.service"; then fail "application service has an ordering dependency on the bootstrap service" diff --git a/ci/testdata/disaster-recovery/assert-coverage-manifest.jq b/ci/testdata/disaster-recovery/assert-coverage-manifest.jq new file mode 100644 index 0000000..2aedb1c --- /dev/null +++ b/ci/testdata/disaster-recovery/assert-coverage-manifest.jq @@ -0,0 +1,6 @@ +.required_coverage == ["database", "application_files", "volume_topology"] and +(.applications | length == 1) and +(.applications[0].databases | length == 1) and +.applications[0].application_files.roots == [env.TEST_DATA_ROOT + "/projects/alpha"] and +.applications[0].volume_topology.declared_named_volumes == ["alpha_data"] and +(.applications[0].volume_topology.service_mounts | length == 3) diff --git a/ci/testdata/disaster-recovery/assert-restore-proof.jq b/ci/testdata/disaster-recovery/assert-restore-proof.jq new file mode 100644 index 0000000..c7752de --- /dev/null +++ b/ci/testdata/disaster-recovery/assert-restore-proof.jq @@ -0,0 +1,4 @@ +.disposable_recovery == true and +.recovery_destroyed == true and +.integrity_verified == true and +.coverage == {database: true, application_files: true, volume_topology: true} diff --git a/ci/testdata/disaster-recovery/compose-apps.sh b/ci/testdata/disaster-recovery/compose-apps.sh new file mode 100644 index 0000000..864de1f --- /dev/null +++ b/ci/testdata/disaster-recovery/compose-apps.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +compose_app_names_array() { + local -n result="$1" + result=(alpha) +} + +source_compose_app_env() { + DOCKER_COMPOSE_DIR="${TEST_DATA_ROOT:?}/projects/$1" + export DOCKER_COMPOSE_DIR +} + +validate_compose_project_dir() { + [[ "$1" == "${TEST_DATA_ROOT:?}/projects/"* ]] +} diff --git a/ci/testdata/disaster-recovery/fake-compose-config.jq b/ci/testdata/disaster-recovery/fake-compose-config.jq new file mode 100644 index 0000000..919adbb --- /dev/null +++ b/ci/testdata/disaster-recovery/fake-compose-config.jq @@ -0,0 +1,12 @@ +{ + services: { + web: { + volumes: [ + {type: "bind", source: ($root + "/projects/alpha/files"), target: "/srv/files", read_only: false}, + {type: "volume", source: "alpha_data", target: "/var/lib/app", read_only: false}, + {type: "tmpfs", source: "", target: "/run/app", read_only: false} + ] + } + }, + volumes: {alpha_data: {name: "alpha_data"}} +} diff --git a/ci/testdata/disaster-recovery/fake-docker.sh b/ci/testdata/disaster-recovery/fake-docker.sh new file mode 100644 index 0000000..9e04044 --- /dev/null +++ b/ci/testdata/disaster-recovery/fake-docker.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fixture_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +[[ "$*" == "compose config --format json" ]] +jq -cn --arg root "${TEST_DATA_ROOT:?}" \ + -f "$fixture_dir/fake-compose-config.jq" diff --git a/ci/testdata/disaster-recovery/fake-install.sh b/ci/testdata/disaster-recovery/fake-install.sh new file mode 100644 index 0000000..46c7700 --- /dev/null +++ b/ci/testdata/disaster-recovery/fake-install.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +args=() +while (($# > 0)); do + case "$1" in + -o | -g) + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac +done +exec /usr/bin/install "${args[@]}" diff --git a/ci/testdata/disaster-recovery/fake-stat.sh b/ci/testdata/disaster-recovery/fake-stat.sh new file mode 100644 index 0000000..17c8e33 --- /dev/null +++ b/ci/testdata/disaster-recovery/fake-stat.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail + +output="$(/usr/bin/stat "$@")" +if [[ "$*" == *"%u:"* ]]; then + printf '0:%s\n' "${output#*:}" +else + printf '%s\n' "$output" +fi diff --git a/ci/testdata/disaster-recovery/good-backup-receipt.jq b/ci/testdata/disaster-recovery/good-backup-receipt.jq new file mode 100644 index 0000000..20e0d65 --- /dev/null +++ b/ci/testdata/disaster-recovery/good-backup-receipt.jq @@ -0,0 +1,12 @@ +{ + schema_version: 1, + kind: "cloud-compose.offhost-backup-receipt", + operation_id: $operation_id, + completed_at: "2026-08-07T12:00:00Z", + manifest_sha256: $manifest_sha256, + encrypted: true, + off_host: true, + status: "succeeded", + remote_id: "contract/backup-1", + coverage: {database: true, application_files: true, volume_topology: true} +} diff --git a/ci/testdata/disaster-recovery/good-driver.sh b/ci/testdata/disaster-recovery/good-driver.sh new file mode 100644 index 0000000..16a684d --- /dev/null +++ b/ci/testdata/disaster-recovery/good-driver.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fixture_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +[[ -z "${SHOULD_NOT_REACH_DRIVER+x}" ]] || exit 90 +printf '%s\n' "$1" >>"${0}.calls" +operation="$1" +shift +declare -A args=() +while (($# > 0)); do + args["$1"]="$2" + shift 2 +done +case "$operation" in + backup) + jq -cn \ + --arg operation_id "${args[--operation-id]}" \ + --arg manifest_sha256 "${args[--manifest-sha256]}" \ + -f "$fixture_dir/good-backup-receipt.jq" >"${args[--receipt]}" + ;; + restore-test) + jq -cn \ + --arg test_id "${args[--test-id]}" \ + --arg manifest_sha256 "${args[--source-manifest-sha256]}" \ + --arg receipt_sha256 "${args[--source-receipt-sha256]}" \ + -f "$fixture_dir/good-restore-proof.jq" >"${args[--proof]}" + ;; + *) exit 64 ;; +esac diff --git a/ci/testdata/disaster-recovery/good-restore-proof.jq b/ci/testdata/disaster-recovery/good-restore-proof.jq new file mode 100644 index 0000000..512892a --- /dev/null +++ b/ci/testdata/disaster-recovery/good-restore-proof.jq @@ -0,0 +1,15 @@ +{ + schema_version: 1, + kind: "cloud-compose.restore-test-proof", + test_id: $test_id, + completed_at: "2026-08-07T13:00:00Z", + source_manifest_sha256: $manifest_sha256, + source_receipt_sha256: $receipt_sha256, + source_encrypted: true, + status: "succeeded", + recovery_id: "contract/recovery-1", + disposable_recovery: true, + recovery_destroyed: true, + integrity_verified: true, + coverage: {database: true, application_files: true, volume_topology: true} +} diff --git a/ci/testdata/disaster-recovery/incomplete-backup-receipt.jq b/ci/testdata/disaster-recovery/incomplete-backup-receipt.jq new file mode 100644 index 0000000..114cf7e --- /dev/null +++ b/ci/testdata/disaster-recovery/incomplete-backup-receipt.jq @@ -0,0 +1,12 @@ +{ + schema_version: 1, + kind: "cloud-compose.offhost-backup-receipt", + operation_id: $operation_id, + completed_at: "2026-08-07T12:00:00Z", + manifest_sha256: $manifest_sha256, + encrypted: true, + off_host: true, + status: "succeeded", + remote_id: "contract/incomplete", + coverage: {database: true, application_files: true, volume_topology: false} +} diff --git a/ci/testdata/disaster-recovery/incomplete-driver.sh b/ci/testdata/disaster-recovery/incomplete-driver.sh new file mode 100644 index 0000000..4b5ed07 --- /dev/null +++ b/ci/testdata/disaster-recovery/incomplete-driver.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fixture_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +operation="$1" +shift +declare -A args=() +while (($# > 0)); do + args["$1"]="$2" + shift 2 +done +[[ "$operation" == "backup" ]] +jq -cn \ + --arg operation_id "${args[--operation-id]}" \ + --arg manifest_sha256 "${args[--manifest-sha256]}" \ + -f "$fixture_dir/incomplete-backup-receipt.jq" >"${args[--receipt]}" diff --git a/ci/testdata/disaster-recovery/profile.sh b/ci/testdata/disaster-recovery/profile.sh new file mode 100644 index 0000000..2c901e6 --- /dev/null +++ b/ci/testdata/disaster-recovery/profile.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export PATH="${TEST_BIN:?}:/usr/bin:/bin" + +acquire_cloud_compose_lifecycle_lock() { + printf '%s\n' "$1" >>"${LOCK_LOG:?}" +} diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 52c1791..42eef98 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -156,6 +156,10 @@ with `systemctl status cloud-compose-bootstrap cloud-compose`. Bootstrap output uses a fixed `info` priority and has no unit-specific Fluent Bit input, so raw bootstrap output is not forwarded to Cloud Logging; systemd's own service failures remain available to the existing warning-level collector. +Root systemd jobs enter through `/usr/local/libexec/cloud-compose`, validate +the root-owned home scripts and control inputs, and only then execute their +allowlisted `/home/cloud-compose` program. Application services retain their +unprivileged execution model. ## Sitectl @@ -176,6 +180,10 @@ package list before validation. Terraform, Ansible, and Salt serialize their resolved map as `SITECTL_PACKAGE_VERSIONS` JSON and the privileged installer validates it again before downloading anything. Per-project package versions are not supported because projects on one host share the same binaries. +`/home/cloud-compose/bin` is reserved for generated `sitectl` symlinks. During +an upgrade from the former application-owned directory, the installer closes +the directory to root and rejects any other inherited command or target rather +than carrying an untrusted PATH entry forward. `sitectl_verify_args` remains a real argument list. The host stores it as JSON and appends each value through an argv-aware wrapper when a lifecycle command @@ -851,10 +859,10 @@ deploy the same `5058610fddc7267ace92d65a5c49713dce570ac3`; an early exact checkout bridges the legacy runtime's branch-only clone behavior without following a moving branch. Its `gcp.cloud_init.initcmd` disables both generations of the -internal-service timer after cloud-init writes the units but before -`/home/cloud-compose/run.sh` starts the potentially long bootstrap, so the -disposable VM cannot suspend itself. The runner checks the units again after -each boot. +internal-service timer after cloud-init writes the units but before the +root-owned `/usr/local/libexec/cloud-compose/run-bootstrap.sh` entrypoint +starts the potentially long bootstrap, so the disposable VM cannot suspend +itself. The runner checks the units again after each boot. Only the ephemeral runner key is authorized, and SSH is limited to that runner's public IPv4 `/32`. diff --git a/internal/contracttest/cloud_smoke_cleanup_test.go b/internal/contracttest/cloud_smoke_cleanup_test.go index 2c2ec3b..451e643 100644 --- a/internal/contracttest/cloud_smoke_cleanup_test.go +++ b/internal/contracttest/cloud_smoke_cleanup_test.go @@ -413,6 +413,7 @@ func TestNonGCPSmokeRunExitCleanupLifecycle(t *testing.T) { command.Env = overriddenEnvironment(map[string]string{ "CLOUD_COMPOSE_CI_BIN": filepath.Join(binDirectory, "cloud-compose-ci"), "CLOUD_COMPOSE_SMOKE_AUTO_APPROVE": "true", + "CLOUD_COMPOSE_SMOKE_BOOT_TIMEOUT": "5", "CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT": "10", "CLOUD_COMPOSE_SMOKE_KEEP": "false", "CLOUD_COMPOSE_SMOKE_RUN_ID": "123456789", @@ -466,6 +467,13 @@ func TestNonGCPSmokeRunExitCleanupLifecycle(t *testing.T) { func writeCloudSmokeLifecycleFakes(t testing.TB, directory string) { t.Helper() + sshFixture, err := os.ReadFile(filepath.Join( + repositoryRoot(t), + "internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh", + )) + if err != nil { + t.Fatalf("read lifecycle SSH fixture: %v", err) + } executables := map[string]string{ "cloud-compose-ci": `#!/usr/bin/env bash set -euo pipefail @@ -534,13 +542,7 @@ printf 'ssh-ed25519 fake-public-key cloud-compose-smoke\n' >"${path}.pub" set -euo pipefail printf '127.0.0.1 ssh-ed25519 fake-host-key\n' `, - "ssh": `#!/usr/bin/env bash -set -euo pipefail -case "$*" in - *cloud-compose-bootstrap-complete*) printf 'complete\n' ;; - *cloud-init\ status*) printf 'cloud-init not installed\n' ;; -esac -`, + "ssh": string(sshFixture), "sitectl": `#!/usr/bin/env bash set -euo pipefail exit 0 diff --git a/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh b/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh new file mode 100644 index 0000000..be68e10 --- /dev/null +++ b/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +case "$*" in + *cloud-compose-diagnostics.sh\ state*) printf 'complete\n' ;; + *cloud-compose-bootstrap-complete*) printf 'complete\n' ;; + *cloud-init\ status*) printf 'cloud-init not installed\n' ;; +esac diff --git a/modules/gcp/variables.tf b/modules/gcp/variables.tf index e5b9643..f843c46 100644 --- a/modules/gcp/variables.tf +++ b/modules/gcp/variables.tf @@ -666,7 +666,7 @@ variable "runcmd" { variable "initcmd" { type = list(string) default = [] - description = "Commands to run before /home/cloud-compose/run.sh" + description = "Commands to run before the root-owned Cloud Compose bootstrap entrypoint" } variable "artifact_registry_repository" { diff --git a/modules/linux-vm-runtime/templates/cloud-init.yml b/modules/linux-vm-runtime/templates/cloud-init.yml index 9e34e12..7a46576 100644 --- a/modules/linux-vm-runtime/templates/cloud-init.yml +++ b/modules/linux-vm-runtime/templates/cloud-init.yml @@ -115,5 +115,5 @@ runcmd: %{ if ROLLOUT_RUNCMD != "" ~} ${ROLLOUT_RUNCMD} %{ endif ~} - rm -f /home/cloud-compose/.cloud-compose-bootstrap-complete - bash /home/cloud-compose/start-cloud-compose-bootstrap.sh + rm -f /var/lib/cloud-compose/bootstrap-complete + bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh diff --git a/rootfs/etc/systemd/system/cloud-compose-bootstrap.service b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service index 48db515..32c72ef 100644 --- a/rootfs/etc/systemd/system/cloud-compose-bootstrap.service +++ b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service @@ -3,13 +3,13 @@ Description=Converge Cloud Compose application bootstrap Wants=network-online.target After=network-online.target RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes -ConditionPathExists=!/home/cloud-compose/.cloud-compose-bootstrap-complete StartLimitIntervalSec=0 [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/bin/bash /home/cloud-compose/run-bootstrap.sh +ExecCondition=/bin/bash /usr/local/libexec/cloud-compose/bootstrap-required.sh +ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-bootstrap.sh Restart=on-failure RestartSec=30s TimeoutStartSec=2h diff --git a/rootfs/etc/systemd/system/cloud-compose-docker-prune.service b/rootfs/etc/systemd/system/cloud-compose-docker-prune.service index 03f8d29..2b81c9c 100644 --- a/rootfs/etc/systemd/system/cloud-compose-docker-prune.service +++ b/rootfs/etc/systemd/system/cloud-compose-docker-prune.service @@ -8,5 +8,5 @@ After=docker.service Type=oneshot User=root Group=root -ExecStart=/bin/bash /home/cloud-compose/docker-prune.sh +ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh docker-prune.sh TimeoutStartSec=30min diff --git a/rootfs/etc/systemd/system/cloud-compose-key-rotation.service b/rootfs/etc/systemd/system/cloud-compose-key-rotation.service index b9b917d..cac15e9 100644 --- a/rootfs/etc/systemd/system/cloud-compose-key-rotation.service +++ b/rootfs/etc/systemd/system/cloud-compose-key-rotation.service @@ -6,7 +6,7 @@ RequiresMountsFor=/mnt/disks/data [Service] Type=oneshot -ExecStart=/bin/bash /home/cloud-compose/rotate-keys-daily.sh +ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh rotate-keys-daily.sh User=root Group=root TimeoutStartSec=1h diff --git a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service index fb04f77..ec04bd7 100644 --- a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service +++ b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service @@ -6,7 +6,7 @@ Before=docker.service [Service] Type=oneshot -ExecStart=/bin/bash /home/cloud-compose/configure-metadata-firewall.sh pre-docker +ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh configure-metadata-firewall.sh pre-docker RemainAfterExit=yes TimeoutStartSec=2min diff --git a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service index 40dec57..bb810f5 100644 --- a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service +++ b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service @@ -6,7 +6,7 @@ PartOf=docker.service [Service] Type=oneshot -ExecStart=/bin/bash /home/cloud-compose/configure-metadata-firewall.sh +ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh configure-metadata-firewall.sh RemainAfterExit=yes TimeoutStartSec=2min diff --git a/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service b/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service index 9ae3aa2..4a1f07f 100644 --- a/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service +++ b/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service @@ -10,7 +10,7 @@ Type=oneshot User=root Group=root UMask=0077 -ExecStart=/bin/bash /home/cloud-compose/offhost-backup.sh +ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh offhost-backup.sh TimeoutStartSec=24h StandardOutput=journal StandardError=journal diff --git a/rootfs/etc/systemd/system/cloud-compose-overlay.service b/rootfs/etc/systemd/system/cloud-compose-overlay.service index c280d3f..7c5b261 100644 --- a/rootfs/etc/systemd/system/cloud-compose-overlay.service +++ b/rootfs/etc/systemd/system/cloud-compose-overlay.service @@ -8,7 +8,7 @@ Type=oneshot User=root Group=root EnvironmentFile=-/home/cloud-compose/.env -ExecStart=/bin/bash /home/cloud-compose/mount-overlays.sh +ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh mount-overlays.sh RemainAfterExit=yes [Install] diff --git a/rootfs/etc/systemd/system/cloud-compose-restore-test.service b/rootfs/etc/systemd/system/cloud-compose-restore-test.service index 98dd9b1..64f9bd6 100644 --- a/rootfs/etc/systemd/system/cloud-compose-restore-test.service +++ b/rootfs/etc/systemd/system/cloud-compose-restore-test.service @@ -9,7 +9,7 @@ Type=oneshot User=root Group=root UMask=0077 -ExecStart=/bin/bash /home/cloud-compose/restore-test.sh +ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh restore-test.sh TimeoutStartSec=24h StandardOutput=journal StandardError=journal diff --git a/rootfs/etc/systemd/system/cloud-compose-vault-agent.service b/rootfs/etc/systemd/system/cloud-compose-vault-agent.service index d3d1f63..51d2cf0 100644 --- a/rootfs/etc/systemd/system/cloud-compose-vault-agent.service +++ b/rootfs/etc/systemd/system/cloud-compose-vault-agent.service @@ -9,10 +9,10 @@ RequiresMountsFor=/mnt/disks/data User=root Group=root EnvironmentFile=-/etc/default/vault-agent -ExecStartPre=/bin/bash /home/cloud-compose/vault-agent-readiness.sh prepare +ExecStartPre=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh prepare ExecStart=/usr/local/bin/vault agent -config=/etc/vault-agent.d/cloud-compose.hcl -ExecStartPost=/bin/bash /home/cloud-compose/vault-agent-readiness.sh wait -ExecStopPost=/bin/bash /home/cloud-compose/vault-agent-readiness.sh clear +ExecStartPost=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh wait +ExecStopPost=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh clear Restart=on-failure RestartSec=10s TimeoutStartSec=2min diff --git a/rootfs/etc/systemd/system/libops-managed-runtime.service b/rootfs/etc/systemd/system/libops-managed-runtime.service index 5ddf8ec..2644d67 100644 --- a/rootfs/etc/systemd/system/libops-managed-runtime.service +++ b/rootfs/etc/systemd/system/libops-managed-runtime.service @@ -8,7 +8,7 @@ RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volu Type=oneshot User=root Group=root -ExecStart=/bin/bash /home/cloud-compose/libops-managed-runtime.sh update +ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh libops-managed-runtime.sh update TimeoutStartSec=1h StandardOutput=journal StandardError=journal diff --git a/rootfs/etc/tmpfiles.d/cloud-compose.conf b/rootfs/etc/tmpfiles.d/cloud-compose.conf index 596c446..78617d7 100644 --- a/rootfs/etc/tmpfiles.d/cloud-compose.conf +++ b/rootfs/etc/tmpfiles.d/cloud-compose.conf @@ -1,3 +1,5 @@ +d /var/lib/cloud-compose 0755 root root - +d /home/cloud-compose 0755 root root - d /run/lock/cloud-compose 0750 root cloud-compose - f /run/lock/cloud-compose/lifecycle.lock 0660 root cloud-compose - d /mnt/disks/data/.cloud-compose-disaster-recovery 0700 root root - diff --git a/rootfs/home/cloud-compose/assert-app-initialized.sh b/rootfs/home/cloud-compose/assert-app-initialized.sh index 6434942..276e4aa 100644 --- a/rootfs/home/cloud-compose/assert-app-initialized.sh +++ b/rootfs/home/cloud-compose/assert-app-initialized.sh @@ -5,7 +5,7 @@ set -euo pipefail # shellcheck disable=SC1090 source "${CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH:-/home/cloud-compose/bootstrap-helpers.sh}" -durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/home/cloud-compose/.cloud-compose-bootstrap-complete}" +durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/var/lib/cloud-compose/bootstrap-complete}" boot_marker="${CLOUD_COMPOSE_APP_INIT_MARKER:-/run/cloud-compose-app-init-complete}" if cloud_compose_marker_exists "$durable_marker" || diff --git a/rootfs/home/cloud-compose/bootstrap-helpers.sh b/rootfs/home/cloud-compose/bootstrap-helpers.sh index 4a4c774..43aa5bf 100644 --- a/rootfs/home/cloud-compose/bootstrap-helpers.sh +++ b/rootfs/home/cloud-compose/bootstrap-helpers.sh @@ -1,9 +1,17 @@ #!/usr/bin/env bash cloud_compose_marker_exists() { - local marker="$1" - - [[ -f "$marker" && ! -L "$marker" ]] + local marker="$1" marker_size payload + + [[ -f "$marker" && ! -L "$marker" ]] || return 1 + if [[ "$marker" == "/var/lib/cloud-compose/bootstrap-complete" ]]; then + [[ "$(stat -c '%u:%g:%a:%F' -- "$(dirname -- "$marker")")" == "0:0:755:directory" && + "$(stat -c '%u:%g:%a:%h:%F' -- "$marker")" == "0:0:644:1:regular file" ]] || return 1 + marker_size="$(stat -c '%s' -- "$marker")" || return 1 + [[ "$marker_size" == "6" ]] || return 1 + IFS= read -r payload <"$marker" || return 1 + [[ "$payload" == "ready" ]] + fi } cloud_compose_should_run_app_init() { @@ -23,12 +31,25 @@ cloud_compose_publish_marker() ( echo "Unsafe Cloud Compose marker directory: $marker_dir" >&2 return 1 fi + if [[ "$marker" == "/var/lib/cloud-compose/bootstrap-complete" && + ( EUID -ne 0 || + "$(stat -c '%u:%g:%a:%F' -- "$marker_dir")" != "0:0:755:directory" ) ]]; then + echo "Durable Cloud Compose readiness requires a root-owned state directory" >&2 + return 1 + fi umask 022 tmp_marker="$(mktemp "${marker}.tmp.XXXXXXXXXX")" || return 1 if ! printf 'ready\n' >"$tmp_marker" || - ! chmod 0644 "$tmp_marker" || - ! mv -fT -- "$tmp_marker" "$marker"; then + ! chmod 0644 "$tmp_marker"; then + rm -f -- "$tmp_marker" + return 1 + fi + if ((EUID == 0)) && ! chown 0:0 "$tmp_marker"; then + rm -f -- "$tmp_marker" + return 1 + fi + if ! mv -fT -- "$tmp_marker" "$marker"; then rm -f -- "$tmp_marker" return 1 fi diff --git a/rootfs/home/cloud-compose/disaster-recovery-lib.sh b/rootfs/home/cloud-compose/disaster-recovery-lib.sh index f976e08..f797fde 100644 --- a/rootfs/home/cloud-compose/disaster-recovery-lib.sh +++ b/rootfs/home/cloud-compose/disaster-recovery-lib.sh @@ -5,6 +5,7 @@ CLOUD_COMPOSE_DR_STATE_ROOT="${CLOUD_COMPOSE_DR_STATE_ROOT:-/mnt/disks/data/.cloud-compose-disaster-recovery}" CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="${CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER:-/usr/local/libexec/cloud-compose/offhost-backup-driver}" +CLOUD_COMPOSE_JQ_PROGRAM_DIR="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/usr/local/share/cloud-compose/jq}" cloud_compose_dr_is_required() { case "${CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED:-false}" in @@ -132,29 +133,14 @@ cloud_compose_dr_validate_backup_receipt() { cloud_compose_dr_validate_json_file "$path" "Off-host backup receipt" || return 1 jq -e \ --arg operation_id "$operation_id" \ - --arg manifest_sha256 "$manifest_sha256" ' - type == "object" and length == 10 and - .schema_version == 1 and - .kind == "cloud-compose.offhost-backup-receipt" and - .operation_id == $operation_id and - .manifest_sha256 == $manifest_sha256 and - .status == "succeeded" and - .encrypted == true and - .off_host == true and - (.completed_at | type == "string" and length == 20 and - (explode | all(.[]; . >= 32 and . != 127))) and - (.remote_id | type == "string" and length >= 1 and length <= 512 and - (explode | all(.[]; . >= 32 and . != 127))) and - (.coverage | type == "object" and length == 3 and - .database == true and - .application_files == true and - .volume_topology == true) - ' "$path" >/dev/null || { + --arg manifest_sha256 "$manifest_sha256" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-validate-backup-receipt.jq" \ + "$path" >/dev/null || { echo "Off-host backup driver returned an invalid or incomplete coverage receipt" >&2 return 1 } - completed_at="$(jq -er '.completed_at' "$path")" || return 1 - remote_id="$(jq -er '.remote_id' "$path")" || return 1 + completed_at="$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-backup-completed-at.jq" "$path")" || return 1 + remote_id="$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-backup-remote-id.jq" "$path")" || return 1 cloud_compose_dr_validate_utc_timestamp "$completed_at" "Off-host backup completion time" || return 1 cloud_compose_dr_validate_remote_id "$remote_id" "Off-host backup remote id" || return 1 } @@ -166,32 +152,14 @@ cloud_compose_dr_validate_restore_proof() { jq -e \ --arg test_id "$test_id" \ --arg manifest_sha256 "$manifest_sha256" \ - --arg receipt_sha256 "$receipt_sha256" ' - type == "object" and length == 13 and - .schema_version == 1 and - .kind == "cloud-compose.restore-test-proof" and - .test_id == $test_id and - .source_manifest_sha256 == $manifest_sha256 and - .source_receipt_sha256 == $receipt_sha256 and - .status == "succeeded" and - .disposable_recovery == true and - .recovery_destroyed == true and - .integrity_verified == true and - (.completed_at | type == "string" and length == 20 and - (explode | all(.[]; . >= 32 and . != 127))) and - (.recovery_id | type == "string" and length >= 1 and length <= 512 and - (explode | all(.[]; . >= 32 and . != 127))) and - (.coverage | type == "object" and length == 3 and - .database == true and - .application_files == true and - .volume_topology == true) and - (.source_encrypted == true) - ' "$path" >/dev/null || { + --arg receipt_sha256 "$receipt_sha256" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-validate-restore-proof.jq" \ + "$path" >/dev/null || { echo "Off-host backup driver returned an invalid restore-test proof" >&2 return 1 } - completed_at="$(jq -er '.completed_at' "$path")" || return 1 - recovery_id="$(jq -er '.recovery_id' "$path")" || return 1 + completed_at="$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-restore-completed-at.jq" "$path")" || return 1 + recovery_id="$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-restore-recovery-id.jq" "$path")" || return 1 cloud_compose_dr_validate_utc_timestamp "$completed_at" "Restore-test completion time" || return 1 cloud_compose_dr_validate_remote_id "$recovery_id" "Restore-test recovery id" || return 1 } diff --git a/rootfs/home/cloud-compose/host-init.sh b/rootfs/home/cloud-compose/host-init.sh index 0c00eb4..673a93d 100644 --- a/rootfs/home/cloud-compose/host-init.sh +++ b/rootfs/home/cloud-compose/host-init.sh @@ -42,7 +42,6 @@ chown root:root /home/cloud-compose chmod 0755 /home/cloud-compose for mutable_dir in \ /home/cloud-compose/apps \ - /home/cloud-compose/bin \ /home/cloud-compose/state \ /home/cloud-compose/.sitectl \ /home/cloud-compose/.cache \ @@ -50,15 +49,32 @@ for mutable_dir in \ /home/cloud-compose/.local; do install -d -m 0750 -o cloud-compose -g cloud-compose "$mutable_dir" done +if [[ -L /home/cloud-compose/bin || ! -d /home/cloud-compose/bin || + "$(stat -c '%u:%g:%a:%F' -- /home/cloud-compose/bin)" != "0:0:755:directory" ]]; then + echo "Managed command directory was not secured by the runtime installer" >&2 + exit 1 +fi find /home/cloud-compose -maxdepth 1 -type f -name '*.sh' \ -exec chown root:root {} + \ -exec chmod 0755 {} + for dispatcher in init up down rollout; do - if [ -f "/home/cloud-compose/$dispatcher" ]; then - chown root:root "/home/cloud-compose/$dispatcher" - chmod 0755 "/home/cloud-compose/$dispatcher" + dispatcher_path="/home/cloud-compose/$dispatcher" + dispatcher_metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$dispatcher_path")" || { + echo "Unable to inspect Cloud Compose lifecycle dispatcher: $dispatcher_path" >&2 + exit 1 + } + IFS=: read -r dispatcher_uid _ dispatcher_mode dispatcher_links dispatcher_kind \ + <<<"$dispatcher_metadata" + if [[ -L "$dispatcher_path" || ! -f "$dispatcher_path" || + "$dispatcher_uid" != "0" || "$dispatcher_links" != "1" || + "$dispatcher_kind" != "regular file" || ! "$dispatcher_mode" =~ ^[0-7]{3,4}$ || + $((8#$dispatcher_mode & 0022)) -ne 0 ]]; then + echo "Unsafe Cloud Compose lifecycle dispatcher: $dispatcher_path" >&2 + exit 1 fi + chown root:root "$dispatcher_path" + chmod 0755 "$dispatcher_path" done for runtime_input in .env compose-projects.json application-env.json; do if [ -f "/home/cloud-compose/$runtime_input" ]; then diff --git a/rootfs/home/cloud-compose/libops-managed-runtime.sh b/rootfs/home/cloud-compose/libops-managed-runtime.sh index 94bc722..41ea817 100644 --- a/rootfs/home/cloud-compose/libops-managed-runtime.sh +++ b/rootfs/home/cloud-compose/libops-managed-runtime.sh @@ -27,20 +27,133 @@ enabled() { } mkdirs() { + local expected_uid expected_gid path mode allow_owner_migration spec + local -a directory_specs + + expected_uid="$EUID" + expected_gid="$(id -g)" + if [[ "$STATE_DIR" == "/mnt/disks/data/libops-managed" || + "$PUBLISHED_BIN_DIR" == "/home/cloud-compose/bin" ]]; then + if ((EUID != 0)); then + log "production managed runtime directories require a root updater" + return 1 + fi + expected_uid=0 + expected_gid=0 + fi + # The managed binary is published through /home/cloud-compose/bin and must # remain traversable after a root-owned bootstrap drops to cloud-compose. - # Converge existing directories as well as new ones so a previously - # restrictive service umask cannot leave the published symlink unusable. - install -d -m 0755 "$STATE_DIR" "$BIN_DIR" - install -d -m 0700 "$TMP_DIR" "$PACKAGE_STATE_DIR" "$ARTIFACT_STATE_DIR" - mkdir -p "$PUBLISHED_BIN_DIR" + # Refuse redirected, non-directory, non-owner-controlled, or writable + # state before creating package staging files beneath the shared data mount. + directory_specs=( + "$STATE_DIR:0755:false" + "$BIN_DIR:0755:false" + "$TMP_DIR:0700:false" + "$PACKAGE_STATE_DIR:0700:false" + "$ARTIFACT_STATE_DIR:0700:false" + "$PUBLISHED_BIN_DIR:0755:true" + ) + for spec in "${directory_specs[@]}"; do + IFS=: read -r path mode allow_owner_migration <<<"$spec" + prepare_managed_runtime_directory \ + "$path" "$mode" "$expected_uid" "$expected_gid" "$allow_owner_migration" || return 1 + done +} + +prepare_managed_runtime_directory() { + local path="$1" mode="$2" expected_uid="$3" expected_gid="$4" + local allow_owner_migration="$5" metadata owner_uid group_gid actual_mode kind resolved desired_mode + local created=false + + if [[ -L "$path" || ( -e "$path" && ! -d "$path" ) ]]; then + log "managed runtime path is not a real directory: ${path}" + return 1 + fi + if [[ ! -e "$path" && ! -L "$path" ]]; then + # mkdir is the creation boundary: if an unprivileged process wins the + # name between inspection and creation, fail rather than adopting its + # pre-populated directory with install -d. + if ! mkdir -m "$mode" -- "$path"; then + log "managed runtime directory appeared during creation: ${path}" + return 1 + fi + created=true + fi + if [[ "$created" != "true" ]]; then + if [[ -L "$path" || ! -d "$path" ]]; then + log "managed runtime path changed during validation: ${path}" + return 1 + fi + metadata="$(stat -c '%u:%g:%a:%F' -- "$path")" || return 1 + IFS=: read -r owner_uid group_gid actual_mode kind <<<"$metadata" + if [[ "$kind" != "directory" || ! "$actual_mode" =~ ^[0-7]{3,4}$ || + $((8#$actual_mode & 0022)) -ne 0 ]]; then + log "managed runtime directory is writable by another account: ${path}" + return 1 + fi + if [[ ( "$allow_owner_migration" != "true" || EUID -ne 0 ) && + ( "$owner_uid" != "$expected_uid" || "$group_gid" != "$expected_gid" ) ]]; then + log "managed runtime directory is not owned by the updater: ${path}" + return 1 + fi + if [[ "$allow_owner_migration" == "true" ]]; then + # Close the legacy application-owned PATH directory before walking + # its entries. The bootstrap libexec boundary has already made its + # parent root-owned, so an old owner cannot race validation. + if ((EUID == 0)); then + install -d -m "$mode" -o "$expected_uid" -g "$expected_gid" -- "$path" || return 1 + else + chmod "$mode" -- "$path" || return 1 + fi + validate_published_bin_directory "$path" || return 1 + fi + fi + + if ((EUID == 0)); then + install -d -m "$mode" -o "$expected_uid" -g "$expected_gid" -- "$path" || return 1 + else + chmod "$mode" -- "$path" || return 1 + fi + resolved="$(readlink -f -- "$path")" || return 1 + desired_mode="$(printf '%o' "$((8#$mode))")" + metadata="$(stat -c '%u:%g:%a:%F' -- "$path")" || return 1 + if [[ "$resolved" != "$path" || "$metadata" != "${expected_uid}:${expected_gid}:${desired_mode}:directory" || + -L "$path" ]]; then + log "managed runtime directory did not converge safely: ${path}" + return 1 + fi +} + +validate_published_bin_directory() { + local path="$1" entry name target + local -a entries + + # /home/cloud-compose/bin was application-owned on older hosts. Preserve + # only the generated sitectl links whose targets remain under the validated + # root-owned package directory; reject every other inherited PATH entry. + shopt -s nullglob dotglob + entries=("$path"/*) + shopt -u nullglob dotglob + for entry in "${entries[@]}"; do + name="${entry##*/}" + if [[ ! "$name" =~ ^sitectl(-[a-z0-9]+)*$ || ! -L "$entry" ]]; then + log "published command directory contains an unmanaged entry: ${entry}" + return 1 + fi + target="$(readlink -- "$entry")" || return 1 + if [[ "$target" != "${BIN_DIR}/${name}" ]]; then + log "published command has an unsafe target: ${entry}" + return 1 + fi + done } with_lock() { local action="$1" shift - mkdirs + mkdirs || return 1 if command -v flock >/dev/null 2>&1; then exec 9>"${STATE_DIR}/runtime.lock" if ! flock -n 9; then @@ -406,7 +519,7 @@ install_sitectl_packages() { local -A desired_packages=() stale_seen=() validate_sitectl_configuration - mkdirs + mkdirs || return 1 mapfile -t packages < <(sitectl_package_list) for package in "${packages[@]}"; do @@ -871,6 +984,11 @@ run_update() { main() { local command="${1:-update}" + if ((EUID != 0)); then + log "managed runtime updates must run as root" + return 1 + fi + case "$command" in install-tools) with_lock run_install_tools diff --git a/rootfs/home/cloud-compose/offhost-backup.sh b/rootfs/home/cloud-compose/offhost-backup.sh index 704e622..f24f1ea 100644 --- a/rootfs/home/cloud-compose/offhost-backup.sh +++ b/rootfs/home/cloud-compose/offhost-backup.sh @@ -6,10 +6,12 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" compose_apps_path="${CLOUD_COMPOSE_COMPOSE_APPS_PATH:-$script_dir/compose-apps.sh}" dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}" +jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/usr/local/share/cloud-compose/jq}" # shellcheck disable=SC1090 source "$profile_path" # shellcheck disable=SC1090 source "$compose_apps_path" +CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir" # shellcheck disable=SC1090 source "$dr_library_path" @@ -111,32 +113,17 @@ for app in "${apps[@]}"; do umask 077 docker compose config --format json >"$compose_config" ) - if ! jq -e ' - type == "object" and - (.services | type == "object" and length > 0) and - all(.services | to_entries[]; - ((.value.volumes // []) | type == "array") and - all((.value.volumes // [])[]; - type == "object" and - (.type | type == "string") and - (.type == "volume" or .type == "bind" or .type == "tmpfs") and - ((.source // "") | type == "string") and - ((.target // "") | type == "string" and length > 0))) and - ((.volumes // {}) | type == "object") - ' "$compose_config" >/dev/null; then + if ! jq -e \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-validate-compose-config.jq" \ + "$compose_config" >/dev/null; then echo "Docker Compose returned unsafe or unsupported volume topology for ${app}" >&2 exit 1 fi - if ! jq -e --arg data_root "$data_root" --arg volumes_root "$volumes_root" ' - all(.services[].volumes[]?; - .type != "bind" or - (.source | type == "string" and - (. == $data_root or startswith($data_root + "/") or - . == $volumes_root or startswith($volumes_root + "/")) and - (explode | all(.[]; . >= 32 and . != 127)) and - (contains("//") | not) and - length > 0)) - ' "$compose_config" >/dev/null; then + if ! jq -e \ + --arg data_root "$data_root" \ + --arg volumes_root "$volumes_root" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-validate-bind-roots.jq" \ + "$compose_config" >/dev/null; then echo "Persistent bind topology escapes managed data roots for ${app}" >&2 exit 1 fi @@ -145,42 +132,16 @@ for app in "${apps[@]}"; do echo "Persistent bind topology contains a dot segment for ${app}" >&2 exit 1 fi - done < <(jq -r '.services[].volumes[]? | select(.type == "bind") | .source' "$compose_config") + done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-bind-sources.jq" "$compose_config") jq -cS \ --arg app "$app" \ --arg project_dir "$project_dir" \ --arg dump_path "$staged_dump" \ --arg dump_sha256 "$dump_sha256" \ - --argjson dump_bytes "$dump_bytes" ' - { - name: $app, - databases: [{ - engine: "mariadb", - format: "sql.gz", - local_recovery_artifact: $dump_path, - sha256: $dump_sha256, - bytes: $dump_bytes - }], - application_files: { - roots: [$project_dir], - bind_mounts: [ - .services | to_entries[] as $service | - ($service.value.volumes // [])[] | - select(.type == "bind") | - {service: $service.key, source: .source, target: .target, read_only: (.read_only // false)} - ] | sort_by(.service, .source, .target) - }, - volume_topology: { - declared_named_volumes: ((.volumes // {}) | keys | sort), - service_mounts: [ - .services | to_entries[] as $service | - ($service.value.volumes // [])[] | - {service: $service.key, type: .type, source: (.source // ""), target: .target, read_only: (.read_only // false)} - ] | sort_by(.service, .type, .source, .target) - } - } - ' "$compose_config" >"$application_row" + --argjson dump_bytes "$dump_bytes" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-build-application-coverage.jq" \ + "$compose_config" >"$application_row" cat "$application_row" >>"$application_rows" rm -f -- "$compose_config" done @@ -190,36 +151,15 @@ jq -cS -s \ --arg operation_id "$operation_id" \ --arg backup_date "$backup_date" \ --arg provider "${CLOUD_COMPOSE_PROVIDER:-unknown}" \ - --arg instance "${CLOUD_COMPOSE_INSTANCE_NAME:-cloud-compose}" ' - { - schema_version: 1, - kind: "cloud-compose.offhost-backup-manifest", - operation_id: $operation_id, - backup_date: $backup_date, - provider: $provider, - instance: $instance, - required_coverage: ["database", "application_files", "volume_topology"], - applications: (sort_by(.name)) - } -' "$application_rows" >"$staged_manifest" + --arg instance "${CLOUD_COMPOSE_INSTANCE_NAME:-cloud-compose}" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-build-manifest.jq" \ + "$application_rows" >"$staged_manifest" chmod 0400 "$staged_manifest" -if ! jq -e --argjson app_count "${#apps[@]}" ' - .schema_version == 1 and - .kind == "cloud-compose.offhost-backup-manifest" and - (.applications | type == "array" and length == $app_count) and - all(.applications[]; - (.name | type == "string" and length >= 1 and length <= 63 and - (explode | all(.[]; . >= 32 and . != 127))) and - (.databases | length == 1) and - (.databases[0].sha256 | type == "string" and length == 64 and - (explode | all(.[]; . >= 32 and . != 127))) and - (.databases[0].bytes | type == "number" and . > 0) and - (.application_files.roots | type == "array" and length > 0) and - (.application_files.bind_mounts | type == "array") and - (.volume_topology.declared_named_volumes | type == "array") and - (.volume_topology.service_mounts | type == "array")) -' "$staged_manifest" >/dev/null; then +if ! jq -e \ + --argjson app_count "${#apps[@]}" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-validate-manifest.jq" \ + "$staged_manifest" >/dev/null; then echo "Generated off-host coverage manifest is incomplete" >&2 exit 1 fi @@ -228,7 +168,7 @@ while IFS=$'\t' read -r manifest_app manifest_sha; do echo "Generated off-host coverage manifest contains an unsafe application name or digest" >&2 exit 1 fi -done < <(jq -r '.applications[] | [.name, .databases[0].sha256] | @tsv' "$staged_manifest") +done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-manifest-app-digests.jq" "$staged_manifest") manifest_sha256="$(sha256sum "$staged_manifest" | awk '{print $1}')" staged_receipt="$staging_dir/receipt.json" diff --git a/rootfs/home/cloud-compose/restore-test.sh b/rootfs/home/cloud-compose/restore-test.sh index e3b5ac1..0ca6a0b 100644 --- a/rootfs/home/cloud-compose/restore-test.sh +++ b/rootfs/home/cloud-compose/restore-test.sh @@ -5,8 +5,10 @@ set -euo pipefail script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}" +jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/usr/local/share/cloud-compose/jq}" # shellcheck disable=SC1090 source "$profile_path" +CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir" # shellcheck disable=SC1090 source "$dr_library_path" diff --git a/rootfs/home/cloud-compose/run.sh b/rootfs/home/cloud-compose/run.sh index c4d5c11..d4c6358 100644 --- a/rootfs/home/cloud-compose/run.sh +++ b/rootfs/home/cloud-compose/run.sh @@ -32,7 +32,7 @@ runtime_enabled() { esac } -durable_bootstrap_marker="/home/cloud-compose/.cloud-compose-bootstrap-complete" +durable_bootstrap_marker="/var/lib/cloud-compose/bootstrap-complete" current_boot_app_init_marker="/run/cloud-compose-app-init-complete" fresh_filesystem_marker="${CLOUD_COMPOSE_FRESH_FILESYSTEM_MARKER:-/mnt/disks/data/.cloud-compose/fresh-filesystem}" fresh_filesystem_identity="${CLOUD_COMPOSE_FRESH_FILESYSTEM_IDENTITY:-fresh}" diff --git a/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh b/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh index 0e55092..79f2480 100644 --- a/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh +++ b/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh @@ -5,7 +5,7 @@ set -euo pipefail # shellcheck disable=SC1090 source "${CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH:-/home/cloud-compose/bootstrap-helpers.sh}" -durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/home/cloud-compose/.cloud-compose-bootstrap-complete}" +durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/var/lib/cloud-compose/bootstrap-complete}" wait_seconds="${CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS:-10800}" bootstrap_unit="cloud-compose-bootstrap.service" diff --git a/rootfs/usr/local/libexec/cloud-compose/bootstrap-required.sh b/rootfs/usr/local/libexec/cloud-compose/bootstrap-required.sh new file mode 100644 index 0000000..2d56f2a --- /dev/null +++ b/rootfs/usr/local/libexec/cloud-compose/bootstrap-required.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -uo pipefail + +# shellcheck disable=SC1091 +if ! source /usr/local/libexec/cloud-compose/bootstrap-security.sh; then + echo "Cloud Compose bootstrap security helper could not be loaded" >&2 + exit 255 +fi + +if ! cloud_compose_bootstrap_require_root; then + exit 255 +fi +if cloud_compose_bootstrap_marker_ready; then + # ExecCondition exit 1 skips an already-complete oneshot without marking it + # failed. Missing or invalid evidence returns zero below and must converge. + exit 1 +fi +exit 0 diff --git a/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh b/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh new file mode 100644 index 0000000..27e77c2 --- /dev/null +++ b/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +readonly CLOUD_COMPOSE_BOOTSTRAP_MARKER="/var/lib/cloud-compose/bootstrap-complete" +readonly CLOUD_COMPOSE_RUNTIME_HOME="/home/cloud-compose" + +cloud_compose_bootstrap_require_root() { + if ((EUID != 0)); then + echo "Cloud Compose bootstrap control must run as root" >&2 + return 1 + fi +} + +cloud_compose_bootstrap_marker_ready() { + local marker="${1:-$CLOUD_COMPOSE_BOOTSTRAP_MARKER}" + local marker_dir marker_dir_metadata marker_metadata marker_size payload + + marker_dir="$(dirname -- "$marker")" + [[ -d "$marker_dir" && ! -L "$marker_dir" && -f "$marker" && ! -L "$marker" ]] || return 1 + marker_dir_metadata="$(stat -c '%u:%g:%a:%F' -- "$marker_dir")" || return 1 + marker_metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$marker")" || return 1 + marker_size="$(stat -c '%s' -- "$marker")" || return 1 + [[ "$marker_dir_metadata" == "0:0:755:directory" && + "$marker_metadata" == "0:0:644:1:regular file" && + "$marker_size" == "6" ]] || return 1 + IFS= read -r payload <"$marker" || return 1 + [[ "$payload" == "ready" ]] +} + +cloud_compose_secure_runtime_home() { + local home_metadata program dispatcher metadata owner_uid group_gid mode links kind + local -a required_programs programs dispatchers control_inputs + + cloud_compose_bootstrap_require_root || return 1 + if [[ -L "$CLOUD_COMPOSE_RUNTIME_HOME" || ! -d "$CLOUD_COMPOSE_RUNTIME_HOME" ]]; then + echo "Cloud Compose runtime home is missing or redirected" >&2 + return 1 + fi + home_metadata="$(stat -c '%a:%F' -- "$CLOUD_COMPOSE_RUNTIME_HOME")" || return 1 + if [[ ! "$home_metadata" =~ ^[0-7]{3,4}:directory$ ]]; then + echo "Cloud Compose runtime home is not a real directory" >&2 + return 1 + fi + + # Close the historical user-owned parent boundary before inspecting any + # program beneath it. A replacement file retains its non-root ownership and + # is rejected below; a now-unwritable parent prevents another replacement. + chown 0:0 "$CLOUD_COMPOSE_RUNTIME_HOME" || return 1 + chmod 0755 "$CLOUD_COMPOSE_RUNTIME_HOME" || return 1 + + required_programs=( + "$CLOUD_COMPOSE_RUNTIME_HOME/run.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/profile.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/bootstrap-helpers.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/host-conf.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/host-init.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/converge-app-filesystems.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/prepare-app-sources.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/rotate-keys-daily.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/vault-agent-init.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/app-init.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/init" + "$CLOUD_COMPOSE_RUNTIME_HOME/up" + "$CLOUD_COMPOSE_RUNTIME_HOME/down" + "$CLOUD_COMPOSE_RUNTIME_HOME/rollout" + ) + for program in "${required_programs[@]}"; do + if [[ -L "$program" || ! -f "$program" ]]; then + echo "Required Cloud Compose bootstrap program is missing or redirected: $program" >&2 + return 1 + fi + done + + shopt -s nullglob + programs=("$CLOUD_COMPOSE_RUNTIME_HOME"/*.sh) + shopt -u nullglob + for program in "${programs[@]}"; do + if [[ -L "$program" || ! -f "$program" ]]; then + echo "Cloud Compose bootstrap program is not a regular file: $program" >&2 + return 1 + fi + metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$program")" || return 1 + IFS=: read -r owner_uid group_gid mode links kind <<<"$metadata" + if [[ "$owner_uid" != "0" || "$group_gid" != "0" || "$links" != "1" || "$kind" != "regular file" || + ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then + echo "Cloud Compose bootstrap program is not root-controlled: $program" >&2 + return 1 + fi + chown 0:0 "$program" || return 1 + chmod 0755 "$program" || return 1 + done + + control_inputs=( + "$CLOUD_COMPOSE_RUNTIME_HOME/.env" + "$CLOUD_COMPOSE_RUNTIME_HOME/compose-projects.json" + "$CLOUD_COMPOSE_RUNTIME_HOME/application-env.json" + "$CLOUD_COMPOSE_RUNTIME_HOME/managed-runtime-artifacts.tsv" + ) + for program in "${control_inputs[@]}"; do + if [[ -L "$program" || ! -f "$program" ]]; then + echo "Required Cloud Compose control input is missing or redirected: $program" >&2 + return 1 + fi + metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$program")" || return 1 + IFS=: read -r owner_uid group_gid mode links kind <<<"$metadata" + if [[ "$owner_uid" != "0" || "$links" != "1" || "$kind" != "regular file" || + ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then + echo "Cloud Compose control input is not root-controlled: $program" >&2 + return 1 + fi + done + + dispatchers=(init up down rollout) + for dispatcher in "${dispatchers[@]}"; do + program="$CLOUD_COMPOSE_RUNTIME_HOME/$dispatcher" + metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$program")" || return 1 + IFS=: read -r owner_uid group_gid mode links kind <<<"$metadata" + if [[ "$owner_uid" != "0" || "$links" != "1" || "$kind" != "regular file" || + ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then + echo "Cloud Compose lifecycle dispatcher is not root-controlled: $program" >&2 + return 1 + fi + done +} diff --git a/rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh b/rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh new file mode 100644 index 0000000..a9c1ac5 --- /dev/null +++ b/rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source /usr/local/libexec/cloud-compose/bootstrap-security.sh + +cloud_compose_bootstrap_require_root +if ! cloud_compose_bootstrap_marker_ready; then + echo "Cloud Compose bootstrap readiness evidence is missing or invalid" >&2 + exit 1 +fi diff --git a/rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh b/rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh new file mode 100644 index 0000000..259f1c1 --- /dev/null +++ b/rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source /usr/local/libexec/cloud-compose/bootstrap-security.sh + +cloud_compose_secure_runtime_home +exec /bin/bash /home/cloud-compose/run.sh diff --git a/rootfs/usr/local/libexec/cloud-compose/run-root-program.sh b/rootfs/usr/local/libexec/cloud-compose/run-root-program.sh new file mode 100644 index 0000000..326455e --- /dev/null +++ b/rootfs/usr/local/libexec/cloud-compose/run-root-program.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source /usr/local/libexec/cloud-compose/bootstrap-security.sh + +program="${1:-}" +if [[ -z "$program" ]]; then + echo "A Cloud Compose root program is required" >&2 + exit 2 +fi +shift + +case "$program" in + configure-metadata-firewall.sh | deploy-rollout.sh | docker-prune.sh | \ + libops-managed-runtime.sh | mount-overlays.sh | offhost-backup.sh | \ + restore-test.sh | rotate-keys-daily.sh | \ + vault-agent-readiness.sh) ;; + *) + echo "Unsupported Cloud Compose root program: $program" >&2 + exit 2 + ;; +esac + +cloud_compose_secure_runtime_home +exec /bin/bash "/home/cloud-compose/$program" "$@" diff --git a/rootfs/usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh b/rootfs/usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh new file mode 100644 index 0000000..02ce01b --- /dev/null +++ b/rootfs/usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source /usr/local/libexec/cloud-compose/bootstrap-security.sh + +cloud_compose_secure_runtime_home +exec /bin/bash /home/cloud-compose/start-cloud-compose-bootstrap.sh "$@" diff --git a/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh b/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh index df07749..7662827 100755 --- a/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh +++ b/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh @@ -7,8 +7,9 @@ set -euo pipefail readonly PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export PATH -readonly bootstrap_marker="/home/cloud-compose/.cloud-compose-bootstrap-complete" +readonly bootstrap_marker="/var/lib/cloud-compose/bootstrap-complete" readonly diagnostics_program="/usr/local/sbin/cloud-compose-diagnostics.sh" +readonly jq_program_dir="/usr/local/share/cloud-compose/jq" readonly process_pattern='[/]home/cloud-compose/run[.]sh|[/]home/cloud-compose/[h]ost-conf[.]sh|[/]home/cloud-compose/[h]ost-init[.]sh|[/]home/cloud-compose/[a]pp-init[.]sh|[/]home/cloud-compose/[i]nstall-dependencies|[a]pt-get|[r]pm-ostree|[d]ocker run|[s]itectl|[g]it clone' usage() { @@ -31,10 +32,20 @@ unit_value() { bootstrap_state() { local bootstrap_load_state bootstrap_active_state bootstrap_sub_state + local marker_dir_metadata marker_metadata marker_size marker_payload if [[ -f "$bootstrap_marker" && ! -L "$bootstrap_marker" ]]; then - echo complete - return 0 + marker_dir_metadata="$(stat -c '%u:%g:%a:%F' -- "$(dirname -- "$bootstrap_marker")" 2>/dev/null || true)" + marker_metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$bootstrap_marker" 2>/dev/null || true)" + marker_size="$(stat -c '%s' -- "$bootstrap_marker" 2>/dev/null || true)" + marker_payload="" + IFS= read -r marker_payload <"$bootstrap_marker" || true + if [[ "$marker_dir_metadata" == "0:0:755:directory" && + "$marker_metadata" == "0:0:644:1:regular file" && + "$marker_size" == "6" && "$marker_payload" == "ready" ]]; then + echo complete + return 0 + fi fi bootstrap_load_state="$(unit_value cloud-compose-bootstrap.service LoadState)" @@ -136,7 +147,8 @@ dump_compose_state() { echo "--- docker compose project state ---" if [[ ! -f "$manifest" || -L "$manifest" ]] || - ! jq -e 'type == "object"' "$manifest" >/dev/null 2>&1; then + ! jq -e -f "$jq_program_dir/diagnostics-validate-compose-projects.jq" \ + "$manifest" >/dev/null 2>&1; then echo "Compose project manifest is unavailable or invalid" return 0 fi @@ -144,8 +156,8 @@ dump_compose_state() { while IFS= read -r encoded; do [[ -n "$encoded" ]] || continue row="$(printf '%s' "$encoded" | base64 -d)" || continue - app="$(jq -er '.key | select(type == "string" and length > 0)' <<<"$row" 2>/dev/null || true)" - project_dir="$(jq -er '.value.project_dir | select(type == "string" and startswith("/mnt/disks/data/"))' <<<"$row" 2>/dev/null || true)" + app="$(jq -er -f "$jq_program_dir/diagnostics-entry-app.jq" <<<"$row" 2>/dev/null || true)" + project_dir="$(jq -er -f "$jq_program_dir/diagnostics-entry-project-dir.jq" <<<"$row" 2>/dev/null || true)" if [[ -z "$app" || -z "$project_dir" || ! -d "$project_dir" || -L "$project_dir" ]]; then echo "Skipping unavailable or unsafe Compose project: ${app:-unknown}" continue @@ -153,7 +165,7 @@ dump_compose_state() { echo "--- docker compose ps: ${app} ---" runuser -u cloud-compose -- env HOME=/home/cloud-compose \ "$docker_path" compose --project-directory "$project_dir" ps 2>&1 || true - done < <(jq -r 'to_entries[] | @base64' "$manifest") + done < <(jq -r -f "$jq_program_dir/diagnostics-project-entries.jq" "$manifest") } diagnostic_dump() { diff --git a/rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-app.jq b/rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-app.jq new file mode 100644 index 0000000..edb6cdb --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-app.jq @@ -0,0 +1 @@ +.key | select(type == "string" and length > 0) diff --git a/rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-project-dir.jq b/rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-project-dir.jq new file mode 100644 index 0000000..f0b9664 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-project-dir.jq @@ -0,0 +1 @@ +.value.project_dir | select(type == "string" and startswith("/mnt/disks/data/")) diff --git a/rootfs/usr/local/share/cloud-compose/jq/diagnostics-project-entries.jq b/rootfs/usr/local/share/cloud-compose/jq/diagnostics-project-entries.jq new file mode 100644 index 0000000..4785439 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/diagnostics-project-entries.jq @@ -0,0 +1 @@ +to_entries[] | @base64 diff --git a/rootfs/usr/local/share/cloud-compose/jq/diagnostics-validate-compose-projects.jq b/rootfs/usr/local/share/cloud-compose/jq/diagnostics-validate-compose-projects.jq new file mode 100644 index 0000000..a25876a --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/diagnostics-validate-compose-projects.jq @@ -0,0 +1 @@ +type == "object" diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-backup-completed-at.jq b/rootfs/usr/local/share/cloud-compose/jq/dr-backup-completed-at.jq new file mode 100644 index 0000000..9822734 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/dr-backup-completed-at.jq @@ -0,0 +1 @@ +.completed_at diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-backup-remote-id.jq b/rootfs/usr/local/share/cloud-compose/jq/dr-backup-remote-id.jq new file mode 100644 index 0000000..f142fb1 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/dr-backup-remote-id.jq @@ -0,0 +1 @@ +.remote_id diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-restore-completed-at.jq b/rootfs/usr/local/share/cloud-compose/jq/dr-restore-completed-at.jq new file mode 100644 index 0000000..9822734 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/dr-restore-completed-at.jq @@ -0,0 +1 @@ +.completed_at diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-restore-recovery-id.jq b/rootfs/usr/local/share/cloud-compose/jq/dr-restore-recovery-id.jq new file mode 100644 index 0000000..fcaabb5 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/dr-restore-recovery-id.jq @@ -0,0 +1 @@ +.recovery_id diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-validate-backup-receipt.jq b/rootfs/usr/local/share/cloud-compose/jq/dr-validate-backup-receipt.jq new file mode 100644 index 0000000..628e5a3 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/dr-validate-backup-receipt.jq @@ -0,0 +1,16 @@ +type == "object" and length == 10 and +.schema_version == 1 and +.kind == "cloud-compose.offhost-backup-receipt" and +.operation_id == $operation_id and +.manifest_sha256 == $manifest_sha256 and +.status == "succeeded" and +.encrypted == true and +.off_host == true and +(.completed_at | type == "string" and length == 20 and + (explode | all(.[]; . >= 32 and . != 127))) and +(.remote_id | type == "string" and length >= 1 and length <= 512 and + (explode | all(.[]; . >= 32 and . != 127))) and +(.coverage | type == "object" and length == 3 and + .database == true and + .application_files == true and + .volume_topology == true) diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-validate-restore-proof.jq b/rootfs/usr/local/share/cloud-compose/jq/dr-validate-restore-proof.jq new file mode 100644 index 0000000..461bd52 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/dr-validate-restore-proof.jq @@ -0,0 +1,19 @@ +type == "object" and length == 13 and +.schema_version == 1 and +.kind == "cloud-compose.restore-test-proof" and +.test_id == $test_id and +.source_manifest_sha256 == $manifest_sha256 and +.source_receipt_sha256 == $receipt_sha256 and +.status == "succeeded" and +.disposable_recovery == true and +.recovery_destroyed == true and +.integrity_verified == true and +(.completed_at | type == "string" and length == 20 and + (explode | all(.[]; . >= 32 and . != 127))) and +(.recovery_id | type == "string" and length >= 1 and length <= 512 and + (explode | all(.[]; . >= 32 and . != 127))) and +(.coverage | type == "object" and length == 3 and + .database == true and + .application_files == true and + .volume_topology == true) and +(.source_encrypted == true) diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-bind-sources.jq b/rootfs/usr/local/share/cloud-compose/jq/offhost-bind-sources.jq new file mode 100644 index 0000000..880c92b --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/offhost-bind-sources.jq @@ -0,0 +1 @@ +.services[].volumes[]? | select(.type == "bind") | .source diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-build-application-coverage.jq b/rootfs/usr/local/share/cloud-compose/jq/offhost-build-application-coverage.jq new file mode 100644 index 0000000..77633bd --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/offhost-build-application-coverage.jq @@ -0,0 +1,27 @@ +{ + name: $app, + databases: [{ + engine: "mariadb", + format: "sql.gz", + local_recovery_artifact: $dump_path, + sha256: $dump_sha256, + bytes: $dump_bytes + }], + application_files: { + roots: [$project_dir], + bind_mounts: [ + .services | to_entries[] as $service | + ($service.value.volumes // [])[] | + select(.type == "bind") | + {service: $service.key, source: .source, target: .target, read_only: (.read_only // false)} + ] | sort_by(.service, .source, .target) + }, + volume_topology: { + declared_named_volumes: ((.volumes // {}) | keys | sort), + service_mounts: [ + .services | to_entries[] as $service | + ($service.value.volumes // [])[] | + {service: $service.key, type: .type, source: (.source // ""), target: .target, read_only: (.read_only // false)} + ] | sort_by(.service, .type, .source, .target) + } +} diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-build-manifest.jq b/rootfs/usr/local/share/cloud-compose/jq/offhost-build-manifest.jq new file mode 100644 index 0000000..a1d4451 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/offhost-build-manifest.jq @@ -0,0 +1,10 @@ +{ + schema_version: 1, + kind: "cloud-compose.offhost-backup-manifest", + operation_id: $operation_id, + backup_date: $backup_date, + provider: $provider, + instance: $instance, + required_coverage: ["database", "application_files", "volume_topology"], + applications: (sort_by(.name)) +} diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-manifest-app-digests.jq b/rootfs/usr/local/share/cloud-compose/jq/offhost-manifest-app-digests.jq new file mode 100644 index 0000000..c3cef5b --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/offhost-manifest-app-digests.jq @@ -0,0 +1 @@ +.applications[] | [.name, .databases[0].sha256] | @tsv diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-bind-roots.jq b/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-bind-roots.jq new file mode 100644 index 0000000..3d8ac7b --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-bind-roots.jq @@ -0,0 +1,8 @@ +all(.services[].volumes[]?; + .type != "bind" or + (.source | type == "string" and + (. == $data_root or startswith($data_root + "/") or + . == $volumes_root or startswith($volumes_root + "/")) and + (explode | all(.[]; . >= 32 and . != 127)) and + (contains("//") | not) and + length > 0)) diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-compose-config.jq b/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-compose-config.jq new file mode 100644 index 0000000..12b0534 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-compose-config.jq @@ -0,0 +1,11 @@ +type == "object" and +(.services | type == "object" and length > 0) and +all(.services | to_entries[]; + ((.value.volumes // []) | type == "array") and + all((.value.volumes // [])[]; + type == "object" and + (.type | type == "string") and + (.type == "volume" or .type == "bind" or .type == "tmpfs") and + ((.source // "") | type == "string") and + ((.target // "") | type == "string" and length > 0))) and +((.volumes // {}) | type == "object") diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-manifest.jq b/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-manifest.jq new file mode 100644 index 0000000..f196ef2 --- /dev/null +++ b/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-manifest.jq @@ -0,0 +1,14 @@ +.schema_version == 1 and +.kind == "cloud-compose.offhost-backup-manifest" and +(.applications | type == "array" and length == $app_count) and +all(.applications[]; + (.name | type == "string" and length >= 1 and length <= 63 and + (explode | all(.[]; . >= 32 and . != 127))) and + (.databases | length == 1) and + (.databases[0].sha256 | type == "string" and length == 64 and + (explode | all(.[]; . >= 32 and . != 127))) and + (.databases[0].bytes | type == "number" and . > 0) and + (.application_files.roots | type == "array" and length > 0) and + (.application_files.bind_mounts | type == "array") and + (.volume_topology.declared_named_volumes | type == "array") and + (.volume_topology.service_mounts | type == "array")) diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls index 6ff0a03..bd192c6 100644 --- a/salt/cloud-compose/init.sls +++ b/salt/cloud-compose/init.sls @@ -673,8 +673,8 @@ cloud-compose-lifecycle-lock: cloud-compose-rootfs-script-modes: cmd.run: - - name: find /home/cloud-compose -type f -name '*.sh' -exec chmod 0755 {} + - - unless: test -z "$(find /home/cloud-compose -type f -name '*.sh' ! -perm -u=x -print -quit)" + - name: find /home/cloud-compose /usr/local/libexec/cloud-compose -maxdepth 1 -type f -name '*.sh' -exec chown root:root {} + -exec chmod 0755 {} + + - unless: test -z "$(find /home/cloud-compose /usr/local/libexec/cloud-compose -maxdepth 1 -type f -name '*.sh' \( ! -user root -o ! -group root -o ! -perm 0755 \) -print -quit)" - require: - file: cloud-compose-rootfs @@ -764,10 +764,18 @@ cloud-compose-systemd-reload: {% if rollout_enabled is sameas true %} cloud-compose-rollout-service: cmd.run: - - name: bash /home/cloud-compose/deploy-rollout.sh + - name: bash /usr/local/libexec/cloud-compose/run-root-program.sh deploy-rollout.sh - require: - file: cloud-compose-env + - file: cloud-compose-application-env + - file: cloud-compose-project-manifest + - file: cloud-compose-managed-runtime-artifacts - file: cloud-compose-rootfs + - cmd: cloud-compose-lifecycle-lock + - cmd: cloud-compose-rootfs-script-modes +{% for lifecycle in ['init', 'up', 'down', 'rollout'] %} + - file: cloud-compose-lifecycle-{{ lifecycle }} +{% endfor %} {% if reload_systemd %} - module: cloud-compose-systemd-reload {% endif %} @@ -776,7 +784,7 @@ cloud-compose-rollout-service: {% if force_bootstrap is sameas true %} cloud-compose-clear-bootstrap-marker: file.absent: - - name: {{ (home ~ '/.cloud-compose-bootstrap-complete') | json }} + - name: /var/lib/cloud-compose/bootstrap-complete - require: - cmd: cloud-compose-host-inputs-valid {% endif %} @@ -784,8 +792,8 @@ cloud-compose-clear-bootstrap-marker: {% if run_bootstrap is sameas true %} cloud-compose-bootstrap: cmd.run: - - name: bash {{ (home ~ '/start-cloud-compose-bootstrap.sh') | json }} - - creates: {{ (home ~ '/.cloud-compose-bootstrap-complete') | json }} + - name: bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh + - unless: bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh - require: {% if install_packages %} - service: cloud-compose-docker @@ -794,6 +802,13 @@ cloud-compose-bootstrap: - file: cloud-compose-application-env - file: cloud-compose-project-manifest - file: cloud-compose-managed-runtime-artifacts + - cmd: cloud-compose-rootfs-script-modes +{% for lifecycle in ['init', 'up', 'down', 'rollout'] %} + - file: cloud-compose-lifecycle-{{ lifecycle }} +{% endfor %} +{% if force_bootstrap is sameas true %} + - file: cloud-compose-clear-bootstrap-marker +{% endif %} {% if compose_projects %} - file: cloud-compose-project-dirs {% endif %} diff --git a/templates/cloud-init.yml b/templates/cloud-init.yml index 6b68bba..9694f3b 100644 --- a/templates/cloud-init.yml +++ b/templates/cloud-init.yml @@ -106,15 +106,12 @@ ${ROOTFS_ARCHIVE_COMMAND} chown cloud-compose:cloud-compose /mnt/disks/volumes chmod 0775 /mnt/disks/volumes install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops - rm -f /home/cloud-compose/.cloud-compose-bootstrap-complete - bash /home/cloud-compose/start-cloud-compose-bootstrap.sh + rm -f /var/lib/cloud-compose/bootstrap-complete + bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh %{ if length(ADDITIONAL_RUNCMD) > 0 ~} - | set -eu - test -f /home/cloud-compose/.cloud-compose-bootstrap-complete || { - echo "Cloud Compose application initialization did not complete; refusing post-initialization commands" >&2 - exit 1 - } + bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh %{ for CMD in ADDITIONAL_RUNCMD ~} ${indent(2, CMD)} %{ endfor ~} From b2057cfdf6c4d923b725093e1efc2bb58774c12b Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 20:44:14 +0000 Subject: [PATCH 09/67] [patch] Satisfy runtime shell lint --- ci/testdata/disaster-recovery/compose-apps.sh | 1 + rootfs/home/cloud-compose/bootstrap-helpers.sh | 2 +- rootfs/home/cloud-compose/libops-managed-runtime.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/testdata/disaster-recovery/compose-apps.sh b/ci/testdata/disaster-recovery/compose-apps.sh index 864de1f..e220836 100644 --- a/ci/testdata/disaster-recovery/compose-apps.sh +++ b/ci/testdata/disaster-recovery/compose-apps.sh @@ -2,6 +2,7 @@ compose_app_names_array() { local -n result="$1" + # shellcheck disable=SC2034 # The caller reads the array through this nameref. result=(alpha) } diff --git a/rootfs/home/cloud-compose/bootstrap-helpers.sh b/rootfs/home/cloud-compose/bootstrap-helpers.sh index 43aa5bf..0e4cfa9 100644 --- a/rootfs/home/cloud-compose/bootstrap-helpers.sh +++ b/rootfs/home/cloud-compose/bootstrap-helpers.sh @@ -32,7 +32,7 @@ cloud_compose_publish_marker() ( return 1 fi if [[ "$marker" == "/var/lib/cloud-compose/bootstrap-complete" && - ( EUID -ne 0 || + ( "$EUID" != "0" || "$(stat -c '%u:%g:%a:%F' -- "$marker_dir")" != "0:0:755:directory" ) ]]; then echo "Durable Cloud Compose readiness requires a root-owned state directory" >&2 return 1 diff --git a/rootfs/home/cloud-compose/libops-managed-runtime.sh b/rootfs/home/cloud-compose/libops-managed-runtime.sh index 41ea817..9fe81cc 100644 --- a/rootfs/home/cloud-compose/libops-managed-runtime.sh +++ b/rootfs/home/cloud-compose/libops-managed-runtime.sh @@ -92,7 +92,7 @@ prepare_managed_runtime_directory() { log "managed runtime directory is writable by another account: ${path}" return 1 fi - if [[ ( "$allow_owner_migration" != "true" || EUID -ne 0 ) && + if [[ ( "$allow_owner_migration" != "true" || "$EUID" != "0" ) && ( "$owner_uid" != "$expected_uid" || "$group_gid" != "$expected_gid" ) ]]; then log "managed runtime directory is not owned by the updater: ${path}" return 1 From 89f395bc9568e32d64a3d98c1c1d48e9e4bc8a43 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 20:49:36 +0000 Subject: [PATCH 10/67] [patch] Align Vault launcher contract --- ci/vault-runtime-contract.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ci/vault-runtime-contract.sh b/ci/vault-runtime-contract.sh index 521413c..1d96543 100644 --- a/ci/vault-runtime-contract.sh +++ b/ci/vault-runtime-contract.sh @@ -179,7 +179,13 @@ cmp -s "$tmp/expected-source-preparation.log" "$source_log" || \ fail "source preparation did not clone every app without executing lifecycle work" grep -Fq 'ExecStartPre=/bin/bash /home/cloud-compose/assert-vault-ready.sh' \ "$repo_root/rootfs/etc/systemd/system/cloud-compose.service" || fail "app service lacks a Vault readiness gate" -grep -Fq 'ExecStartPost=/bin/bash /home/cloud-compose/vault-agent-readiness.sh wait' \ - "$repo_root/rootfs/etc/systemd/system/cloud-compose-vault-agent.service" || fail "Vault unit does not publish token readiness" +vault_unit="$repo_root/rootfs/etc/systemd/system/cloud-compose-vault-agent.service" +for readiness_command in \ + 'ExecStartPre=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh prepare' \ + 'ExecStartPost=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh wait' \ + 'ExecStopPost=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh clear'; do + grep -Fq "$readiness_command" "$vault_unit" || \ + fail "Vault unit does not route readiness through the trusted root launcher: $readiness_command" +done echo "Vault runtime contract passed" From f0744efa2e80c1efe6e5e6538b60aebf769ea0bc Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 20:53:55 +0000 Subject: [PATCH 11/67] [patch] Make Salt rollout prerequisites explicit --- salt/cloud-compose/init.sls | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls index bd192c6..e62ffd5 100644 --- a/salt/cloud-compose/init.sls +++ b/salt/cloud-compose/init.sls @@ -773,9 +773,10 @@ cloud-compose-rollout-service: - file: cloud-compose-rootfs - cmd: cloud-compose-lifecycle-lock - cmd: cloud-compose-rootfs-script-modes -{% for lifecycle in ['init', 'up', 'down', 'rollout'] %} - - file: cloud-compose-lifecycle-{{ lifecycle }} -{% endfor %} + - file: cloud-compose-lifecycle-init + - file: cloud-compose-lifecycle-up + - file: cloud-compose-lifecycle-down + - file: cloud-compose-lifecycle-rollout {% if reload_systemd %} - module: cloud-compose-systemd-reload {% endif %} From fe03f104a2b1a67ee0dbac69385414248c487cd7 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 21:03:07 +0000 Subject: [PATCH 12/67] [patch] Validate checked-in recovery programs --- internal/contracttest/disaster_recovery_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/contracttest/disaster_recovery_test.go b/internal/contracttest/disaster_recovery_test.go index 81d65de..a4d8e03 100644 --- a/internal/contracttest/disaster_recovery_test.go +++ b/internal/contracttest/disaster_recovery_test.go @@ -51,6 +51,11 @@ func TestDisasterRecoveryReceiptAndRestoreContracts(t *testing.T) { library := readRepositoryFile(t, root, "rootfs/home/cloud-compose/disaster-recovery-lib.sh") backup := readRepositoryFile(t, root, "rootfs/home/cloud-compose/offhost-backup.sh") restore := readRepositoryFile(t, root, "rootfs/home/cloud-compose/restore-test.sh") + validationContract := strings.Join([]string{ + library, + readRepositoryFile(t, root, "rootfs/usr/local/share/cloud-compose/jq/dr-validate-backup-receipt.jq"), + readRepositoryFile(t, root, "rootfs/usr/local/share/cloud-compose/jq/dr-validate-restore-proof.jq"), + }, "\n") for marker, label := range map[string]string{ `env -i HOME=/root`: "clean driver environment", @@ -66,7 +71,7 @@ func TestDisasterRecoveryReceiptAndRestoreContracts(t *testing.T) { `.recovery_destroyed == true`: "recovery cleanup proof", `.source_receipt_sha256 == $receipt_sha256`: "source receipt binding", } { - requireContains(t, library, marker, label) + requireContains(t, validationContract, marker, label) } for marker, label := range map[string]string{ From 5e33a74aba84470a4421fb879d7fd4ecbee92de2 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 21:07:19 +0000 Subject: [PATCH 13/67] [patch] Cover checked-in backup manifest program --- internal/contracttest/disaster_recovery_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/contracttest/disaster_recovery_test.go b/internal/contracttest/disaster_recovery_test.go index a4d8e03..083d853 100644 --- a/internal/contracttest/disaster_recovery_test.go +++ b/internal/contracttest/disaster_recovery_test.go @@ -56,6 +56,10 @@ func TestDisasterRecoveryReceiptAndRestoreContracts(t *testing.T) { readRepositoryFile(t, root, "rootfs/usr/local/share/cloud-compose/jq/dr-validate-backup-receipt.jq"), readRepositoryFile(t, root, "rootfs/usr/local/share/cloud-compose/jq/dr-validate-restore-proof.jq"), }, "\n") + backupContract := strings.Join([]string{ + backup, + readRepositoryFile(t, root, "rootfs/usr/local/share/cloud-compose/jq/offhost-build-application-coverage.jq"), + }, "\n") for marker, label := range map[string]string{ `env -i HOME=/root`: "clean driver environment", @@ -81,7 +85,7 @@ func TestDisasterRecoveryReceiptAndRestoreContracts(t *testing.T) { `volume_topology`: "volume topology", `cloud_compose_dr_run_driver`: "provider-neutral backup handoff", } { - requireContains(t, backup, marker, label) + requireContains(t, backupContract, marker, label) } for marker, label := range map[string]string{ From 62896ba9cd5dcd668f61d78c2536025e61ba963a Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 21:25:11 +0000 Subject: [PATCH 14/67] [patch] Normalize rootfs archive ownership --- ci/host-runtime-security.sh | 5 +++-- modules/gcp/main.tf | 2 +- modules/linux-vm-runtime/main.tf | 2 +- modules/linux-vm-runtime/runtime_inputs.tftest.hcl | 6 +++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/host-runtime-security.sh b/ci/host-runtime-security.sh index e298e30..8079f5e 100644 --- a/ci/host-runtime-security.sh +++ b/ci/host-runtime-security.sh @@ -245,9 +245,10 @@ fi archive_source="$repo_root/modules/linux-vm-runtime/main.tf" verify_line="$(grep -n 'sha256sum -c -' "$archive_source" | head -n 1 | cut -d: -f1)" -extract_line="$(grep -n 'tar -xzf "\$tmp/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" +extract_line="$(grep -n 'tar --no-same-owner -xzf "\$tmp/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" [[ -n "$verify_line" && -n "$extract_line" && "$verify_line" -lt "$extract_line" ]] || \ - fail "rootfs archive is not verified before extraction" + fail "rootfs archive is not verified before ownership-safe extraction" +assert_contains "$repo_root/modules/gcp/main.tf" 'tar --no-same-owner -xzf "$tmp/rootfs.tar.gz"' for variables_file in \ "$repo_root/variables.tf" \ diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index 1831ff7..b5b2035 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -445,7 +445,7 @@ rootfs_archive_command_raw = <<-EOT exit 1 fi printf '%s %s\n' "$archive_sha256" "$tmp/rootfs.tar.gz" | sha256sum -c - - tar -xzf "$tmp/rootfs.tar.gz" -C "$tmp" + tar --no-same-owner -xzf "$tmp/rootfs.tar.gz" -C "$tmp" rootfs_dir="$(find "$tmp" -mindepth 1 -maxdepth 3 -type d -name rootfs -print -quit)" if [ -z "$rootfs_dir" ]; then echo "rootfs directory not found in $archive_url" >&2 diff --git a/modules/linux-vm-runtime/main.tf b/modules/linux-vm-runtime/main.tf index b4a4502..8e89792 100644 --- a/modules/linux-vm-runtime/main.tf +++ b/modules/linux-vm-runtime/main.tf @@ -303,7 +303,7 @@ rootfs_archive_prepare_command_raw = <<-EOT --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 900 \ --connect-timeout 10 --max-time 300 -o "$tmp/rootfs.tar.gz" -- "$archive_url" printf '%s %s\n' "$archive_sha256" "$tmp/rootfs.tar.gz" | sha256sum -c - - tar -xzf "$tmp/rootfs.tar.gz" -C "$tmp" + tar --no-same-owner -xzf "$tmp/rootfs.tar.gz" -C "$tmp" rootfs_dir="$(find "$tmp" -mindepth 1 -maxdepth 3 -type d -name rootfs -print -quit)" if [ -z "$rootfs_dir" ] || [ ! -d "$rootfs_dir" ]; then echo "rootfs directory not found in verified archive $archive_url" >&2 diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index dd72833..78bc2a1 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -224,13 +224,13 @@ run "renders_verified_rootfs_archive" { assert { condition = ( - strcontains(output.cloud_init, "tar -xzf \"$tmp/rootfs.tar.gz\"") && + strcontains(output.cloud_init, "tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"") && strcontains(output.cloud_init, "install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"") && strcontains(output.cloud_init, "bash \"$filesystem_prep\"") && strcontains(output.cloud_init, "bash \"$filesystem_persist\"") && strcontains(output.cloud_init, "cp -a \"$rootfs_dir\"/. /") && - length(split("sha256sum -c -", output.cloud_init)[0]) < length(split("tar -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) && - length(split("tar -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) < length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) && + length(split("sha256sum -c -", output.cloud_init)[0]) < length(split("tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) && + length(split("tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) < length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) && length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) < length(split("bash \"$filesystem_prep\"", output.cloud_init)[0]) && length(split("bash \"$filesystem_persist\"", output.cloud_init)[0]) < length(split("cp -a \"$rootfs_dir\"/. /", output.cloud_init)[0]) ) From 38b8862636f567b2302ce07f95b453879de4b96c Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 22:09:26 +0000 Subject: [PATCH 15/67] [patch] Support COS writable runtime paths --- README.md | 2 +- ansible/README.md | 2 +- ansible/roles/cloud_compose/tasks/main.yml | 48 +++++++++--- ci/bootstrap-recovery-contract.sh | 12 +-- ci/cloud-smoke.sh | 2 +- ci/config-management-cloud-smoke-inner.sh | 12 +-- ci/config-management-input-contract.sh | 14 ++-- ci/config-management-smoke-assert.py | 46 ++++++++++- ci/cos-bootstrap-contract.sh | 45 ++++++++--- ci/cos-jq-portability-contract.sh | 2 +- ci/disaster-recovery-contract.sh | 2 +- ci/filesystem-prep-contract.sh | 4 +- ci/gcp-upgrade-smoke-contract.sh | 4 +- ci/host-runtime-security.sh | 41 ++++++++-- ci/rootfs-package-contract.sh | 14 +++- ci/systemd-contract.sh | 20 ++--- ci/vault-runtime-contract.sh | 6 +- docs/disaster-recovery.md | 8 +- docs/runtime-contracts.md | 27 ++++--- .../contracttest/disaster_recovery_test.go | 8 +- modules/digitalocean/variables.tf | 2 +- modules/gcp/main.tf | 11 ++- modules/gcp/runtime_contracts.tftest.hcl | 64 +++++++++++++--- .../libexec/custom-offhost-driver | 1 + modules/gcp/variables.tf | 2 +- modules/linode/variables.tf | 2 +- .../disaster_recovery.tftest.hcl | 6 +- modules/linux-vm-runtime/main.tf | 11 ++- .../runtime_inputs.tftest.hcl | 76 ++++++++++++++----- .../linux-vm-runtime/templates/cloud-init.yml | 18 ++--- .../libexec/custom-offhost-driver | 1 + .../rootfs/etc/cloud-compose/unrelated-config | 1 + modules/linux-vm-runtime/variables.tf | 2 +- providers/do/variables.tf | 2 +- providers/gcp/variables.tf | 2 +- providers/linode/variables.tf | 2 +- .../bin}/cloud-compose-diagnostics.sh | 4 +- .../cloud-compose/jq/diagnostics-entry-app.jq | 0 .../jq/diagnostics-entry-project-dir.jq | 0 .../jq/diagnostics-project-entries.jq | 0 .../diagnostics-validate-compose-projects.jq | 0 .../jq/dr-backup-completed-at.jq | 0 .../cloud-compose/jq/dr-backup-remote-id.jq | 0 .../jq/dr-restore-completed-at.jq | 0 .../jq/dr-restore-recovery-id.jq | 0 .../jq/dr-validate-backup-receipt.jq | 0 .../jq/dr-validate-restore-proof.jq | 0 .../cloud-compose/jq/offhost-bind-sources.jq | 0 .../jq/offhost-build-application-coverage.jq | 0 .../jq/offhost-build-manifest.jq | 0 .../jq/offhost-manifest-app-digests.jq | 0 .../jq/offhost-validate-bind-roots.jq | 0 .../jq/offhost-validate-compose-config.jq | 0 .../jq/offhost-validate-manifest.jq | 0 .../libexec}/bootstrap-required.sh | 2 +- .../libexec}/bootstrap-security.sh | 0 .../cloud-compose/libexec/build-cos-make.sh | 46 +++++++++++ .../libexec}/require-bootstrap-ready.sh | 2 +- .../cloud-compose/libexec}/run-bootstrap.sh | 2 +- .../libexec}/run-root-program.sh | 2 +- .../libexec}/start-cloud-compose-bootstrap.sh | 2 +- .../system/cloud-compose-bootstrap.service | 4 +- .../system/cloud-compose-docker-prune.service | 2 +- .../system/cloud-compose-key-rotation.service | 2 +- ...loud-compose-metadata-firewall-pre.service | 2 +- .../cloud-compose-metadata-firewall.service | 2 +- .../cloud-compose-offhost-backup.service | 2 +- .../system/cloud-compose-overlay.service | 2 +- .../system/cloud-compose-restore-test.service | 2 +- .../system/cloud-compose-vault-agent.service | 6 +- .../system/libops-managed-runtime.service | 2 +- .../cloud-compose/disaster-recovery-lib.sh | 4 +- .../cloud-compose/install-dependencies-cos.sh | 52 ++----------- rootfs/home/cloud-compose/offhost-backup.sh | 2 +- rootfs/home/cloud-compose/restore-test.sh | 2 +- runtime_contracts.tftest.hcl | 4 +- salt/cloud-compose/README.md | 2 +- salt/cloud-compose/init.sls | 35 +++++++-- templates/cloud-init.yml | 14 ++-- .../ansible/invalid-disaster-recovery.yml | 2 +- tests/config-management/ansible/smoke.yml | 4 +- .../salt-pillar/invalid-disaster-recovery.sls | 2 +- .../config-management/salt-pillar/wp-prod.sls | 2 +- variables.tf | 2 +- 84 files changed, 502 insertions(+), 233 deletions(-) create mode 100644 modules/gcp/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver create mode 100644 modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver create mode 100644 modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/unrelated-config rename rootfs/{usr/local/sbin => etc/cloud-compose/bin}/cloud-compose-diagnostics.sh (98%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/diagnostics-entry-app.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/diagnostics-entry-project-dir.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/diagnostics-project-entries.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/diagnostics-validate-compose-projects.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/dr-backup-completed-at.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/dr-backup-remote-id.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/dr-restore-completed-at.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/dr-restore-recovery-id.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/dr-validate-backup-receipt.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/dr-validate-restore-proof.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/offhost-bind-sources.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/offhost-build-application-coverage.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/offhost-build-manifest.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/offhost-manifest-app-digests.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/offhost-validate-bind-roots.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/offhost-validate-compose-config.jq (100%) rename rootfs/{usr/local/share => etc}/cloud-compose/jq/offhost-validate-manifest.jq (100%) rename rootfs/{usr/local/libexec/cloud-compose => etc/cloud-compose/libexec}/bootstrap-required.sh (86%) mode change 100644 => 100755 rename rootfs/{usr/local/libexec/cloud-compose => etc/cloud-compose/libexec}/bootstrap-security.sh (100%) mode change 100644 => 100755 create mode 100755 rootfs/etc/cloud-compose/libexec/build-cos-make.sh rename rootfs/{usr/local/libexec/cloud-compose => etc/cloud-compose/libexec}/require-bootstrap-ready.sh (80%) mode change 100644 => 100755 rename rootfs/{usr/local/libexec/cloud-compose => etc/cloud-compose/libexec}/run-bootstrap.sh (70%) mode change 100644 => 100755 rename rootfs/{usr/local/libexec/cloud-compose => etc/cloud-compose/libexec}/run-root-program.sh (91%) mode change 100644 => 100755 rename rootfs/{usr/local/libexec/cloud-compose => etc/cloud-compose/libexec}/start-cloud-compose-bootstrap.sh (73%) mode change 100644 => 100755 diff --git a/README.md b/README.md index 6a65f89..429bcd4 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ No resources. | [name](#input\_name) | Deployment name. | `string` | n/a | yes | | [cloud\_provider](#input\_cloud\_provider) | Compatibility selector for the root GCP entrypoint. Use providers/do or providers/linode for other clouds. | `string` | `"gcp"` | no | | [gcp](#input\_gcp) | Google Cloud infrastructure settings. | 
object({
    project_id     = optional(string, "")
    project_number = optional(string, "")
    region         = optional(string, "us-east5")
    zone           = optional(string, "us-east5-b")

    identity = optional(object({
      vm_service_account_email  = optional(string, "")
      app_service_account_email = optional(string, "")
      app_credentials_enabled   = optional(bool, false)
    }), {})

    instance = optional(object({
      machine_type = optional(string, "n4-standard-2")
      os           = optional(string, "cos-125-19216-220-185")
      production   = optional(bool, false)
    }), {})

    disks = optional(object({
      type                   = optional(string, "hyperdisk-balanced")
      data_size_gb           = optional(number, 20)
      docker_volumes_size_gb = optional(number, 50)
    }), {})

    network = optional(object({
      create                   = optional(bool, true)
      project_id               = optional(string, "")
      name                     = optional(string, "")
      subnetwork               = optional(string, "")
      ip_cidr_range            = optional(string, "10.42.0.0/24")
      mtu                      = optional(number, 1460)
      power_button_allowed_ips = optional(list(string), [])
      power_button_ip_depth    = optional(number)
      ssh_ipv4                 = optional(list(string), [])
      ssh_ipv6                 = optional(list(string), [])
    }), {})

    snapshots = optional(object({
      enabled = optional(bool, true)
    }), {})

    overlay = optional(object({
      source_instance = optional(string, "")
      volume_names    = optional(list(string), [])
    }), {})

    cloud_init = optional(object({
      initcmd = optional(list(string), [])
      runcmd  = optional(list(string), [])
    }), {})

    artifact_registry = optional(object({
      repository = optional(string, "")
      location   = optional(string, "us")
    }), {})

    power_management = optional(object({
      enabled      = optional(bool, false)
      start_role   = optional(string, "")
      suspend_role = optional(string, "")
      frontend = optional(object({
        image  = string
        port   = optional(number, 8080)
        cpu    = optional(string, "1000m")
        memory = optional(string, "1Gi")
      }), null)
    }), {})

    rollout = optional(object({
      enabled        = optional(bool, false)
      release_url    = optional(string, "")
      release_sha256 = optional(string, "")
      port           = optional(number, 8081)
      jwks_uri       = optional(string, "")
      jwt_audience   = optional(string, "")
      custom_claims  = optional(string, "")
      allowed_ipv4   = optional(list(string), ["10.0.0.0/8"])
    }), {})
  })
| `{}` | no | -| [runtime](#input\_runtime) | Provider-neutral compose/runtime settings. | 
object({
    rootfs                = optional(string, "")
    rootfs_archive_url    = optional(string, "")
    rootfs_archive_sha256 = optional(string, "")
    users                 = optional(map(list(string)), {})

    disaster_recovery = optional(object({
      required    = optional(bool, false)
      driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver")
    }), {})

    compose = optional(object({
      primary      = optional(string, "")
      ingress_port = optional(number, 80)
      ingress = optional(object({
        letsencrypt     = optional(bool, false)
        bot_mitigation  = optional(bool, false)
        mode            = optional(string, "")
        domain          = optional(string, "")
        acme_email      = optional(string, "")
        trusted_ips     = optional(list(string), [])
        max_upload_size = optional(string, "")
        upload_timeout  = optional(string, "")
      }), {})
      repo   = optional(string, "")
      branch = optional(string, "")
      projects = optional(map(object({
        docker_compose_repo   = string
        docker_compose_branch = optional(string)
        project_dir           = optional(string)
        compose_project_name  = optional(string)
        ingress_port          = optional(number)
        ingress = optional(object({
          letsencrypt     = optional(bool)
          bot_mitigation  = optional(bool)
          mode            = optional(string)
          domain          = optional(string)
          acme_email      = optional(string)
          trusted_ips     = optional(list(string))
          max_upload_size = optional(string)
          upload_timeout  = optional(string)
        }), {})
        sitectl_context_name   = optional(string)
        sitectl_plugin         = optional(string)
        sitectl_environment    = optional(string)
        sitectl_packages       = optional(list(string))
        sitectl_verify_args    = optional(list(string))
        docker_compose_init    = optional(list(string))
        docker_compose_up      = optional(list(string))
        docker_compose_down    = optional(list(string))
        docker_compose_rollout = optional(list(string))
      })), {})
      init    = optional(list(string))
      up      = optional(list(string))
      down    = optional(list(string))
      rollout = optional(list(string))
    }), {})

    sitectl = optional(object({
      packages         = optional(list(string))
      version          = optional(string, "latest")
      package_versions = optional(map(string), {})
      context_name     = optional(string, "")
      plugin           = optional(string, "core")
      environment      = optional(string, "production")
      verify_args      = optional(list(string), [])
    }), {})

    docker = optional(object({
      # renovate: datasource=github-releases depName=docker-compose packageName=docker/compose versioning=semver
      compose_version = optional(string, "v5.3.1")
      # renovate: datasource=github-releases depName=docker-buildx packageName=docker/buildx versioning=semver
      buildx_version = optional(string, "v0.35.0")
    }), {})

    managed_runtime = optional(object({
      enabled                       = optional(bool, true)
      internal_services_enabled     = optional(bool, false)
      internal_services_auto_update = optional(bool, false)
      artifacts = optional(list(object({
        name    = string
        url     = string
        sha256  = string
        path    = string
        mode    = optional(string, "0755")
        owner   = optional(string, "root")
        group   = optional(string, "root")
        restart = optional(string, "")
      })), [])
    }), {})

    vault = optional(object({
      addr                    = optional(string, "")
      namespace               = optional(string, "")
      role                    = optional(string, "")
      agent_enabled           = optional(bool, false)
      auth_method             = optional(string, "auto")
      gcp_auth_mount_path     = optional(string, "auth/gcp")
      agent_token_path        = optional(string, "/mnt/disks/data/vault/token")
      agent_additional_config = optional(string, "")
      agent_templates = optional(list(object({
        destination = string
        contents    = string
        perms       = optional(string, "0640")
        command     = optional(string, "")
      })), [])
    }), {})

    extra_env = optional(map(string), {})
  })
| `{}` | no | +| [runtime](#input\_runtime) | Provider-neutral compose/runtime settings. | 
object({
    rootfs                = optional(string, "")
    rootfs_archive_url    = optional(string, "")
    rootfs_archive_sha256 = optional(string, "")
    users                 = optional(map(list(string)), {})

    disaster_recovery = optional(object({
      required    = optional(bool, false)
      driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")
    }), {})

    compose = optional(object({
      primary      = optional(string, "")
      ingress_port = optional(number, 80)
      ingress = optional(object({
        letsencrypt     = optional(bool, false)
        bot_mitigation  = optional(bool, false)
        mode            = optional(string, "")
        domain          = optional(string, "")
        acme_email      = optional(string, "")
        trusted_ips     = optional(list(string), [])
        max_upload_size = optional(string, "")
        upload_timeout  = optional(string, "")
      }), {})
      repo   = optional(string, "")
      branch = optional(string, "")
      projects = optional(map(object({
        docker_compose_repo   = string
        docker_compose_branch = optional(string)
        project_dir           = optional(string)
        compose_project_name  = optional(string)
        ingress_port          = optional(number)
        ingress = optional(object({
          letsencrypt     = optional(bool)
          bot_mitigation  = optional(bool)
          mode            = optional(string)
          domain          = optional(string)
          acme_email      = optional(string)
          trusted_ips     = optional(list(string))
          max_upload_size = optional(string)
          upload_timeout  = optional(string)
        }), {})
        sitectl_context_name   = optional(string)
        sitectl_plugin         = optional(string)
        sitectl_environment    = optional(string)
        sitectl_packages       = optional(list(string))
        sitectl_verify_args    = optional(list(string))
        docker_compose_init    = optional(list(string))
        docker_compose_up      = optional(list(string))
        docker_compose_down    = optional(list(string))
        docker_compose_rollout = optional(list(string))
      })), {})
      init    = optional(list(string))
      up      = optional(list(string))
      down    = optional(list(string))
      rollout = optional(list(string))
    }), {})

    sitectl = optional(object({
      packages         = optional(list(string))
      version          = optional(string, "latest")
      package_versions = optional(map(string), {})
      context_name     = optional(string, "")
      plugin           = optional(string, "core")
      environment      = optional(string, "production")
      verify_args      = optional(list(string), [])
    }), {})

    docker = optional(object({
      # renovate: datasource=github-releases depName=docker-compose packageName=docker/compose versioning=semver
      compose_version = optional(string, "v5.3.1")
      # renovate: datasource=github-releases depName=docker-buildx packageName=docker/buildx versioning=semver
      buildx_version = optional(string, "v0.35.0")
    }), {})

    managed_runtime = optional(object({
      enabled                       = optional(bool, true)
      internal_services_enabled     = optional(bool, false)
      internal_services_auto_update = optional(bool, false)
      artifacts = optional(list(object({
        name    = string
        url     = string
        sha256  = string
        path    = string
        mode    = optional(string, "0755")
        owner   = optional(string, "root")
        group   = optional(string, "root")
        restart = optional(string, "")
      })), [])
    }), {})

    vault = optional(object({
      addr                    = optional(string, "")
      namespace               = optional(string, "")
      role                    = optional(string, "")
      agent_enabled           = optional(bool, false)
      auth_method             = optional(string, "auto")
      gcp_auth_mount_path     = optional(string, "auth/gcp")
      agent_token_path        = optional(string, "/mnt/disks/data/vault/token")
      agent_additional_config = optional(string, "")
      agent_templates = optional(list(object({
        destination = string
        contents    = string
        perms       = optional(string, "0640")
        command     = optional(string, "")
      })), [])
    }), {})

    extra_env = optional(map(string), {})
  })
| `{}` | no | | [template](#input\_template) | Optional compose template preset. Supported values are archivesspace, ojs, isle, drupal, wp, omeka-s, and omeka-classic. Explicit runtime settings override preset defaults. | `string` | `""` | no | ## Outputs diff --git a/ansible/README.md b/ansible/README.md index 8dc3181..96f9875 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -68,7 +68,7 @@ The provider-neutral disaster-recovery interface is shared with Terraform: cloud_compose_runtime: disaster_recovery: required: true - driver_path: /usr/local/libexec/cloud-compose/offhost-backup-driver + driver_path: /etc/cloud-compose/libexec/offhost-backup-driver ``` Install that executable and its credentials separately as root. The role diff --git a/ansible/roles/cloud_compose/tasks/main.yml b/ansible/roles/cloud_compose/tasks/main.yml index 8dcb9e2..0567e84 100644 --- a/ansible/roles/cloud_compose/tasks/main.yml +++ b/ansible/roles/cloud_compose/tasks/main.yml @@ -82,10 +82,10 @@ - (_cc_vault.agent_enabled | default(false)) is boolean - not (_cc_vault.agent_enabled | default(false)) - (_cc_disaster_recovery.required | default(false)) is boolean - - (_cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver')) is string - - (_cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver')) is match('^/[A-Za-z0-9._/+:-]+$') - - "'//' not in (_cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver'))" - - (_cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver') | regex_search('(^|/)\\.\\.?(/|$)')) is none + - (_cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver')) is string + - (_cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver')) is match('^/[A-Za-z0-9._/+:-]+$') + - "'//' not in (_cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver'))" + - (_cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver') | regex_search('(^|/)\\.\\.?(/|$)')) is none fail_msg: "runtime sections must be maps, runtime feature switches must be booleans, template must name a supported app, Vault Agent is currently supported only by Terraform providers (set vault.agent_enabled=false for Ansible), and disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments." - name: Require safe rollout service settings @@ -144,7 +144,7 @@ _cc_rollout_commands: "{{ _cc_compose.rollout | default(cloud_compose_default_rollout) }}" _cc_internal_services_enabled: "{{ _cc_managed.internal_services_enabled | default(cloud_compose_internal_services_enabled) }}" _cc_offhost_backup_required: "{{ _cc_disaster_recovery.required | default(false) }}" - _cc_offhost_backup_driver: "{{ _cc_disaster_recovery.driver_path | default('/usr/local/libexec/cloud-compose/offhost-backup-driver') }}" + _cc_offhost_backup_driver: "{{ _cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver') }}" - name: Reject GCP-only internal services on an on-prem host ansible.builtin.assert: @@ -514,6 +514,19 @@ dest: / mode: preserve +- name: Secure Cloud Compose privileged program directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + loop: + - /etc/cloud-compose + - /etc/cloud-compose/bin + - /etc/cloud-compose/jq + - /etc/cloud-compose/libexec + - name: Inspect cloud-compose lifecycle lock directory ansible.builtin.stat: path: /run/lock/cloud-compose @@ -547,7 +560,8 @@ ansible.builtin.find: paths: - "{{ cloud_compose_home }}" - - /usr/local/libexec/cloud-compose + - /etc/cloud-compose/bin + - /etc/cloud-compose/libexec patterns: "*.sh" recurse: false file_type: file @@ -561,6 +575,22 @@ mode: "0755" loop: "{{ cloud_compose_shell_scripts.files }}" +- name: Find checked-in Cloud Compose jq programs + ansible.builtin.find: + paths: /etc/cloud-compose/jq + patterns: "*.jq" + recurse: false + file_type: file + register: cloud_compose_jq_programs + +- name: Make checked-in Cloud Compose jq programs root-controlled + ansible.builtin.file: + path: "{{ item.path }}" + owner: root + group: root + mode: "0644" + loop: "{{ cloud_compose_jq_programs.files }}" + - name: Write cloud-compose lifecycle dispatchers ansible.builtin.copy: dest: "{{ cloud_compose_home }}/{{ item }}" @@ -621,7 +651,7 @@ - name: Install and start authenticated rollout service ansible.builtin.command: - cmd: bash /usr/local/libexec/cloud-compose/run-root-program.sh deploy-rollout.sh + cmd: bash /etc/cloud-compose/libexec/run-root-program.sh deploy-rollout.sh when: _cc_rollout_service.enabled | bool - name: Clear cloud-compose bootstrap marker @@ -632,7 +662,7 @@ - name: Inspect validated cloud-compose bootstrap readiness ansible.builtin.command: - cmd: bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh + cmd: bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh register: cloud_compose_bootstrap_readiness changed_when: false failed_when: false @@ -640,7 +670,7 @@ - name: Bootstrap cloud-compose runtime ansible.builtin.command: - cmd: bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh + cmd: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh async: "{{ cloud_compose_bootstrap_timeout }}" poll: "{{ cloud_compose_bootstrap_poll_interval }}" when: diff --git a/ci/bootstrap-recovery-contract.sh b/ci/bootstrap-recovery-contract.sh index bc20151..5486cf9 100644 --- a/ci/bootstrap-recovery-contract.sh +++ b/ci/bootstrap-recovery-contract.sh @@ -9,11 +9,11 @@ start_bootstrap="$repo_root/rootfs/home/cloud-compose/start-cloud-compose-bootst run_script="$repo_root/rootfs/home/cloud-compose/run.sh" app_init="$repo_root/rootfs/home/cloud-compose/app-init.sh" run_bootstrap="$repo_root/rootfs/home/cloud-compose/run-bootstrap.sh" -bootstrap_security="$repo_root/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh" -bootstrap_required="$repo_root/rootfs/usr/local/libexec/cloud-compose/bootstrap-required.sh" -bootstrap_entrypoint="$repo_root/rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh" -bootstrap_start_entrypoint="$repo_root/rootfs/usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh" -bootstrap_readiness_gate="$repo_root/rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh" +bootstrap_security="$repo_root/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh" +bootstrap_required="$repo_root/rootfs/etc/cloud-compose/libexec/bootstrap-required.sh" +bootstrap_entrypoint="$repo_root/rootfs/etc/cloud-compose/libexec/run-bootstrap.sh" +bootstrap_start_entrypoint="$repo_root/rootfs/etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh" +bootstrap_readiness_gate="$repo_root/rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh" bootstrap_unit="$repo_root/rootfs/etc/systemd/system/cloud-compose-bootstrap.service" tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT @@ -277,7 +277,7 @@ fi for cloud_init_template in \ "$repo_root/templates/cloud-init.yml" \ "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do - assert_contains "$cloud_init_template" 'bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh' + assert_contains "$cloud_init_template" 'bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' if grep -Fq 'bash /home/cloud-compose/run.sh > /home/cloud-compose/run.log 2>&1' \ "$cloud_init_template"; then fail "cloud-init bypasses the retryable bootstrap unit" diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh index 77164cc..eb6ac11 100755 --- a/ci/cloud-smoke.sh +++ b/ci/cloud-smoke.sh @@ -3,7 +3,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "$0")/.." && pwd)" -readonly diagnostics_program="/usr/local/sbin/cloud-compose-diagnostics.sh" +readonly diagnostics_program="/etc/cloud-compose/bin/cloud-compose-diagnostics.sh" readonly smoke_healthcheck_program="/home/cloud-compose/smoke-healthcheck.sh" usage() { diff --git a/ci/config-management-cloud-smoke-inner.sh b/ci/config-management-cloud-smoke-inner.sh index d872ff5..4a70bf9 100755 --- a/ci/config-management-cloud-smoke-inner.sh +++ b/ci/config-management-cloud-smoke-inner.sh @@ -197,12 +197,12 @@ test -x /home/cloud-compose/down test -x /home/cloud-compose/rollout test -x /home/cloud-compose/run.sh test -x /home/cloud-compose/start-cloud-compose-bootstrap.sh -test -x /usr/local/libexec/cloud-compose/bootstrap-required.sh -test -x /usr/local/libexec/cloud-compose/bootstrap-security.sh -test -x /usr/local/libexec/cloud-compose/run-bootstrap.sh -test -x /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh -test -x /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh -test -x /usr/local/libexec/cloud-compose/run-root-program.sh +test -x /etc/cloud-compose/libexec/bootstrap-required.sh +test -x /etc/cloud-compose/libexec/bootstrap-security.sh +test -x /etc/cloud-compose/libexec/run-bootstrap.sh +test -x /etc/cloud-compose/libexec/require-bootstrap-ready.sh +test -x /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh +test -x /etc/cloud-compose/libexec/run-root-program.sh python3 -m json.tool /home/cloud-compose/compose-projects.json >/dev/null python3 -m json.tool /home/cloud-compose/application-env.json >/dev/null diff --git a/ci/config-management-input-contract.sh b/ci/config-management-input-contract.sh index a1e1d0a..bdc060b 100755 --- a/ci/config-management-input-contract.sh +++ b/ci/config-management-input-contract.sh @@ -13,8 +13,8 @@ cmp -s "$validator" "$salt_validator" || { ansible_tasks="$repo_root/ansible/roles/cloud_compose/tasks/main.yml" salt_state="$repo_root/salt/cloud-compose/init.sls" -root_program_runner="$repo_root/rootfs/usr/local/libexec/cloud-compose/run-root-program.sh" -rollout_runner="bash /usr/local/libexec/cloud-compose/run-root-program.sh deploy-rollout.sh" +root_program_runner="$repo_root/rootfs/etc/cloud-compose/libexec/run-root-program.sh" +rollout_runner="bash /etc/cloud-compose/libexec/run-root-program.sh deploy-rollout.sh" contract_fail() { echo "config-management input contract: $1" >&2 @@ -217,9 +217,9 @@ if "files/validate-runtime-inputs.py" not in ansible_tasks: fail("Ansible does not execute the shared host-input validator") if "--data-root" in ansible_tasks: fail("Ansible makes the production project ownership boundary configurable") -if "cmd: bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh" not in ansible_tasks: +if "cmd: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh" not in ansible_tasks: fail("Ansible bypasses the retryable bootstrap service") -if "cmd: bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh" not in ansible_tasks: +if "cmd: bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh" not in ansible_tasks: fail("Ansible does not validate bootstrap readiness evidence") if 'cmd: bash "{{ cloud_compose_home }}/run.sh"' in ansible_tasks: fail("Ansible still invokes the one-shot bootstrap script directly") @@ -233,9 +233,9 @@ salt_gate = salt_state.find("cloud-compose-host-inputs-valid:") salt_first_mutation = salt_state.find("cloud-compose-packages:") if salt_gate < 0 or salt_first_mutation < 0 or salt_gate > salt_first_mutation: fail("Salt host-input validation does not precede its first host mutation") -if "bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh" not in salt_state: +if "bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh" not in salt_state: fail("Salt bypasses the retryable bootstrap service") -if "bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh" not in salt_state: +if "bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh" not in salt_state: fail("Salt does not validate bootstrap readiness evidence") if "home ~ '/run.sh'" in salt_state: fail("Salt still invokes the one-shot bootstrap script directly") @@ -411,7 +411,7 @@ adapter_fixture_markers = { "invalid-lifecycle": "- 42", "invalid-project-lifecycle": "docker_compose_up: not-a-list", "invalid-internal-services": "internal_services_enabled: true", - "invalid-disaster-recovery": "driver_path: /usr/local/libexec/cloud-compose/../untrusted", + "invalid-disaster-recovery": "driver_path: /etc/cloud-compose/libexec/../untrusted", } for fixture_name, marker in adapter_fixture_markers.items(): for relative_path in ( diff --git a/ci/config-management-smoke-assert.py b/ci/config-management-smoke-assert.py index 247163b..fc777ab 100644 --- a/ci/config-management-smoke-assert.py +++ b/ci/config-management-smoke-assert.py @@ -12,7 +12,10 @@ RUNTIME_HOME = Path("/home/cloud-compose") -BOOTSTRAP_LIBEXEC = Path("/usr/local/libexec/cloud-compose") +PRIVILEGED_PROGRAM_ROOT = Path("/etc/cloud-compose") +DIAGNOSTICS_PROGRAM = PRIVILEGED_PROGRAM_ROOT / "bin/cloud-compose-diagnostics.sh" +BOOTSTRAP_LIBEXEC = Path("/etc/cloud-compose/libexec") +JQ_PROGRAM_DIR = PRIVILEGED_PROGRAM_ROOT / "jq" def load_runtime_env(path: Path) -> dict[str, str]: @@ -52,10 +55,17 @@ def assert_runtime_files() -> None: BOOTSTRAP_LIBEXEC / "require-bootstrap-ready.sh", BOOTSTRAP_LIBEXEC / "run-root-program.sh", BOOTSTRAP_LIBEXEC / "start-cloud-compose-bootstrap.sh", + DIAGNOSTICS_PROGRAM, ]: assert path.exists(), path assert os.access(path, os.X_OK), path + for path in [ + JQ_PROGRAM_DIR / "diagnostics-validate-compose-projects.jq", + JQ_PROGRAM_DIR / "offhost-validate-manifest.jq", + ]: + assert path.exists(), path + cloud_compose_gid = grp.getgrnam("cloud-compose").gr_gid for path, expected_mode in { RUNTIME_HOME / "init": 0o750, @@ -84,6 +94,36 @@ def assert_runtime_files() -> None: oct(stat.S_IMODE(metadata.st_mode)), ) + for path in [ + PRIVILEGED_PROGRAM_ROOT, + DIAGNOSTICS_PROGRAM.parent, + BOOTSTRAP_LIBEXEC, + JQ_PROGRAM_DIR, + ]: + metadata = path.stat() + assert metadata.st_uid == 0, (path, metadata.st_uid) + assert metadata.st_gid == 0, (path, metadata.st_gid) + assert stat.S_IMODE(metadata.st_mode) == 0o755, ( + path, + oct(stat.S_IMODE(metadata.st_mode)), + ) + + diagnostics_metadata = DIAGNOSTICS_PROGRAM.stat() + assert diagnostics_metadata.st_uid == 0 + assert diagnostics_metadata.st_gid == 0 + assert stat.S_IMODE(diagnostics_metadata.st_mode) == 0o755 + + jq_programs = list(JQ_PROGRAM_DIR.glob("*.jq")) + assert jq_programs + for path in jq_programs: + metadata = path.stat() + assert metadata.st_uid == 0, (path, metadata.st_uid) + assert metadata.st_gid == 0, (path, metadata.st_gid) + assert stat.S_IMODE(metadata.st_mode) == 0o644, ( + path, + oct(stat.S_IMODE(metadata.st_mode)), + ) + def assert_ansible_runtime() -> None: env = load_runtime_env(RUNTIME_HOME / ".env") @@ -97,7 +137,7 @@ def assert_ansible_runtime() -> None: assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false" assert ( env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"] - == "/usr/local/libexec/cloud-compose/ansible-offhost" + == "/etc/cloud-compose/libexec/ansible-offhost" ) assert env["DOCKER_COMPOSE_DIR"] == "/mnt/disks/data/libops/isle/isle-prod" assert env["DOCKER_COMPOSE_REPO"] == "https://github.com/libops/isle" @@ -203,7 +243,7 @@ def assert_salt_runtime( assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false" assert ( env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"] - == "/usr/local/libexec/cloud-compose/salt-offhost" + == "/etc/cloud-compose/libexec/salt-offhost" ) assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { "sitectl": "v1.8.2", diff --git a/ci/cos-bootstrap-contract.sh b/ci/cos-bootstrap-contract.sh index adf8219..8b0998d 100755 --- a/ci/cos-bootstrap-contract.sh +++ b/ci/cos-bootstrap-contract.sh @@ -4,6 +4,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" INSTALLER="${ROOT_DIR}/rootfs/home/cloud-compose/install-dependencies-cos.sh" +BUILD_PROGRAM="${ROOT_DIR}/rootfs/etc/cloud-compose/libexec/build-cos-make.sh" TEST_ROOT="$(mktemp -d)" trap 'rm -rf -- "$TEST_ROOT"' EXIT @@ -21,22 +22,22 @@ if ! grep -Fqx 'ALPINE_BUILD_IMAGE="alpine:3.22@sha256:14358309a308569c32bdc37e2 echo "COS bootstrap Alpine image is not digest pinned" >&2 exit 1 fi -grep -Fq '"${alpine_mirror}/v3.22/main"' "$INSTALLER" || { +grep -Fq '"${alpine_mirror}/v3.22/main"' "$BUILD_PROGRAM" || { echo "COS bootstrap Alpine main repository does not match the pinned image release" >&2 exit 1 } -grep -Fq '"${alpine_mirror}/v3.22/community"' "$INSTALLER" || { +grep -Fq '"${alpine_mirror}/v3.22/community"' "$BUILD_PROGRAM" || { echo "COS bootstrap Alpine community repository does not match the pinned image release" >&2 exit 1 } -# Assert the literal command in the bootstrap script. +# Assert the literal command in the checked-in build program. # shellcheck disable=SC2016 -if ! grep -Fq 'echo "${MAKE_SHA256} /tmp/make.tar.gz" | sha256sum -c -' "$INSTALLER"; then +if ! grep -Fq 'echo "${MAKE_SHA256} /tmp/make.tar.gz" | sha256sum -c -' "$BUILD_PROGRAM"; then echo "COS bootstrap does not verify the GNU Make source archive" >&2 exit 1 fi -checksum_line="$(grep -nF 'sha256sum -c -' "$INSTALLER" | head -n1)" -archive_line="$(grep -nF 'tar -xzf /tmp/make.tar.gz' "$INSTALLER" | head -n1)" +checksum_line="$(grep -nF 'sha256sum -c -' "$BUILD_PROGRAM" | head -n1)" +archive_line="$(grep -nF 'tar -xzf /tmp/make.tar.gz' "$BUILD_PROGRAM" | head -n1)" if (( ${checksum_line%%:*} >= ${archive_line%%:*} )); then echo "COS bootstrap extracts GNU Make before checksum verification" >&2 exit 1 @@ -45,6 +46,22 @@ if grep -Eq '^[[:space:]]*--network[[:space:]]+host([[:space:]\\]|$)' "$INSTALLE echo "COS bootstrap exposes third-party build code to the host network" >&2 exit 1 fi +[[ -x "$BUILD_PROGRAM" ]] || { + echo "COS checked-in Make build program is not executable" >&2 + exit 1 +} +grep -Fq -- '-v "${make_build_program}:/tmp/cloud-compose-build-cos-make.sh:ro"' "$INSTALLER" || { + echo "COS bootstrap does not mount its checked-in Make build program read-only" >&2 + exit 1 +} +grep -Fq '/bin/sh /tmp/cloud-compose-build-cos-make.sh; then' "$INSTALLER" || { + echo "COS bootstrap does not invoke its checked-in Make build program by path" >&2 + exit 1 +} +if grep -Eq '/bin/sh[[:space:]]+-[^[:space:]]*c[[:space:]]' "$INSTALLER"; then + echo "COS bootstrap still embeds a shell program in a command argument" >&2 + exit 1 +fi grep -Fq 'iptables -C DOCKER-USER -d 169.254.169.254/32 -j DROP' "$INSTALLER" || { echo "COS bootstrap does not require the GCP metadata deny policy" >&2 exit 1 @@ -68,24 +85,24 @@ for mirror in \ https://dl-cdn.alpinelinux.org/alpine \ https://mirror.math.princeton.edu/pub/alpinelinux \ https://mirror.fel.cvut.cz/alpine; do - grep -Fq "$mirror" "$INSTALLER" || { + grep -Fq "$mirror" "$BUILD_PROGRAM" || { echo "COS bootstrap is missing Alpine package mirror: $mirror" >&2 exit 1 } done -grep -Fq 'for alpine_mirror in ${alpine_mirrors}; do' "$INSTALLER" || { +grep -Fq 'for alpine_mirror in ${alpine_mirrors}; do' "$BUILD_PROGRAM" || { echo "COS bootstrap does not fail over between Alpine package mirrors" >&2 exit 1 } -grep -Fq 'if apk update && apk add build-base curl make tar; then' "$INSTALLER" || { +grep -Fq 'if apk update && apk add build-base curl make tar; then' "$BUILD_PROGRAM" || { echo "COS bootstrap does not validate an index before installing packages" >&2 exit 1 } -grep -Fq 'All configured Alpine package mirrors failed' "$INSTALLER" || { +grep -Fq 'All configured Alpine package mirrors failed' "$BUILD_PROGRAM" || { echo "COS bootstrap accepts exhaustion of all Alpine package mirrors" >&2 exit 1 } -if grep -Fq -- '--allow-untrusted' "$INSTALLER"; then +if grep -Fq -- '--allow-untrusted' "$BUILD_PROGRAM"; then echo "COS bootstrap disables Alpine package signature verification" >&2 exit 1 fi @@ -97,7 +114,7 @@ retry_until_success() { "$@" } export DOCKER_CONFIG="$MOCK_DOCKER_CONFIG" -install_cos_dependencies "$TEST_CLOUD_HOME" "" "$TEST_DOCKER_BIN" +install_cos_dependencies "$TEST_CLOUD_HOME" "" "$TEST_DOCKER_BIN" "$TEST_BUILD_PROGRAM" EOF cat >"${BIN_DIR}/bash" <<'EOF' @@ -156,6 +173,7 @@ PATH="${BIN_DIR}:$PATH" \ TEST_CLOUD_HOME="$CLOUD_HOME" \ TEST_DATA_DIR="$DATA_DIR" \ TEST_DOCKER_BIN="${BIN_DIR}/docker" \ + TEST_BUILD_PROGRAM="$BUILD_PROGRAM" \ MOCK_DOCKER_CONFIG="$DOCKER_CONFIG_DIR" \ MOCK_CALL_LOG="$CALL_LOG" \ /bin/bash "$HARNESS" @@ -163,6 +181,7 @@ PATH="${BIN_DIR}:$PATH" \ grep -Fq "plugin:${DOCKER_CONFIG_DIR}/cli-plugins:${CLOUD_HOME}/install-docker-plugins.sh" "$CALL_LOG" grep -Fq "plugin:${CLOUD_HOME}/.docker/cli-plugins:${CLOUD_HOME}/install-docker-plugins.sh" "$CALL_LOG" grep -Fq 'alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce' "$CALL_LOG" +grep -Fq "${BUILD_PROGRAM}:/tmp/cloud-compose-build-cos-make.sh:ro" "$CALL_LOG" [[ -x "${DATA_DIR}/make" ]] "${DATA_DIR}/make" --version | grep -Fq 'GNU Make 4.4.1' [[ -f "${DATA_DIR}/make.state" ]] @@ -181,6 +200,7 @@ if PATH="${BIN_DIR}:$PATH" \ TEST_CLOUD_HOME="$CLOUD_HOME" \ TEST_DATA_DIR="$DATA_DIR" \ TEST_DOCKER_BIN="${BIN_DIR}/docker" \ + TEST_BUILD_PROGRAM="$BUILD_PROGRAM" \ MOCK_DOCKER_CONFIG="$DOCKER_CONFIG_DIR" \ MOCK_CALL_LOG="$CALL_LOG" \ /bin/bash "$HARNESS" >/dev/null 2>&1; then @@ -201,6 +221,7 @@ if PATH="${BIN_DIR}:$PATH" \ TEST_CLOUD_HOME="$CLOUD_HOME" \ TEST_DATA_DIR="$DATA_DIR" \ TEST_DOCKER_BIN="${BIN_DIR}/docker" \ + TEST_BUILD_PROGRAM="$BUILD_PROGRAM" \ MOCK_DOCKER_CONFIG="$DOCKER_CONFIG_DIR" \ MOCK_CALL_LOG="$CALL_LOG" \ /bin/bash "$HARNESS" >/dev/null 2>&1; then diff --git a/ci/cos-jq-portability-contract.sh b/ci/cos-jq-portability-contract.sh index f349150..48b07b7 100644 --- a/ci/cos-jq-portability-contract.sh +++ b/ci/cos-jq-portability-contract.sh @@ -5,7 +5,7 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" runtime_program_roots=( "$repo_root/rootfs/home/cloud-compose" - "$repo_root/rootfs/usr/local/share/cloud-compose/jq" + "$repo_root/rootfs/etc/cloud-compose/jq" ) regex_call_pattern='(^|[^[:alnum:]_])(test|match|capture|scan|splits|sub|gsub)[[:space:]]*\(' nul_contains_pattern='contains[[:space:]]*\([[:space:]]*"\\u0000"[[:space:]]*\)' diff --git a/ci/disaster-recovery-contract.sh b/ci/disaster-recovery-contract.sh index bb66236..a6c7d5b 100644 --- a/ci/disaster-recovery-contract.sh +++ b/ci/disaster-recovery-contract.sh @@ -39,7 +39,7 @@ export CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh" export CLOUD_COMPOSE_COMPOSE_APPS_PATH="$tmp/compose-apps.sh" export CLOUD_COMPOSE_DR_LIBRARY_PATH="$repo_root/rootfs/home/cloud-compose/disaster-recovery-lib.sh" export CLOUD_COMPOSE_DR_STATE_ROOT="$tmp/data/dr" -export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/usr/local/share/cloud-compose/jq" +export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" export CLOUD_COMPOSE_DATA_ROOT="$tmp/data" export CLOUD_COMPOSE_VOLUMES_ROOT="$tmp/data/volumes" export MARIADB_BACKUP_ROOT="$tmp/data/backups/mariadb" diff --git a/ci/filesystem-prep-contract.sh b/ci/filesystem-prep-contract.sh index 722747d..a6f49bd 100644 --- a/ci/filesystem-prep-contract.sh +++ b/ci/filesystem-prep-contract.sh @@ -643,11 +643,11 @@ for cloud_init_template in \ "$cloud_init_template" || fail "cloud-init does not verify every required mount before initialization" grep -Fq 'Required cloud-compose mount is unavailable:' "$cloud_init_template" || fail "cloud-init mount gate does not report the unavailable path" - grep -Fq ' bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh' \ + grep -Fq ' bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \ "$cloud_init_template" || fail "cloud-init retryable bootstrap startup is outside the fail-closed mount block" marker_reset_line="$(grep -nF ' rm -f /var/lib/cloud-compose/bootstrap-complete' \ "$cloud_init_template" | cut -d: -f1)" - run_line="$(grep -nF ' bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh' \ + run_line="$(grep -nF ' bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \ "$cloud_init_template" | cut -d: -f1)" [[ -n "$marker_reset_line" && -n "$run_line" && "$marker_reset_line" -lt "$run_line" ]] || fail "cloud-init does not clear stale bootstrap readiness before retryable startup" diff --git a/ci/gcp-upgrade-smoke-contract.sh b/ci/gcp-upgrade-smoke-contract.sh index 3c887d7..1fcc795 100755 --- a/ci/gcp-upgrade-smoke-contract.sh +++ b/ci/gcp-upgrade-smoke-contract.sh @@ -47,7 +47,7 @@ grep -Fq "provider_tag_cleanup gcp-wp \"\$run_id\"" "$script" || fail "upgrade runner does not finish cleanup with the verified provider sweep" grep -Fq 'target_env gcp-wp' "$script" || fail "upgrade cleanup does not load the concrete GCP WordPress target environment" -grep -Fq 'readonly diagnostics_program="/usr/local/sbin/cloud-compose-diagnostics.sh"' "$shared_smoke" || +grep -Fq 'readonly diagnostics_program="/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"' "$shared_smoke" || fail "shared smoke diagnostics do not use the checked-in privileged program" for diagnostics_command in state status dump; do grep -Fq "sudo -n \${diagnostics_program} ${diagnostics_command}" "$shared_smoke" || @@ -175,7 +175,7 @@ baseline_initcmd_line="$(grep -nF 'for CMD in ADDITIONAL_INITCMD' "$tmp/baseline fail "baseline cloud-init does not execute fixture initcmd before run.sh" current_bootstrap_line="$( - grep -nF 'bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh' \ + grep -nF 'bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \ "$repo_root/templates/cloud-init.yml" | cut -d: -f1 || true )" diff --git a/ci/host-runtime-security.sh b/ci/host-runtime-security.sh index 8079f5e..859b16f 100644 --- a/ci/host-runtime-security.sh +++ b/ci/host-runtime-security.sh @@ -177,7 +177,7 @@ fi assert_contains "$host_init" 'chown root:root /home/cloud-compose' assert_contains "$host_init" "-exec chown root:root {} +" assert_contains "$host_init" 'Unsafe Cloud Compose lifecycle dispatcher:' -assert_contains "$repo_root/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh" \ +assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh" \ 'Cloud Compose control input is not root-controlled' assert_contains "$host_init" '/home/cloud-compose/.sitectl \' assert_contains "$host_init" 'install -d -m 0750 -o cloud-compose -g cloud-compose "$mutable_dir"' @@ -243,12 +243,39 @@ if grep -Fq 'rotate-keys' "$repo_root/rootfs/home/cloud-compose/docker-prune.sh" fail "provider-neutral Docker prune still invokes GCP key rotation" fi -archive_source="$repo_root/modules/linux-vm-runtime/main.tf" -verify_line="$(grep -n 'sha256sum -c -' "$archive_source" | head -n 1 | cut -d: -f1)" -extract_line="$(grep -n 'tar --no-same-owner -xzf "\$tmp/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" -[[ -n "$verify_line" && -n "$extract_line" && "$verify_line" -lt "$extract_line" ]] || \ - fail "rootfs archive is not verified before ownership-safe extraction" -assert_contains "$repo_root/modules/gcp/main.tf" 'tar --no-same-owner -xzf "$tmp/rootfs.tar.gz"' +for archive_source in \ + "$repo_root/modules/linux-vm-runtime/main.tf" \ + "$repo_root/modules/gcp/main.tf"; do + verify_line="$(grep -n 'sha256sum -c -' "$archive_source" | head -n 1 | cut -d: -f1)" + extract_line="$(grep -n 'tar --no-same-owner -xzf "\$tmp/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" + normalize_line="$(grep -n 'chown -hR 0:0 -- "\$rootfs_dir"' "$archive_source" | head -n 1 | cut -d: -f1)" + copy_line="$(grep -n 'cp -a "\$rootfs_dir"/. /' "$archive_source" | head -n 1 | cut -d: -f1)" + [[ -n "$verify_line" && -n "$extract_line" && -n "$normalize_line" && -n "$copy_line" && + "$verify_line" -lt "$extract_line" && "$extract_line" -lt "$normalize_line" && + "$normalize_line" -lt "$copy_line" ]] || \ + fail "$archive_source does not verify, ownership-safe extract, then normalize the rootfs to root" +done + +if [[ -n "$(git -C "$repo_root" ls-files 'rootfs/usr/**')" ]]; then + fail "Cloud Compose-owned rootfs programs still target immutable /usr" +fi +for trusted_program in \ + rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh \ + rootfs/etc/cloud-compose/libexec/build-cos-make.sh \ + rootfs/etc/cloud-compose/libexec/bootstrap-security.sh \ + rootfs/etc/cloud-compose/libexec/run-bootstrap.sh \ + rootfs/etc/cloud-compose/libexec/run-root-program.sh \ + rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq; do + [[ -f "$repo_root/$trusted_program" ]] || fail "COS-safe trusted program is missing: $trusted_program" +done +for cloud_init_template in \ + "$repo_root/templates/cloud-init.yml" \ + "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do + assert_contains "$cloud_init_template" 'install -d -m 0755 -o root -g root /etc/cloud-compose/bin' + if grep -Eq '^[[:space:]]*- path: /usr/|install -d[^#]* /usr/local' "$cloud_init_template"; then + fail "$cloud_init_template writes Cloud Compose-owned programs beneath immutable /usr" + fi +done for variables_file in \ "$repo_root/variables.tf" \ diff --git a/ci/rootfs-package-contract.sh b/ci/rootfs-package-contract.sh index 47f1737..d4612c6 100755 --- a/ci/rootfs-package-contract.sh +++ b/ci/rootfs-package-contract.sh @@ -21,10 +21,18 @@ tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | awk ' ' tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/home/cloud-compose/run.sh' >/dev/null tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ - grep -Fx 'rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh' >/dev/null + grep -Fx 'rootfs/etc/cloud-compose/libexec/run-bootstrap.sh' >/dev/null tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ - grep -Fx 'rootfs/usr/local/libexec/cloud-compose/run-root-program.sh' >/dev/null + grep -Fx 'rootfs/etc/cloud-compose/libexec/run-root-program.sh' >/dev/null tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ - grep -Fx 'rootfs/usr/local/share/cloud-compose/jq/offhost-validate-manifest.jq' >/dev/null + grep -Fx 'rootfs/etc/cloud-compose/libexec/build-cos-make.sh' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ + grep -Fx 'rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ + grep -Fx 'rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh' >/dev/null +if tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -E '^rootfs/usr/' >/dev/null; then + echo "Rootfs package contains a Cloud Compose-owned immutable /usr path" >&2 + exit 1 +fi echo "Rootfs package contract passed" diff --git a/ci/systemd-contract.sh b/ci/systemd-contract.sh index 9ce70de..aa6f000 100755 --- a/ci/systemd-contract.sh +++ b/ci/systemd-contract.sh @@ -4,10 +4,10 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" unit_dir="$repo_root/rootfs/etc/systemd/system" -diagnostics_program="$repo_root/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh" +diagnostics_program="$repo_root/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh" smoke_healthcheck_program="$repo_root/rootfs/home/cloud-compose/smoke-healthcheck.sh" -bootstrap_security="$repo_root/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh" -root_program_runner="$repo_root/rootfs/usr/local/libexec/cloud-compose/run-root-program.sh" +bootstrap_security="$repo_root/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh" +root_program_runner="$repo_root/rootfs/etc/cloud-compose/libexec/run-root-program.sh" fail() { echo "systemd contract: $*" >&2 @@ -48,8 +48,8 @@ assert_contains "$unit_dir/cloud-compose.service" 'RestartSec=30s' if grep -Fq 'ConditionPathExists=' "$unit_dir/cloud-compose-bootstrap.service"; then fail "bootstrap still trusts an unvalidated marker path condition" fi -assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecCondition=/bin/bash /usr/local/libexec/cloud-compose/bootstrap-required.sh' -assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-bootstrap.sh' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecCondition=/bin/bash /etc/cloud-compose/libexec/bootstrap-required.sh' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecStart=/bin/bash /etc/cloud-compose/libexec/run-bootstrap.sh' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'Restart=on-failure' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'RestartSec=30s' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'TimeoutStartSec=2h' @@ -73,8 +73,8 @@ assert_contains "$bootstrap_security" '"$payload" == "ready"' assert_contains "$bootstrap_security" 'cloud_compose_secure_runtime_home()' assert_contains "$root_program_runner" 'cloud_compose_secure_runtime_home' assert_contains "$root_program_runner" 'Unsupported Cloud Compose root program:' -[[ -f "$repo_root/rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh" ]] || \ - fail "validated bootstrap readiness gate is missing" +[[ -x "$repo_root/rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh" ]] || \ + fail "validated bootstrap readiness gate is missing or not executable" assert_contains "$diagnostics_program" '--- Cloud Compose provisioning heartbeat ---' assert_contains "$diagnostics_program" 'ps -p "$main_pid" -o pid=,ppid=,stat=,etime=,comm=' assert_contains "$smoke_healthcheck_program" 'source /home/cloud-compose/profile.sh' @@ -116,9 +116,9 @@ assert_contains "$docker_metadata_dropin" 'Requires=cloud-compose-metadata-firew assert_contains "$docker_metadata_dropin" 'After=cloud-compose-metadata-firewall-pre.service' assert_contains "$metadata_pre_unit" 'Before=docker.service' assert_contains "$metadata_pre_unit" 'WantedBy=multi-user.target' -assert_contains "$metadata_pre_unit" 'ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh configure-metadata-firewall.sh pre-docker' +assert_contains "$metadata_pre_unit" 'ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh configure-metadata-firewall.sh pre-docker' assert_contains "$unit_dir/cloud-compose-overlay.service" 'Before=docker.service cloud-compose.service' -assert_contains "$unit_dir/cloud-compose-overlay.service" 'ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh mount-overlays.sh' +assert_contains "$unit_dir/cloud-compose-overlay.service" 'ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh mount-overlays.sh' for root_home_unit in \ cloud-compose-docker-prune.service \ cloud-compose-key-rotation.service \ @@ -132,7 +132,7 @@ for root_home_unit in \ if grep -Eq '^Exec(Start|StartPre|StartPost|Stop|StopPost)=/bin/bash /home/cloud-compose/' "$unit_dir/$root_home_unit"; then fail "$root_home_unit executes historically writable home code without the root-owned launcher" fi - assert_contains "$unit_dir/$root_home_unit" '/usr/local/libexec/cloud-compose/run-root-program.sh' + assert_contains "$unit_dir/$root_home_unit" '/etc/cloud-compose/libexec/run-root-program.sh' done if grep -Eq '^(After|Before|BindsTo|PartOf|Requires|Requisite|Wants)=.*cloud-compose-bootstrap\\.service' \ "$unit_dir/cloud-compose.service"; then diff --git a/ci/vault-runtime-contract.sh b/ci/vault-runtime-contract.sh index 1d96543..901d776 100644 --- a/ci/vault-runtime-contract.sh +++ b/ci/vault-runtime-contract.sh @@ -181,9 +181,9 @@ grep -Fq 'ExecStartPre=/bin/bash /home/cloud-compose/assert-vault-ready.sh' \ "$repo_root/rootfs/etc/systemd/system/cloud-compose.service" || fail "app service lacks a Vault readiness gate" vault_unit="$repo_root/rootfs/etc/systemd/system/cloud-compose-vault-agent.service" for readiness_command in \ - 'ExecStartPre=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh prepare' \ - 'ExecStartPost=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh wait' \ - 'ExecStopPost=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh clear'; do + 'ExecStartPre=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh prepare' \ + 'ExecStartPost=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh wait' \ + 'ExecStopPost=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh clear'; do grep -Fq "$readiness_command" "$vault_unit" || \ fail "Vault unit does not route readiness through the trusted root launcher: $readiness_command" done diff --git a/docs/disaster-recovery.md b/docs/disaster-recovery.md index 5ddcf3f..618201e 100644 --- a/docs/disaster-recovery.md +++ b/docs/disaster-recovery.md @@ -16,13 +16,19 @@ root, bind mount, named volume, and service-mount topology in the manifest. Install the reviewed driver and all of its configuration out of band. The file and every directory in its path must be root-owned, must not be a symlink or be group/world writable, and the executable must have exactly one hard link. +`/etc/cloud-compose/libexec/offhost-backup-driver` is the portable default on +COS and conventional Linux hosts. An explicit safe absolute path remains +supported when an operator manages the driver elsewhere. Because COS rebuilds +`/etc` at boot, provide the driver through the configured rootfs overlay or an +equivalent startup provisioner there; a one-time manual copy will not survive a +reboot. Keep credentials outside that overlay. Then enable the provider-neutral runtime input: ```hcl runtime = { disaster_recovery = { required = true - driver_path = "/usr/local/libexec/cloud-compose/offhost-backup-driver" + driver_path = "/etc/cloud-compose/libexec/offhost-backup-driver" } } ``` diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 42eef98..066f9c2 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -156,10 +156,13 @@ with `systemctl status cloud-compose-bootstrap cloud-compose`. Bootstrap output uses a fixed `info` priority and has no unit-specific Fluent Bit input, so raw bootstrap output is not forwarded to Cloud Logging; systemd's own service failures remain available to the existing warning-level collector. -Root systemd jobs enter through `/usr/local/libexec/cloud-compose`, validate -the root-owned home scripts and control inputs, and only then execute their -allowlisted `/home/cloud-compose` program. Application services retain their -unprivileged execution model. +Root systemd jobs enter through `/etc/cloud-compose/libexec`, validate the +root-owned home scripts and control inputs, and only then execute their +allowlisted `/home/cloud-compose` program. The privileged entrypoints, +diagnostics command, and checked-in `jq` programs live below +`/etc/cloud-compose/{libexec,bin,jq}` because COS permits cloud-init to rebuild +that stateless tree while its `/usr` filesystem is immutable. Application +services retain their unprivileged execution model. ## Sitectl @@ -336,7 +339,9 @@ release versions and the Alpine tag/digest pair; review those changes as supply-chain updates rather than accepting an unpinned replacement. The image and its network-fetched package build scripts execute only after the metadata firewall is installed and use the bridge network, so they cannot inherit the -host network's root exemption. +host network's root exemption. The build itself is a checked-in shell program +mounted read-only into the container and invoked by path, keeping the audited +program out of cloud-init and Docker command arguments. The GCP COS VM image name is a reviewed manual pin. Renovate has no built-in GCP Compute image-family datasource, and the shared LibOps preset does not add @@ -662,10 +667,12 @@ used, it must be an HTTPS URL without whitespace and `runtime.rootfs_archive_sha256` is mandatory with a 64-character SHA-256 digest. Boot restricts curl and redirects to HTTPS with TLS 1.2 or newer, downloads to a temporary path, and verifies the complete file before extracting -its `rootfs` directory. The GCP path stages -the caller's packaged rootfs overlay and reapplies it after the archive, so -consumer overrides still win. Use an immutable archive URL; a moving branch and -a pinned checksum intentionally fail as soon as the branch content changes. +its `rootfs` directory. Both archive installers normalize the verified tree to +UID/GID `0:0` before loading a helper or copying it onto the host, so Git forge +source-archive ownership cannot leak into the privileged runtime. The GCP path +stages the caller's packaged rootfs overlay and reapplies it after the archive, +so consumer overrides still win. Use an immutable archive URL; a moving branch +and a pinned checksum intentionally fail as soon as the branch content changes. ## Backups @@ -860,7 +867,7 @@ deploy the same the legacy runtime's branch-only clone behavior without following a moving branch. Its `gcp.cloud_init.initcmd` disables both generations of the internal-service timer after cloud-init writes the units but before the -root-owned `/usr/local/libexec/cloud-compose/run-bootstrap.sh` entrypoint +root-owned `/etc/cloud-compose/libexec/run-bootstrap.sh` entrypoint starts the potentially long bootstrap, so the disposable VM cannot suspend itself. The runner checks the units again after each boot. Only the ephemeral runner key is authorized, and SSH is limited to that diff --git a/internal/contracttest/disaster_recovery_test.go b/internal/contracttest/disaster_recovery_test.go index 083d853..61d6931 100644 --- a/internal/contracttest/disaster_recovery_test.go +++ b/internal/contracttest/disaster_recovery_test.go @@ -19,7 +19,7 @@ func TestDisasterRecoveryInputParity(t *testing.T) { content := readRepositoryFile(t, root, relativePath) requireContains(t, content, "disaster_recovery = optional(object({", relativePath+" disaster-recovery object") requireContains(t, content, "required = optional(bool, false)", relativePath+" required switch") - requireContains(t, content, `driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver")`, relativePath+" driver default") + requireContains(t, content, `driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")`, relativePath+" driver default") requireContains(t, content, "runtime.disaster_recovery.driver_path must be a safe absolute path", relativePath+" path validation") } @@ -53,12 +53,12 @@ func TestDisasterRecoveryReceiptAndRestoreContracts(t *testing.T) { restore := readRepositoryFile(t, root, "rootfs/home/cloud-compose/restore-test.sh") validationContract := strings.Join([]string{ library, - readRepositoryFile(t, root, "rootfs/usr/local/share/cloud-compose/jq/dr-validate-backup-receipt.jq"), - readRepositoryFile(t, root, "rootfs/usr/local/share/cloud-compose/jq/dr-validate-restore-proof.jq"), + readRepositoryFile(t, root, "rootfs/etc/cloud-compose/jq/dr-validate-backup-receipt.jq"), + readRepositoryFile(t, root, "rootfs/etc/cloud-compose/jq/dr-validate-restore-proof.jq"), }, "\n") backupContract := strings.Join([]string{ backup, - readRepositoryFile(t, root, "rootfs/usr/local/share/cloud-compose/jq/offhost-build-application-coverage.jq"), + readRepositoryFile(t, root, "rootfs/etc/cloud-compose/jq/offhost-build-application-coverage.jq"), }, "\n") for marker, label := range map[string]string{ diff --git a/modules/digitalocean/variables.tf b/modules/digitalocean/variables.tf index eef7bb9..26a6104 100644 --- a/modules/digitalocean/variables.tf +++ b/modules/digitalocean/variables.tf @@ -78,7 +78,7 @@ variable "runtime" { disaster_recovery = optional(object({ required = optional(bool, false) - driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver") }), {}) compose = optional(object({ diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index b5b2035..8aa3c14 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -232,11 +232,15 @@ locals { } } ) + rootfs_file_permissions = { + for file in setunion(local.base_files, local.additional_files) : + file => endswith(file, ".sh") || "/${file}" == var.offhost_backup_driver_path ? "0755" : "0644" + } write_files_content = join("\n", [ for file, config in local.all_files : <<-EOT - path: ${jsonencode(config.destination)} - permissions: ${jsonencode(endswith(file, ".sh") ? "0755" : "0644")} + permissions: ${jsonencode(local.rootfs_file_permissions[file])} encoding: gzip+base64 content: ${jsonencode(base64gzip(file(config.source)))} EOT @@ -447,10 +451,11 @@ rootfs_archive_command_raw = <<-EOT printf '%s %s\n' "$archive_sha256" "$tmp/rootfs.tar.gz" | sha256sum -c - tar --no-same-owner -xzf "$tmp/rootfs.tar.gz" -C "$tmp" rootfs_dir="$(find "$tmp" -mindepth 1 -maxdepth 3 -type d -name rootfs -print -quit)" - if [ -z "$rootfs_dir" ]; then + if [ -z "$rootfs_dir" ] || [ ! -d "$rootfs_dir" ]; then echo "rootfs directory not found in $archive_url" >&2 exit 1 fi + chown -hR 0:0 -- "$rootfs_dir" cp -a "$rootfs_dir"/. / if [ -d "$overlay_dir" ]; then cp -a "$overlay_dir"/. / @@ -464,7 +469,7 @@ rollout_runcmd = var.rollout_enabled ? [ "bash /home/cloud-compose/deploy-rollout.sh >> /home/cloud-compose/run.log 2>&1", ] : [] cloud_init_yaml = templatefile("${path.module}/../../templates/cloud-init.yml", { - DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootFs}/usr/local/sbin/cloud-compose-diagnostics.sh"), + DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"), FILESYSTEM_PREP_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/prepare-filesystem.sh"), FILESYSTEM_PERSIST_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/persist-filesystems.sh"), FRESH_FILESYSTEM_IDENTITY = "v1:gcp-disk-id:${google_compute_disk.data.disk_id}", diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index 38166be..0c5fd62 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -535,13 +535,14 @@ run "renders_verified_archive_before_downstream_overlay" { command = plan variables { - name = "gcp-contract" - project_id = "test-project" - project_number = "123456789" - docker_compose_repo = "https://github.com/libops/wp.git" - rootfs = "testdata/rootfs" - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz?literal=$(id)" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + name = "gcp-contract" + project_id = "test-project" + project_number = "123456789" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs = "testdata/rootfs" + rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz?literal=$(id)" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + offhost_backup_driver_path = "/etc/cloud-compose/libexec/custom-offhost-driver" extra_env = { NGINX_CLIENT_MAX_BODY_SIZE = "512m" PHP_UPLOAD_MAX_FILESIZE = "512M" @@ -561,10 +562,10 @@ run "renders_verified_archive_before_downstream_overlay" { assert { condition = can(regex( - "(?s)cp -a \\\"\\$rootfs_dir\\\"/\\. /.*cp -a \\\"\\$overlay_dir\\\"/\\. /", + "(?s)sha256sum -c -.*tar --no-same-owner -xzf \\\"\\$tmp/rootfs\\.tar\\.gz\\\".*chown -hR 0:0 -- \\\"\\$rootfs_dir\\\".*cp -a \\\"\\$rootfs_dir\\\"/\\. /.*cp -a \\\"\\$overlay_dir\\\"/\\. /", local.cloud_init_yaml, )) - error_message = "The consumer rootfs overlay must be applied after the verified base archive." + error_message = "The verified base archive must be extracted ownership-safely, normalized to root, and installed before the consumer rootfs overlay." } assert { @@ -581,6 +582,51 @@ run "renders_verified_archive_before_downstream_overlay" { ) error_message = "The downstream overlay and isolated application environment data must be present in cloud-init." } + + assert { + condition = ( + local.rootfs_file_permissions["etc/cloud-compose/libexec/custom-offhost-driver"] == "0755" && + local.rootfs_file_permissions["etc/cloud-compose-overlay-marker"] == "0644" && + strcontains( + local.write_files_content, + "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose/libexec/custom-offhost-driver\"\n permissions: \"0755\"", + ) && + strcontains( + local.write_files_content, + "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose-overlay-marker\"\n permissions: \"0644\"", + ) + ) + error_message = "Archive-backed GCP overlays must make only the configured off-host backup driver executable." + } +} + +run "renders_embedded_offhost_backup_driver_executable" { + command = plan + + variables { + name = "gcp-embedded-driver-contract" + project_id = "test-project" + project_number = "123456789" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs = "testdata/rootfs" + offhost_backup_driver_path = "/etc/cloud-compose/libexec/custom-offhost-driver" + } + + assert { + condition = ( + local.rootfs_file_permissions["etc/cloud-compose/libexec/custom-offhost-driver"] == "0755" && + local.rootfs_file_permissions["etc/cloud-compose-overlay-marker"] == "0644" && + strcontains( + local.write_files_content, + "- path: \"/etc/cloud-compose/libexec/custom-offhost-driver\"\n permissions: \"0755\"", + ) && + strcontains( + local.write_files_content, + "- path: \"/etc/cloud-compose-overlay-marker\"\n permissions: \"0644\"", + ) + ) + error_message = "Embedded GCP overlays must make only the configured off-host backup driver executable." + } } run "namespaces_rollout_auth_away_from_application_environment" { diff --git a/modules/gcp/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver b/modules/gcp/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver new file mode 100644 index 0000000..df09e99 --- /dev/null +++ b/modules/gcp/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver @@ -0,0 +1 @@ +fixture driver payload diff --git a/modules/gcp/variables.tf b/modules/gcp/variables.tf index f843c46..35aca7f 100644 --- a/modules/gcp/variables.tf +++ b/modules/gcp/variables.tf @@ -616,7 +616,7 @@ variable "offhost_backup_required" { variable "offhost_backup_driver_path" { type = string - default = "/usr/local/libexec/cloud-compose/offhost-backup-driver" + default = "/etc/cloud-compose/libexec/offhost-backup-driver" description = "Absolute path to the operator-supplied, root-owned provider-neutral DR driver. The driver owns its credentials; do not pass them through Terraform." validation { diff --git a/modules/linode/variables.tf b/modules/linode/variables.tf index 45cec74..2ce975e 100644 --- a/modules/linode/variables.tf +++ b/modules/linode/variables.tf @@ -92,7 +92,7 @@ variable "runtime" { disaster_recovery = optional(object({ required = optional(bool, false) - driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver") }), {}) compose = optional(object({ diff --git a/modules/linux-vm-runtime/disaster_recovery.tftest.hcl b/modules/linux-vm-runtime/disaster_recovery.tftest.hcl index d4a3414..495c3fc 100644 --- a/modules/linux-vm-runtime/disaster_recovery.tftest.hcl +++ b/modules/linux-vm-runtime/disaster_recovery.tftest.hcl @@ -9,13 +9,13 @@ run "renders_provider_neutral_disaster_recovery_controls" { volumes_device = "/dev/test-volumes" docker_compose_repo = "https://github.com/libops/wp.git" offhost_backup_required = true - offhost_backup_driver_path = "/usr/local/libexec/cloud-compose/acme-offhost" + offhost_backup_driver_path = "/etc/cloud-compose/libexec/acme-offhost" } assert { condition = ( local.host_env.CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED == "true" && - local.host_env.CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER == "/usr/local/libexec/cloud-compose/acme-offhost" + local.host_env.CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER == "/etc/cloud-compose/libexec/acme-offhost" ) error_message = "The Linux VM runtime must render only the required switch and operator-owned driver path." } @@ -41,7 +41,7 @@ run "rejects_unsafe_disaster_recovery_driver_path" { volumes_device = "/dev/test-volumes" docker_compose_repo = "https://github.com/libops/wp.git" offhost_backup_required = true - offhost_backup_driver_path = "/usr/local/libexec/cloud-compose/../untrusted" + offhost_backup_driver_path = "/etc/cloud-compose/libexec/../untrusted" } expect_failures = [var.offhost_backup_driver_path] diff --git a/modules/linux-vm-runtime/main.tf b/modules/linux-vm-runtime/main.tf index 8e89792..047a98b 100644 --- a/modules/linux-vm-runtime/main.tf +++ b/modules/linux-vm-runtime/main.tf @@ -89,20 +89,24 @@ locals { { for file in local.base_files : file => "${local.rootfs}/${file}" }, { for file in local.embedded_additional_files : file => "${local.additional_rootfs}/${file}" } ) + rootfs_file_permissions = { + for file in setunion(local.base_files, local.additional_files) : + file => endswith(file, ".sh") || "/${file}" == var.offhost_backup_driver_path ? "0755" : "0644" + } archive_additional_rootfs_commands = join("\n", [ for file in local.additional_files : <<-EOT destination="$(printf '%s' '${base64encode("/${file}")}' | base64 -d)" install -d "$(dirname "$destination")" printf '%s' '${filebase64("${local.additional_rootfs}/${file}")}' | base64 -d >"$destination" - chmod ${endswith(file, ".sh") ? "0755" : "0644"} "$destination" + chmod ${local.rootfs_file_permissions[file]} "$destination" EOT ]) write_files_content = join("\n", [ for file, fullpath in local.all_files : <<-EOT - path: ${jsonencode(startswith(file, "mnt/disks/") ? "/var/lib/cloud-compose/mounted-rootfs/${file}" : "/${file}")} - permissions: ${jsonencode(endswith(file, ".sh") ? "0755" : "0644")} + permissions: ${jsonencode(local.rootfs_file_permissions[file])} encoding: gzip+base64 content: ${jsonencode(base64gzip(file(fullpath)))} EOT @@ -309,6 +313,7 @@ rootfs_archive_prepare_command_raw = <<-EOT echo "rootfs directory not found in verified archive $archive_url" >&2 exit 1 fi + chown -hR 0:0 -- "$rootfs_dir" filesystem_prep_source="$rootfs_dir/home/cloud-compose/prepare-filesystem.sh" filesystem_persist_source="$rootfs_dir/home/cloud-compose/persist-filesystems.sh" if [ ! -f "$filesystem_prep_source" ] || [ ! -f "$filesystem_persist_source" ]; then @@ -333,7 +338,7 @@ rootfs_archive_install_command = local.rootfs_archive_url != "" ? local.rootfs_a cloud_init = templatefile("${path.module}/templates/cloud-init.yml", { CLOUD_COMPOSE_SSH_KEYS = var.cloud_compose_ssh_keys SSH_USERS = var.ssh_users - DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootfs}/usr/local/sbin/cloud-compose-diagnostics.sh") + DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootfs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh") DATA_DEVICE = var.data_device VOLUMES_DEVICE = var.volumes_device WRITE_FILES_CONTENT = local.write_files_content diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index 78bc2a1..3a5f841 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -29,9 +29,9 @@ run "renders_safe_ssh_values" { assert { condition = ( strcontains(output.cloud_init, "path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh") && - strcontains(output.cloud_init, "NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh state") && - strcontains(output.cloud_init, "NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh status") && - strcontains(output.cloud_init, "NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh dump") && + strcontains(output.cloud_init, "NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state") && + strcontains(output.cloud_init, "NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status") && + strcontains(output.cloud_init, "NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump") && strcontains(output.cloud_init, "install -m 0755 -o root -g root") ) error_message = "Cloud-init must install one root-owned diagnostics program with exact passwordless sudo commands." @@ -207,14 +207,16 @@ run "renders_verified_rootfs_archive" { command = plan variables { - name = "contract-test" - provider_name = "linode" - region = "us-east" - data_device = "/dev/test-data" - volumes_device = "/dev/test-volumes" - docker_compose_repo = "https://github.com/libops/wp.git" - rootfs_archive_url = "https://example.invalid/cloud-compose-$(id).tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + name = "contract-test" + provider_name = "linode" + region = "us-east" + data_device = "/dev/test-data" + volumes_device = "/dev/test-volumes" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs = "testdata/rootfs" + rootfs_archive_url = "https://example.invalid/cloud-compose-$(id).tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + offhost_backup_driver_path = "/etc/cloud-compose/libexec/custom-offhost-driver" } assert { @@ -225,16 +227,18 @@ run "renders_verified_rootfs_archive" { assert { condition = ( strcontains(output.cloud_init, "tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"") && + strcontains(output.cloud_init, "chown -hR 0:0 -- \"$rootfs_dir\"") && strcontains(output.cloud_init, "install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"") && strcontains(output.cloud_init, "bash \"$filesystem_prep\"") && strcontains(output.cloud_init, "bash \"$filesystem_persist\"") && strcontains(output.cloud_init, "cp -a \"$rootfs_dir\"/. /") && length(split("sha256sum -c -", output.cloud_init)[0]) < length(split("tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) && - length(split("tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) < length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) && + length(split("tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) < length(split("chown -hR 0:0 -- \"$rootfs_dir\"", output.cloud_init)[0]) && + length(split("chown -hR 0:0 -- \"$rootfs_dir\"", output.cloud_init)[0]) < length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) && length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) < length(split("bash \"$filesystem_prep\"", output.cloud_init)[0]) && length(split("bash \"$filesystem_persist\"", output.cloud_init)[0]) < length(split("cp -a \"$rootfs_dir\"/. /", output.cloud_init)[0]) ) - error_message = "Archive cloud-init must verify and extract before loading its helpers, then install the rootfs only after filesystem preparation and persistence." + error_message = "Archive cloud-init must verify, extract, and normalize root ownership before loading its helpers, then install the rootfs only after filesystem preparation and persistence." } assert { @@ -260,18 +264,36 @@ run "renders_verified_rootfs_archive" { ) error_message = "Archive URLs must be rendered as base64 data rather than executable shell syntax." } + + assert { + condition = ( + local.rootfs_file_permissions["etc/cloud-compose/libexec/custom-offhost-driver"] == "0755" && + local.rootfs_file_permissions["etc/cloud-compose/unrelated-config"] == "0644" && + strcontains( + split(base64encode("/etc/cloud-compose/libexec/custom-offhost-driver"), local.archive_additional_rootfs_commands)[1], + "chmod 0755 \"$destination\"", + ) && + strcontains( + split(base64encode("/etc/cloud-compose/unrelated-config"), local.archive_additional_rootfs_commands)[1], + "chmod 0644 \"$destination\"", + ) + ) + error_message = "Archive-backed Linux VM overlays must make only the configured off-host backup driver executable." + } } run "embeds_filesystem_helpers_without_archive" { command = plan variables { - name = "contract-test" - provider_name = "digitalocean" - region = "nyc3" - data_device = "/dev/test-data" - volumes_device = "/dev/test-volumes" - docker_compose_repo = "https://github.com/libops/wp.git" + name = "contract-test" + provider_name = "digitalocean" + region = "nyc3" + data_device = "/dev/test-data" + volumes_device = "/dev/test-volumes" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs = "testdata/rootfs" + offhost_backup_driver_path = "/etc/cloud-compose/libexec/custom-offhost-driver" } assert { @@ -282,6 +304,22 @@ run "embeds_filesystem_helpers_without_archive" { ) error_message = "Inline cloud-init must retain the embedded filesystem-helper bootstrap when no archive is configured." } + + assert { + condition = ( + local.rootfs_file_permissions["etc/cloud-compose/libexec/custom-offhost-driver"] == "0755" && + local.rootfs_file_permissions["etc/cloud-compose/unrelated-config"] == "0644" && + strcontains( + local.write_files_content, + "- path: \"/etc/cloud-compose/libexec/custom-offhost-driver\"\n permissions: \"0755\"", + ) && + strcontains( + local.write_files_content, + "- path: \"/etc/cloud-compose/unrelated-config\"\n permissions: \"0644\"", + ) + ) + error_message = "Embedded Linux VM overlays must make only the configured off-host backup driver executable." + } } run "rejects_rootfs_archive_without_checksum" { diff --git a/modules/linux-vm-runtime/templates/cloud-init.yml b/modules/linux-vm-runtime/templates/cloud-init.yml index 7a46576..d35c7e8 100644 --- a/modules/linux-vm-runtime/templates/cloud-init.yml +++ b/modules/linux-vm-runtime/templates/cloud-init.yml @@ -10,9 +10,9 @@ users: - docker sudo: - ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service - - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh state - - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh status - - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh dump + - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state + - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status + - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump %{ if length(CLOUD_COMPOSE_SSH_KEYS) > 0 ~} ssh_authorized_keys: %{ for key in CLOUD_COMPOSE_SSH_KEYS ~} @@ -65,9 +65,9 @@ runcmd: install -d -m 0755 /etc/sudoers.d printf '%s\n' \ 'cloud-compose ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service' \ - 'cloud-compose ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh state' \ - 'cloud-compose ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh status' \ - 'cloud-compose ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh dump' \ + 'cloud-compose ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state' \ + 'cloud-compose ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status' \ + 'cloud-compose ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump' \ >/etc/sudoers.d/90-cloud-compose chmod 0440 /etc/sudoers.d/90-cloud-compose } @@ -103,10 +103,10 @@ runcmd: cp -a /var/lib/cloud-compose/mounted-rootfs/mnt/disks/. /mnt/disks/ fi %{ endif } - install -d -m 0755 -o root -g root /usr/local/sbin + install -d -m 0755 -o root -g root /etc/cloud-compose/bin install -m 0755 -o root -g root \ /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh \ - /usr/local/sbin/cloud-compose-diagnostics.sh + /etc/cloud-compose/bin/cloud-compose-diagnostics.sh chown root:cloud-compose /mnt/disks/data chmod 1775 /mnt/disks/data chown cloud-compose:cloud-compose /mnt/disks/volumes @@ -116,4 +116,4 @@ runcmd: ${ROLLOUT_RUNCMD} %{ endif ~} rm -f /var/lib/cloud-compose/bootstrap-complete - bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh + bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh diff --git a/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver b/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver new file mode 100644 index 0000000..df09e99 --- /dev/null +++ b/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver @@ -0,0 +1 @@ +fixture driver payload diff --git a/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/unrelated-config b/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/unrelated-config new file mode 100644 index 0000000..578c4da --- /dev/null +++ b/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/unrelated-config @@ -0,0 +1 @@ +fixture config payload diff --git a/modules/linux-vm-runtime/variables.tf b/modules/linux-vm-runtime/variables.tf index 2d8d0f9..5bf5f85 100644 --- a/modules/linux-vm-runtime/variables.tf +++ b/modules/linux-vm-runtime/variables.tf @@ -104,7 +104,7 @@ variable "offhost_backup_required" { variable "offhost_backup_driver_path" { type = string - default = "/usr/local/libexec/cloud-compose/offhost-backup-driver" + default = "/etc/cloud-compose/libexec/offhost-backup-driver" description = "Absolute path to the operator-supplied, root-owned provider-neutral DR driver. The driver owns its credentials; do not pass them through Terraform." validation { diff --git a/providers/do/variables.tf b/providers/do/variables.tf index aad28f9..eab146b 100644 --- a/providers/do/variables.tf +++ b/providers/do/variables.tf @@ -89,7 +89,7 @@ variable "runtime" { disaster_recovery = optional(object({ required = optional(bool, false) - driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver") }), {}) compose = optional(object({ diff --git a/providers/gcp/variables.tf b/providers/gcp/variables.tf index 80a8a4a..df1ef48 100644 --- a/providers/gcp/variables.tf +++ b/providers/gcp/variables.tf @@ -201,7 +201,7 @@ variable "runtime" { disaster_recovery = optional(object({ required = optional(bool, false) - driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver") }), {}) compose = optional(object({ diff --git a/providers/linode/variables.tf b/providers/linode/variables.tf index 4b29dc0..8ab7190 100644 --- a/providers/linode/variables.tf +++ b/providers/linode/variables.tf @@ -104,7 +104,7 @@ variable "runtime" { disaster_recovery = optional(object({ required = optional(bool, false) - driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver") }), {}) compose = optional(object({ diff --git a/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh b/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh similarity index 98% rename from rootfs/usr/local/sbin/cloud-compose-diagnostics.sh rename to rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh index 7662827..185b24b 100755 --- a/rootfs/usr/local/sbin/cloud-compose-diagnostics.sh +++ b/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh @@ -8,8 +8,8 @@ readonly PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export PATH readonly bootstrap_marker="/var/lib/cloud-compose/bootstrap-complete" -readonly diagnostics_program="/usr/local/sbin/cloud-compose-diagnostics.sh" -readonly jq_program_dir="/usr/local/share/cloud-compose/jq" +readonly diagnostics_program="/etc/cloud-compose/bin/cloud-compose-diagnostics.sh" +readonly jq_program_dir="/etc/cloud-compose/jq" readonly process_pattern='[/]home/cloud-compose/run[.]sh|[/]home/cloud-compose/[h]ost-conf[.]sh|[/]home/cloud-compose/[h]ost-init[.]sh|[/]home/cloud-compose/[a]pp-init[.]sh|[/]home/cloud-compose/[i]nstall-dependencies|[a]pt-get|[r]pm-ostree|[d]ocker run|[s]itectl|[g]it clone' usage() { diff --git a/rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-app.jq b/rootfs/etc/cloud-compose/jq/diagnostics-entry-app.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-app.jq rename to rootfs/etc/cloud-compose/jq/diagnostics-entry-app.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-project-dir.jq b/rootfs/etc/cloud-compose/jq/diagnostics-entry-project-dir.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/diagnostics-entry-project-dir.jq rename to rootfs/etc/cloud-compose/jq/diagnostics-entry-project-dir.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/diagnostics-project-entries.jq b/rootfs/etc/cloud-compose/jq/diagnostics-project-entries.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/diagnostics-project-entries.jq rename to rootfs/etc/cloud-compose/jq/diagnostics-project-entries.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/diagnostics-validate-compose-projects.jq b/rootfs/etc/cloud-compose/jq/diagnostics-validate-compose-projects.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/diagnostics-validate-compose-projects.jq rename to rootfs/etc/cloud-compose/jq/diagnostics-validate-compose-projects.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-backup-completed-at.jq b/rootfs/etc/cloud-compose/jq/dr-backup-completed-at.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/dr-backup-completed-at.jq rename to rootfs/etc/cloud-compose/jq/dr-backup-completed-at.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-backup-remote-id.jq b/rootfs/etc/cloud-compose/jq/dr-backup-remote-id.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/dr-backup-remote-id.jq rename to rootfs/etc/cloud-compose/jq/dr-backup-remote-id.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-restore-completed-at.jq b/rootfs/etc/cloud-compose/jq/dr-restore-completed-at.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/dr-restore-completed-at.jq rename to rootfs/etc/cloud-compose/jq/dr-restore-completed-at.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-restore-recovery-id.jq b/rootfs/etc/cloud-compose/jq/dr-restore-recovery-id.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/dr-restore-recovery-id.jq rename to rootfs/etc/cloud-compose/jq/dr-restore-recovery-id.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-validate-backup-receipt.jq b/rootfs/etc/cloud-compose/jq/dr-validate-backup-receipt.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/dr-validate-backup-receipt.jq rename to rootfs/etc/cloud-compose/jq/dr-validate-backup-receipt.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/dr-validate-restore-proof.jq b/rootfs/etc/cloud-compose/jq/dr-validate-restore-proof.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/dr-validate-restore-proof.jq rename to rootfs/etc/cloud-compose/jq/dr-validate-restore-proof.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-bind-sources.jq b/rootfs/etc/cloud-compose/jq/offhost-bind-sources.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/offhost-bind-sources.jq rename to rootfs/etc/cloud-compose/jq/offhost-bind-sources.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-build-application-coverage.jq b/rootfs/etc/cloud-compose/jq/offhost-build-application-coverage.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/offhost-build-application-coverage.jq rename to rootfs/etc/cloud-compose/jq/offhost-build-application-coverage.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-build-manifest.jq b/rootfs/etc/cloud-compose/jq/offhost-build-manifest.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/offhost-build-manifest.jq rename to rootfs/etc/cloud-compose/jq/offhost-build-manifest.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-manifest-app-digests.jq b/rootfs/etc/cloud-compose/jq/offhost-manifest-app-digests.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/offhost-manifest-app-digests.jq rename to rootfs/etc/cloud-compose/jq/offhost-manifest-app-digests.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-bind-roots.jq b/rootfs/etc/cloud-compose/jq/offhost-validate-bind-roots.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/offhost-validate-bind-roots.jq rename to rootfs/etc/cloud-compose/jq/offhost-validate-bind-roots.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-compose-config.jq b/rootfs/etc/cloud-compose/jq/offhost-validate-compose-config.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/offhost-validate-compose-config.jq rename to rootfs/etc/cloud-compose/jq/offhost-validate-compose-config.jq diff --git a/rootfs/usr/local/share/cloud-compose/jq/offhost-validate-manifest.jq b/rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq similarity index 100% rename from rootfs/usr/local/share/cloud-compose/jq/offhost-validate-manifest.jq rename to rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq diff --git a/rootfs/usr/local/libexec/cloud-compose/bootstrap-required.sh b/rootfs/etc/cloud-compose/libexec/bootstrap-required.sh old mode 100644 new mode 100755 similarity index 86% rename from rootfs/usr/local/libexec/cloud-compose/bootstrap-required.sh rename to rootfs/etc/cloud-compose/libexec/bootstrap-required.sh index 2d56f2a..c0ba4fb --- a/rootfs/usr/local/libexec/cloud-compose/bootstrap-required.sh +++ b/rootfs/etc/cloud-compose/libexec/bootstrap-required.sh @@ -3,7 +3,7 @@ set -uo pipefail # shellcheck disable=SC1091 -if ! source /usr/local/libexec/cloud-compose/bootstrap-security.sh; then +if ! source /etc/cloud-compose/libexec/bootstrap-security.sh; then echo "Cloud Compose bootstrap security helper could not be loaded" >&2 exit 255 fi diff --git a/rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh b/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh old mode 100644 new mode 100755 similarity index 100% rename from rootfs/usr/local/libexec/cloud-compose/bootstrap-security.sh rename to rootfs/etc/cloud-compose/libexec/bootstrap-security.sh diff --git a/rootfs/etc/cloud-compose/libexec/build-cos-make.sh b/rootfs/etc/cloud-compose/libexec/build-cos-make.sh new file mode 100755 index 0000000..f4f091a --- /dev/null +++ b/rootfs/etc/cloud-compose/libexec/build-cos-make.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +set -eux + +MAKE_VERSION="4.4.1" +MAKE_SHA256="dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3" + +# A single Alpine CDN outage must not make a healthy VM replacement fail. +# These are HTTPS endpoints from the Alpine official mirror list; apk still +# verifies the signed indexes and packages with the keys baked into the pinned +# image. +alpine_mirrors=" + https://dl-cdn.alpinelinux.org/alpine + https://mirror.math.princeton.edu/pub/alpinelinux + https://mirror.fel.cvut.cz/alpine +" +packages_installed=false +for alpine_mirror in ${alpine_mirrors}; do + printf "%s\n%s\n" \ + "${alpine_mirror}/v3.22/main" \ + "${alpine_mirror}/v3.22/community" \ + >/etc/apk/repositories + rm -f /var/cache/apk/* + if apk update && apk add build-base curl make tar; then + packages_installed=true + break + fi + echo "Alpine package mirror failed: ${alpine_mirror}" >&2 +done +if [ "${packages_installed}" != true ]; then + echo "All configured Alpine package mirrors failed" >&2 + exit 1 +fi + +curl -fsSL --proto "=https" --proto-redir "=https" --tlsv1.2 \ + --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 900 \ + --connect-timeout 10 --max-time 300 \ + "https://ftp.gnu.org/gnu/make/make-${MAKE_VERSION}.tar.gz" -o /tmp/make.tar.gz +echo "${MAKE_SHA256} /tmp/make.tar.gz" | sha256sum -c - +tar -xzf /tmp/make.tar.gz -C /tmp +cd "/tmp/make-${MAKE_VERSION}" +LDFLAGS="-static" ./configure --disable-nls +make -j2 +install -m 0755 make /out/.cloud-compose-make.pending +/out/.cloud-compose-make.pending --version | grep -Fqm 1 "GNU Make ${MAKE_VERSION}" +mv -f /out/.cloud-compose-make.pending /out/make diff --git a/rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh b/rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh old mode 100644 new mode 100755 similarity index 80% rename from rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh rename to rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh index a9c1ac5..1526af1 --- a/rootfs/usr/local/libexec/cloud-compose/require-bootstrap-ready.sh +++ b/rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh @@ -3,7 +3,7 @@ set -euo pipefail # shellcheck disable=SC1091 -source /usr/local/libexec/cloud-compose/bootstrap-security.sh +source /etc/cloud-compose/libexec/bootstrap-security.sh cloud_compose_bootstrap_require_root if ! cloud_compose_bootstrap_marker_ready; then diff --git a/rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh b/rootfs/etc/cloud-compose/libexec/run-bootstrap.sh old mode 100644 new mode 100755 similarity index 70% rename from rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh rename to rootfs/etc/cloud-compose/libexec/run-bootstrap.sh index 259f1c1..eda9d3b --- a/rootfs/usr/local/libexec/cloud-compose/run-bootstrap.sh +++ b/rootfs/etc/cloud-compose/libexec/run-bootstrap.sh @@ -3,7 +3,7 @@ set -euo pipefail # shellcheck disable=SC1091 -source /usr/local/libexec/cloud-compose/bootstrap-security.sh +source /etc/cloud-compose/libexec/bootstrap-security.sh cloud_compose_secure_runtime_home exec /bin/bash /home/cloud-compose/run.sh diff --git a/rootfs/usr/local/libexec/cloud-compose/run-root-program.sh b/rootfs/etc/cloud-compose/libexec/run-root-program.sh old mode 100644 new mode 100755 similarity index 91% rename from rootfs/usr/local/libexec/cloud-compose/run-root-program.sh rename to rootfs/etc/cloud-compose/libexec/run-root-program.sh index 326455e..b5e37e3 --- a/rootfs/usr/local/libexec/cloud-compose/run-root-program.sh +++ b/rootfs/etc/cloud-compose/libexec/run-root-program.sh @@ -3,7 +3,7 @@ set -euo pipefail # shellcheck disable=SC1091 -source /usr/local/libexec/cloud-compose/bootstrap-security.sh +source /etc/cloud-compose/libexec/bootstrap-security.sh program="${1:-}" if [[ -z "$program" ]]; then diff --git a/rootfs/usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh b/rootfs/etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh old mode 100644 new mode 100755 similarity index 73% rename from rootfs/usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh rename to rootfs/etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh index 02ce01b..b38af25 --- a/rootfs/usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh +++ b/rootfs/etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh @@ -3,7 +3,7 @@ set -euo pipefail # shellcheck disable=SC1091 -source /usr/local/libexec/cloud-compose/bootstrap-security.sh +source /etc/cloud-compose/libexec/bootstrap-security.sh cloud_compose_secure_runtime_home exec /bin/bash /home/cloud-compose/start-cloud-compose-bootstrap.sh "$@" diff --git a/rootfs/etc/systemd/system/cloud-compose-bootstrap.service b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service index 32c72ef..1149e95 100644 --- a/rootfs/etc/systemd/system/cloud-compose-bootstrap.service +++ b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service @@ -8,8 +8,8 @@ StartLimitIntervalSec=0 [Service] Type=oneshot RemainAfterExit=yes -ExecCondition=/bin/bash /usr/local/libexec/cloud-compose/bootstrap-required.sh -ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-bootstrap.sh +ExecCondition=/bin/bash /etc/cloud-compose/libexec/bootstrap-required.sh +ExecStart=/bin/bash /etc/cloud-compose/libexec/run-bootstrap.sh Restart=on-failure RestartSec=30s TimeoutStartSec=2h diff --git a/rootfs/etc/systemd/system/cloud-compose-docker-prune.service b/rootfs/etc/systemd/system/cloud-compose-docker-prune.service index 2b81c9c..701132d 100644 --- a/rootfs/etc/systemd/system/cloud-compose-docker-prune.service +++ b/rootfs/etc/systemd/system/cloud-compose-docker-prune.service @@ -8,5 +8,5 @@ After=docker.service Type=oneshot User=root Group=root -ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh docker-prune.sh +ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh docker-prune.sh TimeoutStartSec=30min diff --git a/rootfs/etc/systemd/system/cloud-compose-key-rotation.service b/rootfs/etc/systemd/system/cloud-compose-key-rotation.service index cac15e9..4a39855 100644 --- a/rootfs/etc/systemd/system/cloud-compose-key-rotation.service +++ b/rootfs/etc/systemd/system/cloud-compose-key-rotation.service @@ -6,7 +6,7 @@ RequiresMountsFor=/mnt/disks/data [Service] Type=oneshot -ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh rotate-keys-daily.sh +ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh rotate-keys-daily.sh User=root Group=root TimeoutStartSec=1h diff --git a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service index ec04bd7..e8742ab 100644 --- a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service +++ b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service @@ -6,7 +6,7 @@ Before=docker.service [Service] Type=oneshot -ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh configure-metadata-firewall.sh pre-docker +ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh configure-metadata-firewall.sh pre-docker RemainAfterExit=yes TimeoutStartSec=2min diff --git a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service index bb810f5..0333f5a 100644 --- a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service +++ b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service @@ -6,7 +6,7 @@ PartOf=docker.service [Service] Type=oneshot -ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh configure-metadata-firewall.sh +ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh configure-metadata-firewall.sh RemainAfterExit=yes TimeoutStartSec=2min diff --git a/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service b/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service index 4a1f07f..c1d4069 100644 --- a/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service +++ b/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service @@ -10,7 +10,7 @@ Type=oneshot User=root Group=root UMask=0077 -ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh offhost-backup.sh +ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh offhost-backup.sh TimeoutStartSec=24h StandardOutput=journal StandardError=journal diff --git a/rootfs/etc/systemd/system/cloud-compose-overlay.service b/rootfs/etc/systemd/system/cloud-compose-overlay.service index 7c5b261..69f8544 100644 --- a/rootfs/etc/systemd/system/cloud-compose-overlay.service +++ b/rootfs/etc/systemd/system/cloud-compose-overlay.service @@ -8,7 +8,7 @@ Type=oneshot User=root Group=root EnvironmentFile=-/home/cloud-compose/.env -ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh mount-overlays.sh +ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh mount-overlays.sh RemainAfterExit=yes [Install] diff --git a/rootfs/etc/systemd/system/cloud-compose-restore-test.service b/rootfs/etc/systemd/system/cloud-compose-restore-test.service index 64f9bd6..baa3b0a 100644 --- a/rootfs/etc/systemd/system/cloud-compose-restore-test.service +++ b/rootfs/etc/systemd/system/cloud-compose-restore-test.service @@ -9,7 +9,7 @@ Type=oneshot User=root Group=root UMask=0077 -ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh restore-test.sh +ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh restore-test.sh TimeoutStartSec=24h StandardOutput=journal StandardError=journal diff --git a/rootfs/etc/systemd/system/cloud-compose-vault-agent.service b/rootfs/etc/systemd/system/cloud-compose-vault-agent.service index 51d2cf0..c2e74f9 100644 --- a/rootfs/etc/systemd/system/cloud-compose-vault-agent.service +++ b/rootfs/etc/systemd/system/cloud-compose-vault-agent.service @@ -9,10 +9,10 @@ RequiresMountsFor=/mnt/disks/data User=root Group=root EnvironmentFile=-/etc/default/vault-agent -ExecStartPre=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh prepare +ExecStartPre=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh prepare ExecStart=/usr/local/bin/vault agent -config=/etc/vault-agent.d/cloud-compose.hcl -ExecStartPost=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh wait -ExecStopPost=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh vault-agent-readiness.sh clear +ExecStartPost=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh wait +ExecStopPost=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh clear Restart=on-failure RestartSec=10s TimeoutStartSec=2min diff --git a/rootfs/etc/systemd/system/libops-managed-runtime.service b/rootfs/etc/systemd/system/libops-managed-runtime.service index 2644d67..94d6781 100644 --- a/rootfs/etc/systemd/system/libops-managed-runtime.service +++ b/rootfs/etc/systemd/system/libops-managed-runtime.service @@ -8,7 +8,7 @@ RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volu Type=oneshot User=root Group=root -ExecStart=/bin/bash /usr/local/libexec/cloud-compose/run-root-program.sh libops-managed-runtime.sh update +ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh libops-managed-runtime.sh update TimeoutStartSec=1h StandardOutput=journal StandardError=journal diff --git a/rootfs/home/cloud-compose/disaster-recovery-lib.sh b/rootfs/home/cloud-compose/disaster-recovery-lib.sh index f797fde..8cba3df 100644 --- a/rootfs/home/cloud-compose/disaster-recovery-lib.sh +++ b/rootfs/home/cloud-compose/disaster-recovery-lib.sh @@ -4,8 +4,8 @@ # The caller must enable `set -euo pipefail` before sourcing this file. CLOUD_COMPOSE_DR_STATE_ROOT="${CLOUD_COMPOSE_DR_STATE_ROOT:-/mnt/disks/data/.cloud-compose-disaster-recovery}" -CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="${CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER:-/usr/local/libexec/cloud-compose/offhost-backup-driver}" -CLOUD_COMPOSE_JQ_PROGRAM_DIR="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/usr/local/share/cloud-compose/jq}" +CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="${CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER:-/etc/cloud-compose/libexec/offhost-backup-driver}" +CLOUD_COMPOSE_JQ_PROGRAM_DIR="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/etc/cloud-compose/jq}" cloud_compose_dr_is_required() { case "${CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED:-false}" in diff --git a/rootfs/home/cloud-compose/install-dependencies-cos.sh b/rootfs/home/cloud-compose/install-dependencies-cos.sh index 0f6320e..8273c56 100644 --- a/rootfs/home/cloud-compose/install-dependencies-cos.sh +++ b/rootfs/home/cloud-compose/install-dependencies-cos.sh @@ -18,7 +18,7 @@ valid_make_binary() { } # This function is sourced by the COS bootstrap contract, which supplies all -# three arguments even though the executable entrypoint below uses defaults. +# four arguments even though the executable entrypoint below uses defaults. # shellcheck disable=SC2120 install_cos_dependencies() { local cloud_compose_home="${1:-/home/cloud-compose}" @@ -27,6 +27,7 @@ install_cos_dependencies() { # PATH consumes the published symlink. local tool_state_dir="${2:-${COS_TOOL_STATE_DIR:-/mnt/disks/data/cloud-compose-tools}}" local docker_bin="${3:-/usr/bin/docker}" + local make_build_program="${4:-/etc/cloud-compose/libexec/build-cos-make.sh}" local make_path pending_make_path make_state_path make_state_tmp make_sha local tool_mount_options installer_uid installer_gid @@ -81,6 +82,10 @@ install_cos_dependencies() { return 1 fi fi + if [[ -L "$make_build_program" || ! -f "$make_build_program" || ! -x "$make_build_program" ]]; then + echo "COS Make build program is missing or unsafe: $make_build_program" >&2 + return 1 + fi rm -f -- "$pending_make_path" # The GCP metadata deny policy is installed before this container runs. # Keep the build on Docker's isolated bridge: `--network host` would @@ -89,50 +94,9 @@ install_cos_dependencies() { # shellcheck disable=SC2016 if ! retry_until_success "$docker_bin" run --rm \ -v "${tool_state_dir}:/out" \ + -v "${make_build_program}:/tmp/cloud-compose-build-cos-make.sh:ro" \ "$ALPINE_BUILD_IMAGE" \ - /bin/sh -euxc ' - MAKE_VERSION="4.4.1" - MAKE_SHA256="dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3" - - # A single Alpine CDN outage must not make a healthy VM - # replacement fail. These are HTTPS endpoints from the Alpine - # official mirror list; apk still verifies the signed indexes - # and packages with the keys baked into the pinned image. - alpine_mirrors=" - https://dl-cdn.alpinelinux.org/alpine - https://mirror.math.princeton.edu/pub/alpinelinux - https://mirror.fel.cvut.cz/alpine - " - packages_installed=false - for alpine_mirror in ${alpine_mirrors}; do - printf "%s\n%s\n" \ - "${alpine_mirror}/v3.22/main" \ - "${alpine_mirror}/v3.22/community" \ - >/etc/apk/repositories - rm -f /var/cache/apk/* - if apk update && apk add build-base curl make tar; then - packages_installed=true - break - fi - echo "Alpine package mirror failed: ${alpine_mirror}" >&2 - done - if [ "${packages_installed}" != true ]; then - echo "All configured Alpine package mirrors failed" >&2 - exit 1 - fi - curl -fsSL --proto "=https" --proto-redir "=https" --tlsv1.2 \ - --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 900 \ - --connect-timeout 10 --max-time 300 \ - "https://ftp.gnu.org/gnu/make/make-${MAKE_VERSION}.tar.gz" -o /tmp/make.tar.gz - echo "${MAKE_SHA256} /tmp/make.tar.gz" | sha256sum -c - - tar -xzf /tmp/make.tar.gz -C /tmp - cd "/tmp/make-${MAKE_VERSION}" - LDFLAGS="-static" ./configure --disable-nls - make -j2 - install -m 0755 make /out/.cloud-compose-make.pending - /out/.cloud-compose-make.pending --version | grep -Fqm 1 "GNU Make ${MAKE_VERSION}" - mv -f /out/.cloud-compose-make.pending /out/make - '; then + /bin/sh /tmp/cloud-compose-build-cos-make.sh; then rm -f -- "$pending_make_path" return 1 fi diff --git a/rootfs/home/cloud-compose/offhost-backup.sh b/rootfs/home/cloud-compose/offhost-backup.sh index f24f1ea..e84ea8b 100644 --- a/rootfs/home/cloud-compose/offhost-backup.sh +++ b/rootfs/home/cloud-compose/offhost-backup.sh @@ -6,7 +6,7 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" compose_apps_path="${CLOUD_COMPOSE_COMPOSE_APPS_PATH:-$script_dir/compose-apps.sh}" dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}" -jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/usr/local/share/cloud-compose/jq}" +jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/etc/cloud-compose/jq}" # shellcheck disable=SC1090 source "$profile_path" # shellcheck disable=SC1090 diff --git a/rootfs/home/cloud-compose/restore-test.sh b/rootfs/home/cloud-compose/restore-test.sh index 0ca6a0b..92c766f 100644 --- a/rootfs/home/cloud-compose/restore-test.sh +++ b/rootfs/home/cloud-compose/restore-test.sh @@ -5,7 +5,7 @@ set -euo pipefail script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}" -jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/usr/local/share/cloud-compose/jq}" +jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/etc/cloud-compose/jq}" # shellcheck disable=SC1090 source "$profile_path" CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir" diff --git a/runtime_contracts.tftest.hcl b/runtime_contracts.tftest.hcl index 7cdcf89..d818bdd 100644 --- a/runtime_contracts.tftest.hcl +++ b/runtime_contracts.tftest.hcl @@ -161,7 +161,7 @@ run "public_entrypoint_accepts_provider_neutral_disaster_recovery_driver" { runtime = { disaster_recovery = { required = true - driver_path = "/usr/local/libexec/cloud-compose/acme-offhost" + driver_path = "/etc/cloud-compose/libexec/acme-offhost" } } } @@ -169,7 +169,7 @@ run "public_entrypoint_accepts_provider_neutral_disaster_recovery_driver" { assert { condition = ( local.runtime.disaster_recovery.required && - local.runtime.disaster_recovery.driver_path == "/usr/local/libexec/cloud-compose/acme-offhost" + local.runtime.disaster_recovery.driver_path == "/etc/cloud-compose/libexec/acme-offhost" ) error_message = "The public entrypoint must preserve provider-neutral DR controls." } diff --git a/salt/cloud-compose/README.md b/salt/cloud-compose/README.md index b554c61..5d08c82 100644 --- a/salt/cloud-compose/README.md +++ b/salt/cloud-compose/README.md @@ -72,7 +72,7 @@ cloud_compose: runtime: disaster_recovery: required: true - driver_path: /usr/local/libexec/cloud-compose/offhost-backup-driver + driver_path: /etc/cloud-compose/libexec/offhost-backup-driver ``` Install that executable and its credentials separately as root. The formula diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls index e62ffd5..bcbeb54 100644 --- a/salt/cloud-compose/init.sls +++ b/salt/cloud-compose/init.sls @@ -144,7 +144,7 @@ {% set ignored = invalid_runtime_inputs.append('Vault Agent is currently supported only by Terraform providers; set vault.agent_enabled=false for Salt') %} {% endif %} {% set offhost_backup_required = disaster_recovery.get('required', False) %} -{% set offhost_backup_driver = disaster_recovery.get('driver_path', '/usr/local/libexec/cloud-compose/offhost-backup-driver') %} +{% set offhost_backup_driver = disaster_recovery.get('driver_path', '/etc/cloud-compose/libexec/offhost-backup-driver') %} {% if offhost_backup_required is not boolean %} {% set ignored = invalid_runtime_inputs.append('runtime.disaster_recovery.required must be a boolean') %} {% endif %} @@ -658,6 +658,19 @@ cloud-compose-rootfs: - require: - user: cloud-compose-user +cloud-compose-privileged-program-directories: + file.directory: + - names: + - /etc/cloud-compose + - /etc/cloud-compose/bin + - /etc/cloud-compose/jq + - /etc/cloud-compose/libexec + - user: root + - group: root + - mode: '0755' + - require: + - file: cloud-compose-rootfs + cloud-compose-lifecycle-lock: cmd.run: - name: systemd-tmpfiles --create /etc/tmpfiles.d/cloud-compose.conf @@ -673,10 +686,19 @@ cloud-compose-lifecycle-lock: cloud-compose-rootfs-script-modes: cmd.run: - - name: find /home/cloud-compose /usr/local/libexec/cloud-compose -maxdepth 1 -type f -name '*.sh' -exec chown root:root {} + -exec chmod 0755 {} + - - unless: test -z "$(find /home/cloud-compose /usr/local/libexec/cloud-compose -maxdepth 1 -type f -name '*.sh' \( ! -user root -o ! -group root -o ! -perm 0755 \) -print -quit)" + - name: find /home/cloud-compose /etc/cloud-compose/bin /etc/cloud-compose/libexec -maxdepth 1 -type f -name '*.sh' -exec chown root:root {} + -exec chmod 0755 {} + + - unless: test -z "$(find /home/cloud-compose /etc/cloud-compose/bin /etc/cloud-compose/libexec -maxdepth 1 -type f -name '*.sh' \( ! -user root -o ! -group root -o ! -perm 0755 \) -print -quit)" + - require: + - file: cloud-compose-rootfs + - file: cloud-compose-privileged-program-directories + +cloud-compose-rootfs-jq-modes: + cmd.run: + - name: find /etc/cloud-compose/jq -maxdepth 1 -type f -name '*.jq' -exec chown root:root {} + -exec chmod 0644 {} + + - unless: test -z "$(find /etc/cloud-compose/jq -maxdepth 1 -type f -name '*.jq' \( ! -user root -o ! -group root -o ! -perm 0644 \) -print -quit)" - require: - file: cloud-compose-rootfs + - file: cloud-compose-privileged-program-directories {% for lifecycle in ['init', 'up', 'down', 'rollout'] %} cloud-compose-lifecycle-{{ lifecycle }}: @@ -764,7 +786,7 @@ cloud-compose-systemd-reload: {% if rollout_enabled is sameas true %} cloud-compose-rollout-service: cmd.run: - - name: bash /usr/local/libexec/cloud-compose/run-root-program.sh deploy-rollout.sh + - name: bash /etc/cloud-compose/libexec/run-root-program.sh deploy-rollout.sh - require: - file: cloud-compose-env - file: cloud-compose-application-env @@ -793,8 +815,8 @@ cloud-compose-clear-bootstrap-marker: {% if run_bootstrap is sameas true %} cloud-compose-bootstrap: cmd.run: - - name: bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh - - unless: bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh + - name: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh + - unless: bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh - require: {% if install_packages %} - service: cloud-compose-docker @@ -804,6 +826,7 @@ cloud-compose-bootstrap: - file: cloud-compose-project-manifest - file: cloud-compose-managed-runtime-artifacts - cmd: cloud-compose-rootfs-script-modes + - cmd: cloud-compose-rootfs-jq-modes {% for lifecycle in ['init', 'up', 'down', 'rollout'] %} - file: cloud-compose-lifecycle-{{ lifecycle }} {% endfor %} diff --git a/templates/cloud-init.yml b/templates/cloud-init.yml index 9694f3b..38b136b 100644 --- a/templates/cloud-init.yml +++ b/templates/cloud-init.yml @@ -10,9 +10,9 @@ users: - docker sudo: - ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service - - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh state - - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh status - - ALL=(root) NOPASSWD:/usr/local/sbin/cloud-compose-diagnostics.sh dump + - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state + - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status + - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump %{ if length(CLOUD_COMPOSE_SSH_KEYS) > 0 ~} ssh_authorized_keys: %{ for key in CLOUD_COMPOSE_SSH_KEYS ~} @@ -90,10 +90,10 @@ runcmd: ${ROOTFS_ARCHIVE_COMMAND} - | set -eu - install -d -m 0755 -o root -g root /usr/local/sbin + install -d -m 0755 -o root -g root /etc/cloud-compose/bin install -m 0755 -o root -g root \ /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh \ - /usr/local/sbin/cloud-compose-diagnostics.sh + /etc/cloud-compose/bin/cloud-compose-diagnostics.sh test -f /run/cloud-compose-filesystems-ready || { echo "Cloud Compose filesystems were not prepared; refusing application initialization" >&2 exit 1 @@ -107,11 +107,11 @@ ${ROOTFS_ARCHIVE_COMMAND} chmod 0775 /mnt/disks/volumes install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops rm -f /var/lib/cloud-compose/bootstrap-complete - bash /usr/local/libexec/cloud-compose/start-cloud-compose-bootstrap.sh + bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh %{ if length(ADDITIONAL_RUNCMD) > 0 ~} - | set -eu - bash /usr/local/libexec/cloud-compose/require-bootstrap-ready.sh + bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh %{ for CMD in ADDITIONAL_RUNCMD ~} ${indent(2, CMD)} %{ endfor ~} diff --git a/tests/config-management/ansible/invalid-disaster-recovery.yml b/tests/config-management/ansible/invalid-disaster-recovery.yml index 896513c..19aac33 100644 --- a/tests/config-management/ansible/invalid-disaster-recovery.yml +++ b/tests/config-management/ansible/invalid-disaster-recovery.yml @@ -11,6 +11,6 @@ cloud_compose_runtime: disaster_recovery: required: true - driver_path: /usr/local/libexec/cloud-compose/../untrusted + driver_path: /etc/cloud-compose/libexec/../untrusted roles: - role: cloud_compose diff --git a/tests/config-management/ansible/smoke.yml b/tests/config-management/ansible/smoke.yml index 96ed382..f03fb35 100644 --- a/tests/config-management/ansible/smoke.yml +++ b/tests/config-management/ansible/smoke.yml @@ -14,7 +14,7 @@ cloud_compose_runtime: disaster_recovery: required: false - driver_path: /usr/local/libexec/cloud-compose/ansible-offhost + driver_path: /etc/cloud-compose/libexec/ansible-offhost extra_env: BASH_ENV: /tmp/cloud-compose-ansible-untrusted-bash-env LD_PRELOAD: /tmp/cloud-compose-ansible-untrusted-preload.so @@ -100,7 +100,7 @@ - >- 'CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED="false"' in (cloud_compose_env_file.content | b64decode) - >- - 'CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="/usr/local/libexec/cloud-compose/ansible-offhost"' in (cloud_compose_env_file.content | b64decode) + 'CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="/etc/cloud-compose/libexec/ansible-offhost"' in (cloud_compose_env_file.content | b64decode) - >- 'DOCKER_COMPOSE_DIR="/mnt/disks/data/libops/isle/isle-prod"' in (cloud_compose_env_file.content | b64decode) - >- diff --git a/tests/config-management/salt-pillar/invalid-disaster-recovery.sls b/tests/config-management/salt-pillar/invalid-disaster-recovery.sls index a9ca01c..9454410 100644 --- a/tests/config-management/salt-pillar/invalid-disaster-recovery.sls +++ b/tests/config-management/salt-pillar/invalid-disaster-recovery.sls @@ -9,4 +9,4 @@ cloud_compose: runtime: disaster_recovery: required: true - driver_path: /usr/local/libexec/cloud-compose/../untrusted + driver_path: /etc/cloud-compose/libexec/../untrusted diff --git a/tests/config-management/salt-pillar/wp-prod.sls b/tests/config-management/salt-pillar/wp-prod.sls index 1c910c4..e02485e 100644 --- a/tests/config-management/salt-pillar/wp-prod.sls +++ b/tests/config-management/salt-pillar/wp-prod.sls @@ -12,7 +12,7 @@ cloud_compose: runtime: disaster_recovery: required: false - driver_path: /usr/local/libexec/cloud-compose/salt-offhost + driver_path: /etc/cloud-compose/libexec/salt-offhost extra_env: BASH_ENV: /tmp/cloud-compose-salt-untrusted-bash-env LD_PRELOAD: /tmp/cloud-compose-salt-untrusted-preload.so diff --git a/variables.tf b/variables.tf index 23f2c11..5809f27 100644 --- a/variables.tf +++ b/variables.tf @@ -212,7 +212,7 @@ variable "runtime" { disaster_recovery = optional(object({ required = optional(bool, false) - driver_path = optional(string, "/usr/local/libexec/cloud-compose/offhost-backup-driver") + driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver") }), {}) compose = optional(object({ From c6984fda93e833248dd6df3871702f8feb1d5581 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 22:15:22 +0000 Subject: [PATCH 16/67] [patch] Use valid GCP contract name --- modules/gcp/runtime_contracts.tftest.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index 0c5fd62..d65163c 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -604,7 +604,7 @@ run "renders_embedded_offhost_backup_driver_executable" { command = plan variables { - name = "gcp-embedded-driver-contract" + name = "gcp-embedded-driver" project_id = "test-project" project_number = "123456789" docker_compose_repo = "https://github.com/libops/wp.git" From b5a9f671b8cf230d43e400124eaa4b781063fb26 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 22:41:31 +0000 Subject: [PATCH 17/67] [patch] Normalize trusted bootstrap files --- ci/host-runtime-security.sh | 10 ++ ci/rootfs-package-contract.sh | 2 + modules/gcp/main.tf | 6 + modules/linux-vm-runtime/main.tf | 6 + .../linux-vm-runtime/templates/cloud-init.yml | 1 + .../libexec/harden-bootstrap-paths.sh | 106 ++++++++++++++++++ templates/cloud-init.yml | 1 + 7 files changed, 132 insertions(+) create mode 100755 rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh diff --git a/ci/host-runtime-security.sh b/ci/host-runtime-security.sh index 859b16f..5b13cd1 100644 --- a/ci/host-runtime-security.sh +++ b/ci/host-runtime-security.sh @@ -261,6 +261,7 @@ if [[ -n "$(git -C "$repo_root" ls-files 'rootfs/usr/**')" ]]; then fi for trusted_program in \ rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh \ + rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh \ rootfs/etc/cloud-compose/libexec/build-cos-make.sh \ rootfs/etc/cloud-compose/libexec/bootstrap-security.sh \ rootfs/etc/cloud-compose/libexec/run-bootstrap.sh \ @@ -268,6 +269,15 @@ for trusted_program in \ rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq; do [[ -f "$repo_root/$trusted_program" ]] || fail "COS-safe trusted program is missing: $trusted_program" done + +assert_contains "$repo_root/templates/cloud-init.yml" \ + '/etc/cloud-compose/libexec/harden-bootstrap-paths.sh' +assert_contains "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml" \ + '/etc/cloud-compose/libexec/harden-bootstrap-paths.sh' +assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh" \ + 'chown root:root "$cloud_compose_home"' +assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh" \ + "0:1:regular file" for cloud_init_template in \ "$repo_root/templates/cloud-init.yml" \ "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do diff --git a/ci/rootfs-package-contract.sh b/ci/rootfs-package-contract.sh index d4612c6..19a1a6d 100755 --- a/ci/rootfs-package-contract.sh +++ b/ci/rootfs-package-contract.sh @@ -26,6 +26,8 @@ tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ grep -Fx 'rootfs/etc/cloud-compose/libexec/run-root-program.sh' >/dev/null tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ grep -Fx 'rootfs/etc/cloud-compose/libexec/build-cos-make.sh' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ + grep -Fx 'rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh' >/dev/null tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ grep -Fx 'rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq' >/dev/null tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index 8aa3c14..046d136 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -240,6 +240,7 @@ locals { write_files_content = join("\n", [ for file, config in local.all_files : <<-EOT - path: ${jsonencode(config.destination)} + owner: "root:root" permissions: ${jsonencode(local.rootfs_file_permissions[file])} encoding: gzip+base64 content: ${jsonencode(base64gzip(file(config.source)))} @@ -248,6 +249,7 @@ EOT docker_compose_scripts = join("\n", [ for name in ["init", "up", "down", "rollout"] : <<-EOT - path: "/home/cloud-compose/${name}" + owner: "root:root" permissions: "0755" encoding: gzip+base64 content: ${jsonencode(base64gzip(<<-EOS @@ -263,6 +265,7 @@ EOT ]) compose_projects_file = <<-EOT - path: "/home/cloud-compose/compose-projects.json" + owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(jsonencode(local.validated_compose_projects)))} @@ -281,6 +284,7 @@ managed_runtime_artifact_lines = [ ] managed_runtime_artifacts_file = <<-EOT - path: "/home/cloud-compose/managed-runtime-artifacts.tsv" + owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(join("\n", local.managed_runtime_artifact_lines)))} @@ -412,12 +416,14 @@ host_env = merge({ }, local.fresh_filesystem_env, local.rollout_env) env_file_content = <<-EOT - path: "/home/cloud-compose/.env" + owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(module.runtime_env.content))} EOT application_env_file_content = <<-EOT - path: "/home/cloud-compose/application-env.json" + owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(jsonencode(var.extra_env)))} diff --git a/modules/linux-vm-runtime/main.tf b/modules/linux-vm-runtime/main.tf index 047a98b..97022b8 100644 --- a/modules/linux-vm-runtime/main.tf +++ b/modules/linux-vm-runtime/main.tf @@ -106,6 +106,7 @@ locals { write_files_content = join("\n", [ for file, fullpath in local.all_files : <<-EOT - path: ${jsonencode(startswith(file, "mnt/disks/") ? "/var/lib/cloud-compose/mounted-rootfs/${file}" : "/${file}")} + owner: "root:root" permissions: ${jsonencode(local.rootfs_file_permissions[file])} encoding: gzip+base64 content: ${jsonencode(base64gzip(file(fullpath)))} @@ -115,6 +116,7 @@ EOT docker_compose_scripts = join("\n", [ for name in ["init", "up", "down", "rollout"] : <<-EOT - path: "/home/cloud-compose/${name}" + owner: "root:root" permissions: "0755" encoding: gzip+base64 content: ${jsonencode(base64gzip(<<-EOS @@ -132,6 +134,7 @@ EOT compose_projects_content = jsonencode(local.validated_compose_projects) compose_projects_file = <<-EOT - path: "/home/cloud-compose/compose-projects.json" + owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(local.compose_projects_content))} @@ -152,6 +155,7 @@ managed_runtime_artifact_lines = [ managed_runtime_artifacts_content = join("\n", local.managed_runtime_artifact_lines) managed_runtime_artifacts_file = <<-EOT - path: "/home/cloud-compose/managed-runtime-artifacts.tsv" + owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(local.managed_runtime_artifacts_content))} @@ -258,6 +262,7 @@ host_env = { env_file_content = <<-EOT - path: "/home/cloud-compose/.env" + owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(module.runtime_env.content))} @@ -265,6 +270,7 @@ env_file_content = <<-EOT application_env_file_content = <<-EOT - path: "/home/cloud-compose/application-env.json" + owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(jsonencode(var.extra_env)))} diff --git a/modules/linux-vm-runtime/templates/cloud-init.yml b/modules/linux-vm-runtime/templates/cloud-init.yml index d35c7e8..7cac7c0 100644 --- a/modules/linux-vm-runtime/templates/cloud-init.yml +++ b/modules/linux-vm-runtime/templates/cloud-init.yml @@ -116,4 +116,5 @@ runcmd: ${ROLLOUT_RUNCMD} %{ endif ~} rm -f /var/lib/cloud-compose/bootstrap-complete + /etc/cloud-compose/libexec/harden-bootstrap-paths.sh bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh diff --git a/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh b/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh new file mode 100755 index 0000000..eccc403 --- /dev/null +++ b/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cloud_compose_home="/home/cloud-compose" + +if [[ -L "$cloud_compose_home" || ! -d "$cloud_compose_home" ]]; then + echo "Cloud Compose home is unavailable or unsafe" >&2 + exit 1 +fi + +# cloud-init creates the operator account before all providers finish placing +# the checked-in runtime files. Close that initial writable-home window before +# any root process sources or executes those files. Files must already be +# root-owned, regular, and unlinked before this script normalizes archive modes; +# an operator replacement therefore fails closed instead of being blessed. +chown root:root "$cloud_compose_home" +chmod 0755 "$cloud_compose_home" + +require_root_owned_regular_file() { + local path="$1" + local metadata + + if [[ -L "$path" || ! -f "$path" ]]; then + echo "Unsafe Cloud Compose bootstrap file: $path" >&2 + exit 1 + fi + metadata="$(stat -c '%u:%h:%F' -- "$path")" + if [[ "$metadata" != "0:1:regular file" ]]; then + echo "Cloud Compose bootstrap file is not an unlinked root-owned regular file: $path" >&2 + exit 1 + fi +} + +shopt -s nullglob +programs=( + "$cloud_compose_home"/*.sh + "$cloud_compose_home"/*.jq + "$cloud_compose_home"/*.awk +) +shopt -u nullglob +for path in "${programs[@]}"; do + require_root_owned_regular_file "$path" +done + +for dispatcher in init up down rollout; do + path="${cloud_compose_home}/${dispatcher}" + if [[ -L "$path" || ( -e "$path" && ! -f "$path" ) ]]; then + echo "Unsafe Cloud Compose dispatcher: $path" >&2 + exit 1 + fi + if [[ -f "$path" ]]; then + require_root_owned_regular_file "$path" + fi +done + +for input in .env compose-projects.json application-env.json managed-runtime-artifacts.tsv; do + path="${cloud_compose_home}/${input}" + if [[ -L "$path" || ( -e "$path" && ! -f "$path" ) ]]; then + echo "Unsafe Cloud Compose input: $path" >&2 + exit 1 + fi + if [[ -f "$path" ]]; then + require_root_owned_regular_file "$path" + fi +done + +find "$cloud_compose_home" -mindepth 1 -maxdepth 1 -type f -name '*.sh' \ + -exec chown root:root {} + \ + -exec chmod 0755 {} + +find "$cloud_compose_home" -mindepth 1 -maxdepth 1 -type f -name '*.jq' \ + -exec chown root:root {} + \ + -exec chmod 0644 {} + +find "$cloud_compose_home" -mindepth 1 -maxdepth 1 -type f -name '*.awk' \ + -exec chown root:root {} + \ + -exec chmod 0644 {} + + +for dispatcher in init up down rollout; do + path="${cloud_compose_home}/${dispatcher}" + if [[ -f "$path" ]]; then + chown root:root "$path" + chmod 0755 "$path" + fi +done + +for input in .env compose-projects.json application-env.json managed-runtime-artifacts.tsv; do + path="${cloud_compose_home}/${input}" + if [[ -f "$path" ]]; then + chown root:cloud-compose "$path" + chmod 0640 "$path" + fi +done + +if [[ -L "${cloud_compose_home}/bin" || + ( -e "${cloud_compose_home}/bin" && ! -d "${cloud_compose_home}/bin" ) ]]; then + echo "Unsafe Cloud Compose command directory" >&2 + exit 1 +fi +install -d -m 0755 -o root -g root "${cloud_compose_home}/bin" + +home_identity="$(stat -Lc '%U:%G:%a' -- "$cloud_compose_home")" +bin_identity="$(stat -Lc '%U:%G:%a' -- "${cloud_compose_home}/bin")" +if [[ "$home_identity" != "root:root:755" || "$bin_identity" != "root:root:755" ]]; then + echo "Cloud Compose privileged paths have unsafe ownership or modes" >&2 + exit 1 +fi diff --git a/templates/cloud-init.yml b/templates/cloud-init.yml index 38b136b..94d2d13 100644 --- a/templates/cloud-init.yml +++ b/templates/cloud-init.yml @@ -107,6 +107,7 @@ ${ROOTFS_ARCHIVE_COMMAND} chmod 0775 /mnt/disks/volumes install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops rm -f /var/lib/cloud-compose/bootstrap-complete + /etc/cloud-compose/libexec/harden-bootstrap-paths.sh bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh %{ if length(ADDITIONAL_RUNCMD) > 0 ~} - | From 1025545b70a1cd8002363ef473ffbad30353e100 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 23:09:44 +0000 Subject: [PATCH 18/67] [patch] Apply shared bootstrap hardening to adapters --- ansible/README.md | 3 ++- ansible/roles/cloud_compose/tasks/main.yml | 5 +++++ ci/config-management-input-contract.sh | 4 ++++ salt/cloud-compose/README.md | 4 +++- salt/cloud-compose/init.sls | 17 +++++++++++++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/ansible/README.md b/ansible/README.md index 96f9875..f619bea 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -88,7 +88,8 @@ The role installs lifecycle dispatchers as `root:cloud-compose` mode `0750` and the root-consumed `.env`, project/application JSON, and managed-artifact manifest as `root:cloud-compose` mode `0640`. Reapplying the role restores that ownership boundary while leaving app checkout directories writable by the -`cloud-compose` account. +`cloud-compose` account. Before a requested runtime bootstrap, the role invokes +the same checked-in bootstrap path hardener used by the Terraform modules. The normal on-prem shape is one app per machine. Put each machine in the `cloud_compose` inventory group and set that host's template/runtime variables. diff --git a/ansible/roles/cloud_compose/tasks/main.yml b/ansible/roles/cloud_compose/tasks/main.yml index 0567e84..9fe1bea 100644 --- a/ansible/roles/cloud_compose/tasks/main.yml +++ b/ansible/roles/cloud_compose/tasks/main.yml @@ -644,6 +644,11 @@ mode: "0640" no_log: true +- name: Harden trusted cloud-compose bootstrap paths + ansible.builtin.command: + cmd: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh + when: cloud_compose_run_bootstrap | bool + - name: Reload systemd units ansible.builtin.systemd: daemon_reload: true diff --git a/ci/config-management-input-contract.sh b/ci/config-management-input-contract.sh index bdc060b..f1fcae4 100755 --- a/ci/config-management-input-contract.sh +++ b/ci/config-management-input-contract.sh @@ -27,6 +27,10 @@ if grep -Fq 'cmd: bash "{{ cloud_compose_home }}/' "$ansible_tasks" || \ grep -Fq 'cmd: bash /home/cloud-compose/' "$ansible_tasks"; then contract_fail "Ansible rollout executes a root program directly from the runtime home" fi +grep -Fq 'cmd: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh' "$ansible_tasks" || \ + contract_fail "Ansible does not apply the shared bootstrap path hardener" +grep -Fq -- '- name: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh' "$salt_state" || \ + contract_fail "Salt does not apply the shared bootstrap path hardener" grep -Fq 'configure-metadata-firewall.sh | deploy-rollout.sh | docker-prune.sh' "$root_program_runner" || \ contract_fail "the trusted root-program runner does not allow deploy-rollout.sh" diff --git a/salt/cloud-compose/README.md b/salt/cloud-compose/README.md index 5d08c82..4829b17 100644 --- a/salt/cloud-compose/README.md +++ b/salt/cloud-compose/README.md @@ -92,7 +92,9 @@ The formula installs lifecycle dispatchers as `root:cloud-compose` mode `0750` and the root-consumed `.env`, project/application JSON, and managed-artifact manifest as `root:cloud-compose` mode `0640`. Reapplying the state restores that ownership boundary while leaving application checkout directories writable by -the `cloud-compose` account. +the `cloud-compose` account. Before a requested runtime bootstrap, the formula +invokes the same checked-in bootstrap path hardener used by the Terraform +modules. The normal on-prem shape is one app per machine. Use pillar targeting to give each minion its own `cloud_compose` values, then apply the same diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls index bcbeb54..914e325 100644 --- a/salt/cloud-compose/init.sls +++ b/salt/cloud-compose/init.sls @@ -775,6 +775,22 @@ cloud-compose-managed-runtime-artifacts: - require: - file: cloud-compose-rootfs +{% if run_bootstrap is sameas true %} +cloud-compose-bootstrap-paths-hardened: + cmd.run: + - name: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh + - require: + - file: cloud-compose-env + - file: cloud-compose-application-env + - file: cloud-compose-project-manifest + - file: cloud-compose-managed-runtime-artifacts + - cmd: cloud-compose-rootfs-script-modes + - cmd: cloud-compose-rootfs-jq-modes +{% for lifecycle in ['init', 'up', 'down', 'rollout'] %} + - file: cloud-compose-lifecycle-{{ lifecycle }} +{% endfor %} +{% endif %} + {% if reload_systemd %} cloud-compose-systemd-reload: module.run: @@ -827,6 +843,7 @@ cloud-compose-bootstrap: - file: cloud-compose-managed-runtime-artifacts - cmd: cloud-compose-rootfs-script-modes - cmd: cloud-compose-rootfs-jq-modes + - cmd: cloud-compose-bootstrap-paths-hardened {% for lifecycle in ['init', 'up', 'down', 'rollout'] %} - file: cloud-compose-lifecycle-{{ lifecycle }} {% endfor %} From f6dc2f49c1bea8713bd619c3e37ab97b7b5386ee Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 23:10:44 +0000 Subject: [PATCH 19/67] [patch] Assert root-owned cloud-init overlays --- modules/gcp/runtime_contracts.tftest.hcl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index d65163c..50554ff 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -589,11 +589,11 @@ run "renders_verified_archive_before_downstream_overlay" { local.rootfs_file_permissions["etc/cloud-compose-overlay-marker"] == "0644" && strcontains( local.write_files_content, - "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose/libexec/custom-offhost-driver\"\n permissions: \"0755\"", + "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose/libexec/custom-offhost-driver\"\n owner: \"root:root\"\n permissions: \"0755\"", ) && strcontains( local.write_files_content, - "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose-overlay-marker\"\n permissions: \"0644\"", + "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose-overlay-marker\"\n owner: \"root:root\"\n permissions: \"0644\"", ) ) error_message = "Archive-backed GCP overlays must make only the configured off-host backup driver executable." @@ -618,11 +618,11 @@ run "renders_embedded_offhost_backup_driver_executable" { local.rootfs_file_permissions["etc/cloud-compose-overlay-marker"] == "0644" && strcontains( local.write_files_content, - "- path: \"/etc/cloud-compose/libexec/custom-offhost-driver\"\n permissions: \"0755\"", + "- path: \"/etc/cloud-compose/libexec/custom-offhost-driver\"\n owner: \"root:root\"\n permissions: \"0755\"", ) && strcontains( local.write_files_content, - "- path: \"/etc/cloud-compose-overlay-marker\"\n permissions: \"0644\"", + "- path: \"/etc/cloud-compose-overlay-marker\"\n owner: \"root:root\"\n permissions: \"0644\"", ) ) error_message = "Embedded GCP overlays must make only the configured off-host backup driver executable." From 15a9275740f790a26bac3684074f8d7074720226 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 23:15:49 +0000 Subject: [PATCH 20/67] [patch] Assert root-owned portable overlays --- modules/linux-vm-runtime/runtime_inputs.tftest.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index 3a5f841..8aa38fb 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -311,11 +311,11 @@ run "embeds_filesystem_helpers_without_archive" { local.rootfs_file_permissions["etc/cloud-compose/unrelated-config"] == "0644" && strcontains( local.write_files_content, - "- path: \"/etc/cloud-compose/libexec/custom-offhost-driver\"\n permissions: \"0755\"", + "- path: \"/etc/cloud-compose/libexec/custom-offhost-driver\"\n owner: \"root:root\"\n permissions: \"0755\"", ) && strcontains( local.write_files_content, - "- path: \"/etc/cloud-compose/unrelated-config\"\n permissions: \"0644\"", + "- path: \"/etc/cloud-compose/unrelated-config\"\n owner: \"root:root\"\n permissions: \"0644\"", ) ) error_message = "Embedded Linux VM overlays must make only the configured off-host backup driver executable." From 1712f2c1ef4b4fb5e9fc24eadb3ce9c5b7d08dd8 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 8 Aug 2026 23:32:49 +0000 Subject: [PATCH 21/67] [patch] Preserve empty managed artifact manifest --- modules/gcp/main.tf | 5 +++-- modules/linux-vm-runtime/main.tf | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index 046d136..0833c86 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -282,12 +282,13 @@ managed_runtime_artifact_lines = [ try(artifact.restart, ""), ]) ] -managed_runtime_artifacts_file = <<-EOT +managed_runtime_artifacts_content = length(local.managed_runtime_artifact_lines) == 0 ? "\n" : "${join("\n", local.managed_runtime_artifact_lines)}\n" +managed_runtime_artifacts_file = <<-EOT - path: "/home/cloud-compose/managed-runtime-artifacts.tsv" owner: "root:root" permissions: "0640" encoding: gzip+base64 - content: ${jsonencode(base64gzip(join("\n", local.managed_runtime_artifact_lines)))} + content: ${jsonencode(base64gzip(local.managed_runtime_artifacts_content))} EOT vault_agent_template_stanzas = join("\n", [ for template in var.vault_agent_templates : <<-EOT diff --git a/modules/linux-vm-runtime/main.tf b/modules/linux-vm-runtime/main.tf index 97022b8..fc5e168 100644 --- a/modules/linux-vm-runtime/main.tf +++ b/modules/linux-vm-runtime/main.tf @@ -152,7 +152,7 @@ managed_runtime_artifact_lines = [ try(artifact.restart, ""), ]) ] -managed_runtime_artifacts_content = join("\n", local.managed_runtime_artifact_lines) +managed_runtime_artifacts_content = length(local.managed_runtime_artifact_lines) == 0 ? "\n" : "${join("\n", local.managed_runtime_artifact_lines)}\n" managed_runtime_artifacts_file = <<-EOT - path: "/home/cloud-compose/managed-runtime-artifacts.tsv" owner: "root:root" From d2e0f62caf72e7cdd82b4fd6ebd8bcfd57bba139 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 00:19:00 +0000 Subject: [PATCH 22/67] [patch] Restore noninteractive host bootstrap --- ansible/roles/cloud_compose/defaults/main.yml | 2 +- ci/cos-bootstrap-contract.sh | 6 +++++- ci/sitectl-version-contract.sh | 11 +++++++++++ docs/runtime-contracts.md | 10 ++++++---- modules/gcp/variables.tf | 2 +- modules/linux-vm-runtime/variables.tf | 2 +- rootfs/home/cloud-compose/install-dependencies-cos.sh | 11 ++++++++--- rootfs/home/cloud-compose/libops-managed-runtime.sh | 8 +++++--- salt/cloud-compose/init.sls | 2 +- 9 files changed, 39 insertions(+), 15 deletions(-) diff --git a/ansible/roles/cloud_compose/defaults/main.yml b/ansible/roles/cloud_compose/defaults/main.yml index 4dff754..34bd496 100644 --- a/ansible/roles/cloud_compose/defaults/main.yml +++ b/ansible/roles/cloud_compose/defaults/main.yml @@ -53,7 +53,7 @@ cloud_compose_default_ingress: upload_timeout: "" cloud_compose_default_init: - - 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --project-name "${CLOUD_COMPOSE_INSTANCE_NAME}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --default' + - 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --yolo --default' cloud_compose_default_up: - 'sitectl compose --context "${SITECTL_CONTEXT_NAME}" up -d --remove-orphans' - 'sitectl healthcheck --context "${SITECTL_CONTEXT_NAME}" --persist' diff --git a/ci/cos-bootstrap-contract.sh b/ci/cos-bootstrap-contract.sh index 8b0998d..43125b6 100755 --- a/ci/cos-bootstrap-contract.sh +++ b/ci/cos-bootstrap-contract.sh @@ -66,7 +66,7 @@ grep -Fq 'iptables -C DOCKER-USER -d 169.254.169.254/32 -j DROP' "$INSTALLER" || echo "COS bootstrap does not require the GCP metadata deny policy" >&2 exit 1 } -grep -Fq 'COS_TOOL_STATE_DIR:-/mnt/disks/data/cloud-compose-tools' "$INSTALLER" || { +grep -Fq 'COS_TOOL_STATE_DIR:-/mnt/disks/data/libops-managed/bin' "$INSTALLER" || { echo "COS bootstrap stores an executable tool on a potentially noexec filesystem" >&2 exit 1 } @@ -187,6 +187,10 @@ grep -Fq "${BUILD_PROGRAM}:/tmp/cloud-compose-build-cos-make.sh:ro" "$CALL_LOG" [[ -f "${DATA_DIR}/make.state" ]] [[ -L "${CLOUD_HOME}/bin/make" ]] [[ "$(readlink "${CLOUD_HOME}/bin/make")" == "${DATA_DIR}/make" ]] +if grep -A3 -F 'chown -R cloud-compose:cloud-compose' "$INSTALLER" | grep -Fq '"${cloud_compose_home}/bin"'; then + echo "COS bootstrap gives the application account ownership of the privileged command directory" >&2 + exit 1 +fi # A failed replacement must not promote a partial binary or leave a pending # artifact that a later boot could mistake for a completed build. diff --git a/ci/sitectl-version-contract.sh b/ci/sitectl-version-contract.sh index b93e13a..f868d64 100644 --- a/ci/sitectl-version-contract.sh +++ b/ci/sitectl-version-contract.sh @@ -53,6 +53,17 @@ CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh" \ fi rm -f "$PUBLISHED_BIN_DIR/docker" + touch "$BIN_DIR/make" + ln -s "$BIN_DIR/make" "$PUBLISHED_BIN_DIR/make" + mkdirs + rm -f "$PUBLISHED_BIN_DIR/make" + ln -s "$2/unsafe-make" "$PUBLISHED_BIN_DIR/make" + if mkdirs; then + echo "managed runtime accepted an unsafe published Make target" >&2 + exit 1 + fi + rm -f "$PUBLISHED_BIN_DIR/make" + unsafe_target="$2/unsafe-target" unsafe_state="$2/unsafe-state" mkdir -p "$unsafe_target" diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 066f9c2..1febd43 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -183,10 +183,12 @@ package list before validation. Terraform, Ansible, and Salt serialize their resolved map as `SITECTL_PACKAGE_VERSIONS` JSON and the privileged installer validates it again before downloading anything. Per-project package versions are not supported because projects on one host share the same binaries. -`/home/cloud-compose/bin` is reserved for generated `sitectl` symlinks. During -an upgrade from the former application-owned directory, the installer closes -the directory to root and rejects any other inherited command or target rather -than carrying an untrusted PATH entry forward. +`/home/cloud-compose/bin` is reserved for generated managed-tool symlinks. They +target the root-owned managed binary directory and normally comprise `sitectl`, +its plugins, and the verified static GNU Make build required by Container- +Optimized OS. During an upgrade from the former application-owned directory, +the installer closes the directory to root and rejects any other inherited +command or target rather than carrying an untrusted PATH entry forward. `sitectl_verify_args` remains a real argument list. The host stores it as JSON and appends each value through an argv-aware wrapper when a lifecycle command diff --git a/modules/gcp/variables.tf b/modules/gcp/variables.tf index 35aca7f..df05258 100644 --- a/modules/gcp/variables.tf +++ b/modules/gcp/variables.tf @@ -262,7 +262,7 @@ variable "docker_compose_branch" { variable "docker_compose_init" { type = list(string) default = [ - "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --project-name \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --default" + "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --yolo --default" ] nullable = false description = "After cloning the docker compose git repo, any initialization that needs to happen before the docker compose project can start. One command per list value" diff --git a/modules/linux-vm-runtime/variables.tf b/modules/linux-vm-runtime/variables.tf index 5bf5f85..2ec8c3c 100644 --- a/modules/linux-vm-runtime/variables.tf +++ b/modules/linux-vm-runtime/variables.tf @@ -209,7 +209,7 @@ variable "compose_projects" { variable "docker_compose_init" { type = list(string) default = [ - "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --project-name \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --default" + "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --yolo --default" ] nullable = false description = "Commands run after a compose repository is cloned." diff --git a/rootfs/home/cloud-compose/install-dependencies-cos.sh b/rootfs/home/cloud-compose/install-dependencies-cos.sh index 8273c56..ba1da78 100644 --- a/rootfs/home/cloud-compose/install-dependencies-cos.sh +++ b/rootfs/home/cloud-compose/install-dependencies-cos.sh @@ -25,7 +25,7 @@ install_cos_dependencies() { # COS mounts both /home and /var with noexec. Keep the verified Make binary # on the executable persistent data disk; only the unprivileged application # PATH consumes the published symlink. - local tool_state_dir="${2:-${COS_TOOL_STATE_DIR:-/mnt/disks/data/cloud-compose-tools}}" + local tool_state_dir="${2:-${COS_TOOL_STATE_DIR:-/mnt/disks/data/libops-managed/bin}}" local docker_bin="${3:-/usr/bin/docker}" local make_build_program="${4:-/etc/cloud-compose/libexec/build-cos-make.sh}" local make_path pending_make_path make_state_path make_state_tmp make_sha @@ -62,14 +62,19 @@ install_cos_dependencies() { installer_gid="$(id -g)" chown "${installer_uid}:${installer_gid}" "$tool_state_dir" chmod 0755 "$tool_state_dir" + # This directory is on the privileged host PATH. Close legacy + # application ownership before publishing any verified tool link. The + # installer is root in production; the numeric identity keeps the + # standalone contract harness unprivileged. + chown "${installer_uid}:${installer_gid}" "${cloud_compose_home}/bin" + chmod 0755 "${cloud_compose_home}/bin" DOCKER_CLI_PLUGIN_DIR="${DOCKER_CONFIG}/cli-plugins" \ bash "${cloud_compose_home}/install-docker-plugins.sh" DOCKER_CLI_PLUGIN_DIR="${cloud_compose_home}/.docker/cli-plugins" \ bash "${cloud_compose_home}/install-docker-plugins.sh" chown -R cloud-compose:cloud-compose \ "$DOCKER_CONFIG" \ - "${cloud_compose_home}/.docker" \ - "${cloud_compose_home}/bin" + "${cloud_compose_home}/.docker" make_path="${tool_state_dir}/make" make_state_path="${tool_state_dir}/make.state" diff --git a/rootfs/home/cloud-compose/libops-managed-runtime.sh b/rootfs/home/cloud-compose/libops-managed-runtime.sh index 9fe81cc..9ef8b0a 100644 --- a/rootfs/home/cloud-compose/libops-managed-runtime.sh +++ b/rootfs/home/cloud-compose/libops-managed-runtime.sh @@ -130,14 +130,16 @@ validate_published_bin_directory() { local -a entries # /home/cloud-compose/bin was application-owned on older hosts. Preserve - # only the generated sitectl links whose targets remain under the validated - # root-owned package directory; reject every other inherited PATH entry. + # only generated managed-tool links whose targets remain under the + # validated root-owned package directory; reject every other inherited PATH + # entry. COS publishes its verified static Make build here because /home + # and /var are mounted noexec. shopt -s nullglob dotglob entries=("$path"/*) shopt -u nullglob dotglob for entry in "${entries[@]}"; do name="${entry##*/}" - if [[ ! "$name" =~ ^sitectl(-[a-z0-9]+)*$ || ! -L "$entry" ]]; then + if [[ ! "$name" =~ ^(make|sitectl(-[a-z0-9]+)*)$ || ! -L "$entry" ]]; then log "published command directory contains an unmanaged entry: ${entry}" return 1 fi diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls index 914e325..122b563 100644 --- a/salt/cloud-compose/init.sls +++ b/salt/cloud-compose/init.sls @@ -209,7 +209,7 @@ 'upload_timeout': '' } %} {% set default_init = [ - 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --project-name "${CLOUD_COMPOSE_INSTANCE_NAME}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --default' + 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --yolo --default' ] %} {% set default_up = [ 'sitectl compose --context "${SITECTL_CONTEXT_NAME}" up -d --remove-orphans', From c83bb6ca9c33c00f8f27ca8f5bfd222ef30fdfe9 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 00:51:33 +0000 Subject: [PATCH 23/67] [patch] Discover secrets in Compose Specification files --- ci/compose-runtime-contract.sh | 14 ++++++++++++++ rootfs/home/cloud-compose/compose-apps.sh | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index 31537ad..6a28199 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -27,6 +27,20 @@ mkdir -p "$CLOUD_COMPOSE_DATA_ROOT/project" # shellcheck disable=SC1091 source "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" +cat >"$CLOUD_COMPOSE_DATA_ROOT/project/compose.yaml" <<'EOF' +secrets: + DB_ROOT_PASSWORD: + file: ./secrets/DB_ROOT_PASSWORD + WORDPRESS_DB_PASSWORD: + file: ./secrets/WORDPRESS_DB_PASSWORD +EOF +mapfile -t compose_secrets < <( + cd "$CLOUD_COMPOSE_DATA_ROOT/project" + compose_secret_files +) +[[ "${compose_secrets[*]}" == "./secrets/DB_ROOT_PASSWORD ./secrets/WORDPRESS_DB_PASSWORD" ]] || \ + fail "Compose Specification compose.yaml secret files were not discovered" + jq -n \ --arg project_dir "$CLOUD_COMPOSE_DATA_ROOT/project" \ --arg trailing_command $'printf "trailing newline preserved"\n' '{ diff --git a/rootfs/home/cloud-compose/compose-apps.sh b/rootfs/home/cloud-compose/compose-apps.sh index d1834b9..f5cfd62 100644 --- a/rootfs/home/cloud-compose/compose-apps.sh +++ b/rootfs/home/cloud-compose/compose-apps.sh @@ -570,7 +570,7 @@ EOF compose_secret_files() { local compose_file - for compose_file in docker-compose.yaml docker-compose.yml; do + for compose_file in compose.yaml compose.yml docker-compose.yaml docker-compose.yml; do if [ ! -f "$compose_file" ]; then continue fi From 15c52fda6603da6d0e2b766d6cf6349cb3825e16 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 00:53:43 +0000 Subject: [PATCH 24/67] [patch] Invoke Compose secret parser from file --- rootfs/home/cloud-compose/compose-apps.sh | 19 +++---------------- .../cloud-compose/compose-secret-files.awk | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 rootfs/home/cloud-compose/compose-secret-files.awk diff --git a/rootfs/home/cloud-compose/compose-apps.sh b/rootfs/home/cloud-compose/compose-apps.sh index f5cfd62..81438ef 100644 --- a/rootfs/home/cloud-compose/compose-apps.sh +++ b/rootfs/home/cloud-compose/compose-apps.sh @@ -6,6 +6,8 @@ COMPOSE_PROJECTS_FILE="${COMPOSE_PROJECTS_FILE:-/home/cloud-compose/compose-proj COMPOSE_APPS_ENV_DIR="${COMPOSE_APPS_ENV_DIR:-/home/cloud-compose/apps}" COMPOSE_APPS_STATE_DIR="${COMPOSE_APPS_STATE_DIR:-/home/cloud-compose/state}" CLOUD_COMPOSE_DATA_ROOT="${CLOUD_COMPOSE_DATA_ROOT:-/mnt/disks/data}" +compose_apps_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_SECRET_FILES_PROGRAM="${CLOUD_COMPOSE_COMPOSE_SECRET_FILES_PROGRAM:-$compose_apps_dir/compose-secret-files.awk}" shell_env_line() { local name="$1" @@ -575,22 +577,7 @@ compose_secret_files() { continue fi - awk ' - /^[[:space:]]*services:/ { in_secrets = 0 } - /^[^[:space:]][^:]*:/ { - if ($0 ~ /^secrets:/) { - in_secrets = 1 - } else if (in_secrets) { - in_secrets = 0 - } - } - in_secrets && /^[[:space:]]*file:[[:space:]]*/ { - value = $0 - sub(/^[[:space:]]*file:[[:space:]]*/, "", value) - gsub(/^["'\'']|["'\'']$/, "", value) - print value - } - ' "$compose_file" + awk -f "$COMPOSE_SECRET_FILES_PROGRAM" "$compose_file" done | sort -u } diff --git a/rootfs/home/cloud-compose/compose-secret-files.awk b/rootfs/home/cloud-compose/compose-secret-files.awk new file mode 100644 index 0000000..1fc8827 --- /dev/null +++ b/rootfs/home/cloud-compose/compose-secret-files.awk @@ -0,0 +1,18 @@ +/^[[:space:]]*services:/ { + in_secrets = 0 +} + +/^[^[:space:]][^:]*:/ { + if ($0 ~ /^secrets:/) { + in_secrets = 1 + } else if (in_secrets) { + in_secrets = 0 + } +} + +in_secrets && /^[[:space:]]*file:[[:space:]]*/ { + value = $0 + sub(/^[[:space:]]*file:[[:space:]]*/, "", value) + gsub(/^["']|["']$/, "", value) + print value +} From 8aad96d09db19ca0ea7d50fb1114c33fc56e2973 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 05:40:22 +0000 Subject: [PATCH 25/67] [minor] Harden first-customer provisioning runtime --- .github/workflows/cloud-smoke.yml | 10 +- .github/workflows/github-release.yaml | 21 + .github/workflows/publish-rootfs.yaml | 8 +- ansible/README.md | 6 + ansible/roles/cloud_compose/defaults/main.yml | 14 +- ansible/roles/cloud_compose/tasks/main.yml | 19 +- ci/bootstrap-recovery-contract.sh | 17 +- ci/cloud-smoke.sh | 52 +- ci/compose-runtime-contract.sh | 39 ++ ci/config-management-cloud-smoke-container.sh | 8 + ci/config-management-cloud-smoke-inner.sh | 190 +++---- ci/config-management-cloud-smoke.sh | 40 +- ci/config-management-input-contract.py | 483 ++++++++++++++++++ ci/config-management-input-contract.sh | 380 +------------- ...onfig-management-runtime-state-contract.py | 47 ++ ci/config-management-smoke-assert.py | 8 + ci/config-management-smoke-container.sh | 8 + ci/config-management-smoke-inner.sh | 26 +- ci/config-management-smoke.sh | 8 +- ci/filesystem-prep-contract.sh | 104 ++-- .../config-management-lifecycle-lock.sh | 56 ++ .../gcp-upgrade-smoke-contract-harness.sh | 56 ++ ci/fixtures/source-trust-rollout.sh | 13 + ci/gcp-upgrade-smoke-contract.sh | 152 +++--- ci/gcp-upgrade-smoke.sh | 134 ++--- ci/host-runtime-security.sh | 50 +- ci/lifecycle-program-contract.sh | 118 +++++ ci/package-rootfs.sh | 12 +- ci/remote/config-management-deploy-salt.sh | 94 ++++ ci/remote/config-management-diagnostics.sh | 26 + ci/remote/config-management-verify.sh | 50 ++ .../gcp-upgrade-assert-services-disabled.sh | 14 + ...cp-upgrade-container-metadata-isolation.sh | 12 + ci/remote/gcp-upgrade-read-filesystem-size.sh | 15 + .../gcp-upgrade-verify-metadata-isolation.sh | 34 ++ ci/remote/gcp-upgrade-write-disk-sentinels.sh | 14 + ci/rootfs-archive-path-contract.awk | 3 + ci/rootfs-package-contract.sh | 36 +- ci/source-trust-contract.sh | 156 +++++- ci/systemd-contract.sh | 4 + ci/terraform-validate.sh | 6 +- ci/verify-rootfs-release.sh | 125 +++++ docs/examples.md | 15 +- docs/non-gcp-providers.md | 47 +- docs/rollout.md | 18 +- docs/runtime-contracts.md | 79 ++- examples/digitalocean/variables.tf | 9 +- examples/linode/variables.tf | 9 +- .../contracttest/provider_boundary_test.go | 1 + internal/contracttest/rollout_parity_test.go | 5 +- modules/digitalocean/main.tf | 59 +-- .../digitalocean/runtime_inputs.tftest.hcl | 8 + modules/digitalocean/variables.tf | 4 +- modules/gcp/main.tf | 387 +++++++------- modules/gcp/runtime_contracts.tftest.hcl | 62 ++- modules/gcp/variables.tf | 17 +- modules/linode/main.tf | 59 +-- modules/linode/runtime_inputs.tftest.hcl | 8 + modules/linode/variables.tf | 4 +- modules/linux-vm-runtime/main.tf | 338 ++++++------ modules/linux-vm-runtime/outputs.tf | 15 + .../runtime_inputs.tftest.hcl | 192 +++++-- .../linux-vm-runtime/templates/cloud-init.yml | 97 +--- modules/linux-vm-runtime/variables.tf | 31 +- modules/linux-vm-runtime/versions.tf | 7 + providers/do/template_versions.tftest.hcl | 8 + providers/linode/template_versions.tftest.hcl | 8 + .../cloud-compose/jq/sitectl-verify-args.jq | 5 + .../libexec/bootstrap-security.sh | 1 + .../libexec/gcp-cloud-init-finalize.sh | 32 ++ .../libexec/gcp-cloud-init-post-bootstrap.sh | 27 + .../libexec/gcp-filesystem-boot.sh | 53 ++ .../libexec/linux-vm-cloud-init.sh | 83 +++ .../cloud-compose/libexec/rootfs-archive.sh | 349 +++++++++++++ .../libexec/run-lifecycle-program.sh | 83 +++ .../system/cloud-compose-bootstrap.service | 6 +- .../etc/systemd/system/cloud-compose.service | 5 +- .../home/cloud-compose/bootstrap-helpers.sh | 17 + rootfs/home/cloud-compose/compose-apps.sh | 24 +- .../home/cloud-compose/default-lifecycle.sh | 86 ++++ .../cloud-compose/lifecycle-entrypoint.sh | 16 + rootfs/home/cloud-compose/run.sh | 4 +- salt/cloud-compose/README.md | 6 + salt/cloud-compose/init.sls | 28 +- template_versions.tftest.hcl | 8 + templates/cloud-init.yml | 104 ++-- templates/gcp-cloud-init.mime.tftpl | 18 + templates/gcp-filesystem-boothook.sh.tftpl | 19 + tests/smoke/do/main.tf | 30 +- tests/smoke/do/variables.tf | 8 +- tests/smoke/gcp-upgrade/main.tf | 8 +- .../lifecycle.d/gcp-upgrade-up.sh | 7 + .../gcp-upgrade-prepare-repository.sh | 44 ++ tests/smoke/linode/main.tf | 30 +- tests/smoke/linode/variables.tf | 8 +- tests/smoke/modules/context/main.tf | 20 +- tests/smoke/modules/context/outputs.tf | 2 +- tests/smoke/modules/context/variables.tf | 14 + 98 files changed, 3693 insertions(+), 1607 deletions(-) create mode 100755 ci/config-management-cloud-smoke-container.sh create mode 100755 ci/config-management-input-contract.py create mode 100644 ci/config-management-runtime-state-contract.py create mode 100755 ci/config-management-smoke-container.sh create mode 100755 ci/fixtures/config-management-lifecycle-lock.sh create mode 100755 ci/fixtures/gcp-upgrade-smoke-contract-harness.sh create mode 100755 ci/fixtures/source-trust-rollout.sh create mode 100755 ci/lifecycle-program-contract.sh create mode 100755 ci/remote/config-management-deploy-salt.sh create mode 100755 ci/remote/config-management-diagnostics.sh create mode 100755 ci/remote/config-management-verify.sh create mode 100755 ci/remote/gcp-upgrade-assert-services-disabled.sh create mode 100755 ci/remote/gcp-upgrade-container-metadata-isolation.sh create mode 100755 ci/remote/gcp-upgrade-read-filesystem-size.sh create mode 100755 ci/remote/gcp-upgrade-verify-metadata-isolation.sh create mode 100755 ci/remote/gcp-upgrade-write-disk-sentinels.sh create mode 100644 ci/rootfs-archive-path-contract.awk create mode 100755 ci/verify-rootfs-release.sh create mode 100644 rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq create mode 100755 rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh create mode 100755 rootfs/etc/cloud-compose/libexec/gcp-cloud-init-post-bootstrap.sh create mode 100755 rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh create mode 100755 rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh create mode 100755 rootfs/etc/cloud-compose/libexec/rootfs-archive.sh create mode 100755 rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh create mode 100644 rootfs/home/cloud-compose/default-lifecycle.sh create mode 100644 rootfs/home/cloud-compose/lifecycle-entrypoint.sh create mode 100644 templates/gcp-cloud-init.mime.tftpl create mode 100644 templates/gcp-filesystem-boothook.sh.tftpl create mode 100755 tests/smoke/gcp-upgrade/rootfs/etc/cloud-compose/lifecycle.d/gcp-upgrade-up.sh create mode 100755 tests/smoke/gcp-upgrade/rootfs/home/cloud-compose/gcp-upgrade-prepare-repository.sh diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml index e011fb3..cd10e02 100644 --- a/.github/workflows/cloud-smoke.yml +++ b/.github/workflows/cloud-smoke.yml @@ -78,7 +78,9 @@ jobs: cancel-in-progress: false env: CLOUD_COMPOSE_SMOKE_AUTO_APPROVE: "true" - CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "1800" + CLOUD_COMPOSE_SMOKE_BOOT_TIMEOUT: "900" + CLOUD_COMPOSE_SMOKE_CONFIG_MANAGEMENT_TIMEOUT: "2400" + CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "900" CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS: "true" CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }} steps: @@ -103,12 +105,14 @@ jobs: terraform_wrapper: false - name: Run Linode config-management smoke test + timeout-minutes: 100 env: LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} run: make config-management-cloud-smoke METHOD=${{ matrix.method }} - name: Destroy Linode config-management smoke resources if: always() + timeout-minutes: 20 env: LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} run: ci/config-management-cloud-smoke.sh destroy-${{ matrix.method }}-drupal @@ -149,7 +153,7 @@ jobs: CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "1800" CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS: "true" CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }} - CLOUD_COMPOSE_SOURCE_REF: ${{ github.event.pull_request.head.sha }} + CLOUD_COMPOSE_SOURCE_REF: ${{ github.sha }} steps: - name: Checkout repository @@ -208,7 +212,7 @@ jobs: CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "1800" CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS: "true" CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }} - CLOUD_COMPOSE_SOURCE_REF: ${{ github.event.pull_request.head.sha }} + CLOUD_COMPOSE_SOURCE_REF: ${{ github.sha }} CLOUD_COMPOSE_UPGRADE_BASE_SHA: f33117cdbbf4a9c7d59006a4db986baef118e6bb CLOUD_COMPOSE_UPGRADE_CURRENT_REF: ${{ github.sha }} GCLOUD_OIDC_POOL: ${{ vars.GCLOUD_OIDC_POOL || secrets.GCLOUD_OIDC_POOL }} diff --git a/.github/workflows/github-release.yaml b/.github/workflows/github-release.yaml index 0956dd4..dc04bf3 100644 --- a/.github/workflows/github-release.yaml +++ b/.github/workflows/github-release.yaml @@ -15,3 +15,24 @@ jobs: contents: write actions: write secrets: inherit + + verify-rootfs-assets: + name: Verify rootfs release assets + needs: release + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Checkout merged source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ github.event.pull_request.merge_commit_sha }} + + - name: Wait for and verify immutable rootfs assets + env: + GH_TOKEN: ${{ github.token }} + run: >- + ci/verify-rootfs-release.sh commit + "${{ github.event.pull_request.merge_commit_sha }}" diff --git a/.github/workflows/publish-rootfs.yaml b/.github/workflows/publish-rootfs.yaml index 2592899..533bb12 100644 --- a/.github/workflows/publish-rootfs.yaml +++ b/.github/workflows/publish-rootfs.yaml @@ -20,11 +20,17 @@ jobs: ci/rootfs-package-contract.sh ci/package-rootfs.sh dist - - name: Upload rootfs asset and checksum + - name: Upload rootfs asset, archive checksum, and source contract env: GH_TOKEN: ${{ github.token }} run: >- gh release upload "$GITHUB_REF_NAME" dist/cloud-compose-rootfs.tar.gz dist/cloud-compose-rootfs.tar.gz.sha256 + dist/cloud-compose-rootfs.contract.sha256 --clobber + + - name: Verify published rootfs release assets + env: + GH_TOKEN: ${{ github.token }} + run: ci/verify-rootfs-release.sh tag "$GITHUB_REF_NAME" diff --git a/ansible/README.md b/ansible/README.md index f619bea..4051c02 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -62,6 +62,12 @@ is meaningful: it disables that phase and is not replaced by the default. Runtime feature switches must be YAML booleans, not quoted strings; ambiguous values are rejected before host mutation. +Lifecycle list entries are program selectors, not shell source. Use the +built-in `/home/cloud-compose/default-lifecycle.sh ACTION`, `true`, `false`, or +one argument-free root-owned executable immediately below +`/etc/cloud-compose/lifecycle.d`. Put multi-step logic and quoting inside that +checked program file. + The provider-neutral disaster-recovery interface is shared with Terraform: ```yaml diff --git a/ansible/roles/cloud_compose/defaults/main.yml b/ansible/roles/cloud_compose/defaults/main.yml index 34bd496..716153d 100644 --- a/ansible/roles/cloud_compose/defaults/main.yml +++ b/ansible/roles/cloud_compose/defaults/main.yml @@ -29,6 +29,7 @@ cloud_compose_force_bootstrap: false cloud_compose_dedicated_host_acknowledged: false cloud_compose_bootstrap_timeout: 14400 cloud_compose_bootstrap_poll_interval: 15 +cloud_compose_bootstrap_wait_seconds: 10800 cloud_compose_extra_env: {} cloud_compose_runtime: {} @@ -53,15 +54,10 @@ cloud_compose_default_ingress: upload_timeout: "" cloud_compose_default_init: - - 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --yolo --default' + - '/home/cloud-compose/default-lifecycle.sh init' cloud_compose_default_up: - - 'sitectl compose --context "${SITECTL_CONTEXT_NAME}" up -d --remove-orphans' - - 'sitectl healthcheck --context "${SITECTL_CONTEXT_NAME}" --persist' - - 'if [ "${SITECTL_ENVIRONMENT}" != "production" ]; then sitectl verify --context "${SITECTL_CONTEXT_NAME}" ${SITECTL_VERIFY_ARGS:-}; fi' + - '/home/cloud-compose/default-lifecycle.sh up' cloud_compose_default_down: - - 'sitectl compose --context "${SITECTL_CONTEXT_NAME}" down' + - '/home/cloud-compose/default-lifecycle.sh down' cloud_compose_default_rollout: - - 'TARGET_REF="${GIT_REF:-${GIT_BRANCH:-}}"' - - 'if [ -n "$TARGET_REF" ]; then sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --ref "$TARGET_REF"; else sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --skip-git; fi' - - 'sitectl healthcheck --context "${SITECTL_CONTEXT_NAME}" --persist' - - 'if [ "${SITECTL_ENVIRONMENT}" != "production" ]; then sitectl verify --context "${SITECTL_CONTEXT_NAME}" ${SITECTL_VERIFY_ARGS:-}; fi' + - '/home/cloud-compose/default-lifecycle.sh rollout' diff --git a/ansible/roles/cloud_compose/tasks/main.yml b/ansible/roles/cloud_compose/tasks/main.yml index 9fe1bea..d315f18 100644 --- a/ansible/roles/cloud_compose/tasks/main.yml +++ b/ansible/roles/cloud_compose/tasks/main.yml @@ -33,7 +33,13 @@ - cloud_compose_managed_runtime_enabled is boolean - cloud_compose_internal_services_enabled is boolean - cloud_compose_internal_services_auto_update is boolean - fail_msg: "The adapter requires an onprem provider, a runtime/extra_env map, a name matching ^[a-z][a-z0-9-]*$, and cloud_compose_dedicated_host_acknowledged=true before it can own Docker and host runtime configuration." + - cloud_compose_bootstrap_wait_seconds is number + - cloud_compose_bootstrap_wait_seconds is not boolean + - (cloud_compose_bootstrap_wait_seconds | int) == cloud_compose_bootstrap_wait_seconds + - (cloud_compose_bootstrap_wait_seconds | int) >= 1 + - (cloud_compose_bootstrap_wait_seconds | int) <= 43200 + - (cloud_compose_bootstrap_timeout | int) > (cloud_compose_bootstrap_wait_seconds | int) + fail_msg: "The adapter requires an onprem provider, a runtime/extra_env map, a name matching ^[a-z][a-z0-9-]*$, dedicated-host acknowledgement, and a bootstrap async timeout longer than the 1-43200 second service wait." - name: Require safe cloud-compose environment entries ansible.builtin.assert: @@ -593,17 +599,12 @@ - name: Write cloud-compose lifecycle dispatchers ansible.builtin.copy: + src: /home/cloud-compose/lifecycle-entrypoint.sh dest: "{{ cloud_compose_home }}/{{ item }}" + remote_src: true owner: root group: "{{ cloud_compose_group }}" mode: "0750" - content: | - #!/usr/bin/env bash - - set -eou pipefail - - source /home/cloud-compose/profile.sh - exec bash /home/cloud-compose/compose-dispatch.sh "{{ item }}" loop: - init - up @@ -678,6 +679,8 @@ cmd: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh async: "{{ cloud_compose_bootstrap_timeout }}" poll: "{{ cloud_compose_bootstrap_poll_interval }}" + environment: + CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS: "{{ cloud_compose_bootstrap_wait_seconds | int }}" when: - cloud_compose_run_bootstrap | bool - (cloud_compose_bootstrap_readiness.rc | default(1)) != 0 diff --git a/ci/bootstrap-recovery-contract.sh b/ci/bootstrap-recovery-contract.sh index 5486cf9..54712bb 100644 --- a/ci/bootstrap-recovery-contract.sh +++ b/ci/bootstrap-recovery-contract.sh @@ -228,6 +228,7 @@ assert_contains "$bootstrap_security" '"$marker_size" == "6"' assert_contains "$bootstrap_security" '"$payload" == "ready"' assert_contains "$bootstrap_security" 'Cloud Compose control input is not root-controlled' assert_contains "$bootstrap_security" 'Cloud Compose lifecycle dispatcher is not root-controlled' +assert_contains "$bootstrap_security" '"$CLOUD_COMPOSE_RUNTIME_HOME/default-lifecycle.sh"' if rg -n 'bootstrap\\.log|exec (>>|>)[^[:space:]]' "$run_bootstrap" >/dev/null; then fail "bootstrap wrapper writes an independently unbounded log file" fi @@ -274,15 +275,19 @@ if rg -n '_SYSTEMD_UNIT=cloud-compose-bootstrap\\.service' \ fail "raw bootstrap output was added to Fluent Bit" fi -for cloud_init_template in \ - "$repo_root/templates/cloud-init.yml" \ - "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do - assert_contains "$cloud_init_template" 'bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' +for cloud_init_program in \ + "$repo_root/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh" \ + "$repo_root/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh"; do + assert_contains "$cloud_init_program" 'bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' if grep -Fq 'bash /home/cloud-compose/run.sh > /home/cloud-compose/run.log 2>&1' \ - "$cloud_init_template"; then - fail "cloud-init bypasses the retryable bootstrap unit" + "$cloud_init_program"; then + fail "$cloud_init_program bypasses the retryable bootstrap unit" fi done +assert_contains "$repo_root/templates/cloud-init.yml" \ + '/var/lib/cloud-compose/bootstrap/gcp-cloud-init-finalize.sh' +assert_contains "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml" \ + '/var/lib/cloud-compose/bootstrap/linux-vm-cloud-init.sh' : >"$systemctl_log" printf '0\n' >"$active_calls" diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh index eb6ac11..04a84c0 100755 --- a/ci/cloud-smoke.sh +++ b/ci/cloud-smoke.sh @@ -266,7 +266,7 @@ target_workdir() { target_var_args() { local root="$1" key_path="$2" target="$3" run_id="$4" run_namespace="$5" local public_key provider template - local source_ref source_sha256 source_cache_key checksum_dir checksum_file archive_tmp + local source_ref source_sha256 source_cache_key checksum_dir checksum_file archive_tmp checkout_sha provider="$(target_provider "$target")" template="$(target_template "$target")" @@ -285,14 +285,19 @@ target_var_args() { printf '%s\0%s\0' "-var" "template=${template}" fi if grep -q 'variable "cloud_compose_source_ref"' "$root/variables.tf"; then - source_ref="${CLOUD_COMPOSE_SOURCE_REF:-${GITHUB_SHA:-main}}" + checkout_sha="$(git -C "$repo_root" rev-parse HEAD)" + source_ref="${CLOUD_COMPOSE_SOURCE_REF:-${GITHUB_SHA:-$checkout_sha}}" + if [[ ! "$source_ref" =~ ^[0-9a-f]{40}$ || "$source_ref" != "$checkout_sha" ]]; then + echo "CLOUD_COMPOSE_SOURCE_REF must equal the exact lowercase checked-out commit ${checkout_sha}" >&2 + return 1 + fi printf '%s\0%s\0' "-var" "cloud_compose_source_ref=${source_ref}" fi if grep -q 'variable "cloud_compose_source_sha256"' "$root/variables.tf"; then source_sha256="${CLOUD_COMPOSE_SOURCE_SHA256:-}" if [[ -z "$source_sha256" ]]; then checksum_dir="$(target_workdir "$target")" - source_cache_key="$(printf '%s' "$source_ref" | sha256sum | awk '{print $1}')" + source_cache_key="$(printf '%s' "$source_ref" | sha256sum | cut -d' ' -f1)" checksum_file="${checksum_dir}/cloud-compose-source-${source_cache_key}.sha256" mkdir -p "$checksum_dir" if [[ -s "$checksum_file" ]]; then @@ -303,7 +308,7 @@ target_var_args() { curl -fsSL --retry 3 \ "https://github.com/libops/cloud-compose/archive/${source_ref}.tar.gz" \ -o "$archive_tmp" - source_sha256="$(sha256sum "$archive_tmp" | awk '{print $1}')" + source_sha256="$(sha256sum "$archive_tmp" | cut -d' ' -f1)" rm -f "$archive_tmp" printf '%s\n' "$source_sha256" > "$checksum_file" fi @@ -594,6 +599,40 @@ run_healthcheck() { --format table } +run_lifecycle_program_contract() { + local home_dir="$1" key_path="$2" output_json="$3" + local host port user remote_contract_dir remote_contract status + + host="$(jq -r '.host' "$output_json")" + port="$(jq -r '.ssh_port' "$output_json")" + user="$(jq -r '.ssh_user' "$output_json")" + remote_contract_dir="$(ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + 'mktemp -d /tmp/cloud-compose-hosted-contract.XXXXXX')" || return 1 + if [[ ! "$remote_contract_dir" =~ ^/tmp/cloud-compose-hosted-contract\.[A-Za-z0-9]+$ ]]; then + echo "Remote lifecycle contract directory is unsafe: $remote_contract_dir" >&2 + return 1 + fi + + remote_contract="$remote_contract_dir/lifecycle-program-contract.sh" + if ! ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "install -m 0700 /dev/stdin $remote_contract" \ + <"$repo_root/ci/lifecycle-program-contract.sh"; then + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "rmdir -- $remote_contract_dir" || true + return 1 + fi + + if ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "test -x /home/cloud-compose/default-lifecycle.sh && bash $remote_contract /home/cloud-compose/default-lifecycle.sh"; then + status=0 + else + status=$? + fi + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "rm -f -- $remote_contract && rmdir -- $remote_contract_dir" || true + return "$status" +} + run_target() ( set -euo pipefail @@ -691,6 +730,11 @@ run_target() ( return 1 fi + if ! run_lifecycle_program_contract "$home_dir" "$key_path" "$output_json"; then + dump_remote_logs "$home_dir" "$key_path" "$host" "$port" "$user" "$project_dir" + return 1 + fi + configure_sitectl_context "$home_dir" "$key_path" "$output_json" if ! run_healthcheck "$home_dir" "$key_path" "$output_json"; then dump_remote_logs "$home_dir" "$key_path" "$host" "$port" "$user" "$project_dir" diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index 6a28199..49e9580 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -11,12 +11,51 @@ fail() { exit 1 } +lifecycle_program="$repo_root/rootfs/home/cloud-compose/default-lifecycle.sh" +for defaults_file in \ + modules/gcp/variables.tf \ + modules/linux-vm-runtime/variables.tf \ + ansible/roles/cloud_compose/defaults/main.yml \ + salt/cloud-compose/init.sls; do + for action in init up down rollout; do + if [[ "$(grep -Fc -- "/home/cloud-compose/default-lifecycle.sh $action" "$repo_root/$defaults_file")" -ne 1 ]]; then + fail "$defaults_file does not invoke the $action lifecycle program exactly once" + fi + done + if grep -Fq -- 'TARGET_REF=' "$repo_root/$defaults_file"; then + fail "$defaults_file still splits rollout state across lifecycle command entries" + fi +done +bash "$repo_root/ci/lifecycle-program-contract.sh" \ + "$lifecycle_program" \ + "$repo_root/rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq" +grep -Fq -- 'run_lifecycle_program_contract "$home_dir" "$key_path" "$output_json"' \ + "$repo_root/ci/cloud-smoke.sh" || fail "provider smoke does not execute the lifecycle program contract" +grep -Fq -- 'bash "$lifecycle_program_contract" /home/cloud-compose/default-lifecycle.sh' \ + "$repo_root/ci/remote/config-management-verify.sh" || \ + fail "config-management smoke does not execute the lifecycle program contract" + grep -Fq 'cd "$script_dir"' "$repo_root/rootfs/home/cloud-compose/prepare-app-sources.sh" || \ fail "source preparation does not enter an accessible working directory before dropping privileges" grep -Fq 'run_as_cloud_compose() (' "$repo_root/rootfs/home/cloud-compose/run.sh" || \ fail "privilege-drop helper does not isolate its working-directory change" grep -Fq 'cd /home/cloud-compose' "$repo_root/rootfs/home/cloud-compose/run.sh" || \ fail "privilege-drop helper can inherit an inaccessible caller working directory" +if grep -Fq 'su -s /bin/bash -c' "$repo_root/rootfs/home/cloud-compose/run.sh"; then + fail "privilege-drop helper still synthesizes a shell program through su" +fi +grep -Fq '"$COMPOSE_LIFECYCLE_EXECUTOR" "$lifecycle" "$command"' \ + "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" || \ + fail "Compose lifecycle entries do not pass through the checked executor" +grep -Fq 'readonly COMPOSE_LIFECYCLE_EXECUTOR="/etc/cloud-compose/libexec/run-lifecycle-program.sh"' \ + "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" || \ + fail "Compose lifecycle entries do not use the canonical checked executor path" +grep -Fq '"$COMPOSE_LIFECYCLE_EXECUTOR" --validate "$lifecycle" "$command" || return 1' \ + "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" || \ + fail "Compose lifecycle program sets are not validated before execution" +if grep -Fq 'bash -c "$command"' "$repo_root/rootfs/home/cloud-compose/compose-apps.sh"; then + fail "Compose lifecycle entries still execute as shell strings" +fi export COMPOSE_PROJECTS_FILE="$tmp/compose-projects.json" export COMPOSE_APPS_ENV_DIR="$tmp/apps" diff --git a/ci/config-management-cloud-smoke-container.sh b/ci/config-management-cloud-smoke-container.sh new file mode 100755 index 0000000..a83bb9d --- /dev/null +++ b/ci/config-management-cloud-smoke-container.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +install -d -m 0755 /work +tar -C /work -xf - +cd /work +exec bash ci/config-management-cloud-smoke-inner.sh diff --git a/ci/config-management-cloud-smoke-inner.sh b/ci/config-management-cloud-smoke-inner.sh index 4a70bf9..0652f3a 100755 --- a/ci/config-management-cloud-smoke-inner.sh +++ b/ci/config-management-cloud-smoke-inner.sh @@ -86,6 +86,8 @@ all: cloud_compose_name: ${SMOKE_NAME} cloud_compose_template: ${SMOKE_TEMPLATE} cloud_compose_dedicated_host_acknowledged: true + cloud_compose_bootstrap_timeout: 1500 + cloud_compose_bootstrap_wait_seconds: 1200 cloud_compose_runtime: compose: ingress_port: 80 @@ -100,6 +102,8 @@ EOF } deploy_salt() { + local remote_command + if ! tar -C /work \ --exclude="./.git" \ --exclude="./.terraform" \ @@ -110,147 +114,69 @@ deploy_salt() { return 1 fi - # Smoke settings must be expanded locally for the remote shell. - # shellcheck disable=SC2029 - if ! ssh "${ssh_opts[@]}" "$ssh_target" \ - "SMOKE_NAME=${SMOKE_NAME} SMOKE_TEMPLATE=${SMOKE_TEMPLATE} SMOKE_ENVIRONMENT=${SMOKE_ENVIRONMENT} SMOKE_PROJECT_DIR=${SMOKE_PROJECT_DIR} bash -s" <<'REMOTE' -set -euo pipefail - -export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get install -y --no-install-recommends python3-venv ca-certificates - -python3 -m venv /opt/cloud-compose-salt-smoke -/opt/cloud-compose-salt-smoke/bin/python -m pip install --no-cache-dir \ - salt==3007.1 \ - tornado==6.4.2 \ - looseversion==1.3.0 \ - PyYAML==6.0.2 \ - packaging==24.2 \ - msgpack==1.1.0 \ - distro==1.9.0 \ - Jinja2==3.1.4 - -mkdir -p /tmp/cloud-compose-salt/etc /tmp/cloud-compose-salt/cache /tmp/cloud-compose-salt/pki /srv/cloud-compose/.smoke-pillar -cat >/tmp/cloud-compose-salt/etc/minion </srv/cloud-compose/.smoke-pillar/top.sls </srv/cloud-compose/.smoke-pillar/cloud-compose.sls </tmp/cloud-compose-salt-show-sls.txt - -/opt/cloud-compose-salt-smoke/bin/salt-call \ - --local \ - --retcode-passthrough \ - --config-dir=/tmp/cloud-compose-salt/etc \ - state.apply cloud-compose -REMOTE - then + printf -v remote_command '%q ' \ + /srv/cloud-compose/ci/remote/config-management-deploy-salt.sh \ + "$SMOKE_NAME" \ + "$SMOKE_TEMPLATE" \ + "$SMOKE_ENVIRONMENT" \ + "$SMOKE_PROJECT_DIR" + if ! ssh "${ssh_opts[@]}" "$ssh_target" "$remote_command"; then return 1 fi } verify_remote() { - # Smoke settings must be expanded locally for the remote shell. - # shellcheck disable=SC2029 - ssh "${ssh_opts[@]}" "$ssh_target" \ - "SMOKE_NAME=${SMOKE_NAME} SMOKE_TEMPLATE=${SMOKE_TEMPLATE} SMOKE_ENVIRONMENT=${SMOKE_ENVIRONMENT} SMOKE_PROJECT_DIR=${SMOKE_PROJECT_DIR} bash -s" <<'REMOTE' -set -euo pipefail - -test -x /home/cloud-compose/init -test -x /home/cloud-compose/up -test -x /home/cloud-compose/down -test -x /home/cloud-compose/rollout -test -x /home/cloud-compose/run.sh -test -x /home/cloud-compose/start-cloud-compose-bootstrap.sh -test -x /etc/cloud-compose/libexec/bootstrap-required.sh -test -x /etc/cloud-compose/libexec/bootstrap-security.sh -test -x /etc/cloud-compose/libexec/run-bootstrap.sh -test -x /etc/cloud-compose/libexec/require-bootstrap-ready.sh -test -x /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh -test -x /etc/cloud-compose/libexec/run-root-program.sh -python3 -m json.tool /home/cloud-compose/compose-projects.json >/dev/null -python3 -m json.tool /home/cloud-compose/application-env.json >/dev/null - -python3 - <<'PY' -import json -import os -import subprocess -from pathlib import Path - -name = os.environ["SMOKE_NAME"] -template = os.environ["SMOKE_TEMPLATE"] -project_dir = os.environ["SMOKE_PROJECT_DIR"] -def load_runtime_env(path): - result = subprocess.run( - [ - "env", "-i", "PATH=/usr/bin:/bin", f"CLOUD_COMPOSE_ENV_FILE={path}", - "bash", "--noprofile", "--norc", "-c", - "source /home/cloud-compose/profile.sh; env -0", - ], - check=True, - stdout=subprocess.PIPE, - ) - return { - entry.split(b"=", 1)[0].decode(): entry.split(b"=", 1)[1].decode() - for entry in result.stdout.split(b"\0") if b"=" in entry - } + local remote_contract_dir lifecycle_contract runtime_state_contract verification_program + local remote_command status -env = load_runtime_env(Path("/home/cloud-compose/.env")) + remote_contract_dir="$(ssh "${ssh_opts[@]}" "$ssh_target" \ + 'mktemp -d /tmp/cloud-compose-hosted-contract.XXXXXX')" || return 1 + if [[ ! "$remote_contract_dir" =~ ^/tmp/cloud-compose-hosted-contract\.[A-Za-z0-9]+$ ]]; then + echo "Remote contract directory is unsafe: $remote_contract_dir" >&2 + return 1 + fi -projects = json.loads(Path("/home/cloud-compose/compose-projects.json").read_text()) -project = projects[name] + lifecycle_contract="$remote_contract_dir/lifecycle-program-contract.sh" + runtime_state_contract="$remote_contract_dir/config-management-runtime-state-contract.py" + verification_program="$remote_contract_dir/config-management-verify.sh" + if ! ssh "${ssh_opts[@]}" "$ssh_target" \ + "install -m 0700 /dev/stdin $lifecycle_contract" \ + &2 - ssh_cmd "$home_dir" "$key_path" "$host" "bash -lc 'set +e -echo \"--- cloud-init status ---\" -cloud-init status --long -echo \"--- /var/log/cloud-init-output.log ---\" -tail -n 300 /var/log/cloud-init-output.log -echo \"--- cloud-compose bootstrap unit ---\" -sudo journalctl -u cloud-compose-bootstrap --no-pager -n 300 -echo \"--- cloud-compose unit ---\" -journalctl -u cloud-compose --no-pager -n 300 -echo \"--- docker ps ---\" -docker ps -a -echo \"--- compose manifest ---\" -cat /home/cloud-compose/compose-projects.json -'" || true + ssh_cmd "$home_dir" "$key_path" "$host" \ + "install -d -m 0700 $diagnostics_dir && install -m 0700 /dev/stdin $diagnostics_path" \ + <"$diagnostics_source" || return 0 + ssh_cmd "$home_dir" "$key_path" "$host" "$diagnostics_path" || true } target_var_args() { @@ -272,7 +270,7 @@ target_var_args() { deploy_config_management() { local target="$1" key_path="$2" output_json="$3" - local method host name template environment project_dir image + local method host name template environment project_dir image deploy_timeout container_entrypoint method="$(jq -r '.method' "$output_json")" host="$(jq -r '.host' "$output_json")" @@ -281,6 +279,12 @@ deploy_config_management() { environment="$(jq -r '.environment' "$output_json")" project_dir="$(jq -r '.project_dir' "$output_json")" image="${CLOUD_COMPOSE_CONFIG_MANAGEMENT_IMAGE:-$CONFIG_MANAGEMENT_IMAGE_DEFAULT}" + deploy_timeout="$(config_management_timeout_seconds)" + container_entrypoint="$repo_root/ci/config-management-cloud-smoke-container.sh" + [[ -f "$container_entrypoint" && ! -L "$container_entrypoint" ]] || { + echo "Config-management smoke container entrypoint is missing or unsafe" >&2 + return 1 + } if [[ -L "$key_path" || ! -f "$key_path" ]]; then echo "Config-management smoke SSH private-key path is missing or unsafe: $key_path" >&2 @@ -297,7 +301,7 @@ deploy_config_management() { # would also copy ignored Terraform state and the generated private key into # the helper container and, for Salt, onward to the provisioned VM. git -C "$repo_root" archive --format=tar HEAD | - docker run --rm -i \ + timeout --signal=TERM --kill-after=30s "${deploy_timeout}s" docker run --rm -i \ --env "SMOKE_METHOD=${method}" \ --env "SMOKE_HOST=${host}" \ --env "SMOKE_NAME=${name}" \ @@ -305,10 +309,11 @@ deploy_config_management() { --env "SMOKE_ENVIRONMENT=${environment}" \ --env "SMOKE_PROJECT_DIR=${project_dir}" \ --mount "type=bind,src=${key_path},dst=/run/secrets/cloud-compose-ssh-key,readonly" \ + --mount "type=bind,src=${container_entrypoint},dst=/usr/local/libexec/cloud-compose-config-management-smoke,readonly" \ --tmpfs /run \ --tmpfs /tmp \ "$image" \ - bash -lc 'mkdir -p /work && tar -C /work -xf - && cd /work && bash ci/config-management-cloud-smoke-inner.sh' + /usr/local/libexec/cloud-compose-config-management-smoke } require_run_commands() { @@ -319,6 +324,7 @@ require_run_commands() { require_cmd ssh-keygen require_cmd ssh-keyscan require_cmd terraform + require_cmd timeout } require_destroy_commands() { diff --git a/ci/config-management-input-contract.py b/ci/config-management-input-contract.py new file mode 100755 index 0000000..f824584 --- /dev/null +++ b/ci/config-management-input-contract.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python3 + +import base64 +import copy +import json +import os +import stat +import subprocess +import sys +import tempfile +from pathlib import Path + +repo_root = Path(sys.argv[1]) +validator = Path(sys.argv[2]) + + +def fail(message): + raise SystemExit(f"config-management input contract: {message}") + + +def snapshot(root): + entries = [] + for path in sorted(root.rglob("*")): + metadata = path.lstat() + relative = str(path.relative_to(root)) + if path.is_symlink(): + payload = ("symlink", os.readlink(path)) + elif path.is_file(): + payload = ("file", path.read_bytes()) + else: + payload = ("other", b"") + entries.append( + ( + relative, + stat.S_IMODE(metadata.st_mode), + metadata.st_uid, + metadata.st_gid, + payload, + ) + ) + return entries + + +with tempfile.TemporaryDirectory(prefix="cloud-compose-input-contract.") as temp_dir: + test_root = Path(temp_dir) + data_root = test_root / "mnt" / "disks" / "data" + outside = test_root / "outside" + data_root.mkdir(parents=True) + outside.mkdir() + (outside / "sentinel").write_text("must-not-change\n") + (data_root / "escape").symlink_to(outside, target_is_directory=True) + + safe_artifact = { + "name": "rollout-agent", + "url": "https://example.invalid/rollout-agent", + "sha256": "a" * 64, + "path": "/usr/local/bin/rollout-agent", + "mode": "0750", + "owner": "root", + "group": "root", + "restart": "cloud-compose-rollout.service", + } + safe_payload = { + "projects": [{"name": "app", "project_dir": str(data_root / "app")}], + "artifacts": [safe_artifact], + } + + def run(payload): + before = snapshot(test_root) + environment = os.environ.copy() + environment["CLOUD_COMPOSE_VALIDATION_PAYLOAD_B64"] = base64.b64encode( + json.dumps(payload).encode("utf-8") + ).decode("ascii") + result = subprocess.run( + [sys.executable, str(validator), "--data-root", str(data_root)], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + after = snapshot(test_root) + if before != after: + fail("validator mutated the host fixture") + return result + + accepted = run(safe_payload) + if accepted.returncode != 0: + fail(f"safe payload was rejected: {accepted.stderr}") + + def reject(label, payload, expected): + result = run(payload) + if result.returncode == 0: + fail(f"{label} was accepted") + if expected not in result.stderr: + fail(f"{label} did not report {expected!r}: {result.stderr}") + + invalid_project_paths = { + "root project path": "/", + "system project path": "/etc", + "data root itself": str(data_root), + "traversal project path": str(data_root / ".." / "outside"), + "dot project segment": f"{data_root}/./app", + "empty project segment": f"{data_root}//app", + "control-character project segment": f"{data_root}/bad\x7fname", + "symlink escape": str(data_root / "escape" / "app"), + } + for label, project_dir in invalid_project_paths.items(): + payload = copy.deepcopy(safe_payload) + payload["projects"][0]["project_dir"] = project_dir + reject(label, payload, "project_dir") + + duplicate_ports = copy.deepcopy(safe_payload) + duplicate_ports["projects"] = [ + {"name": "alpha", "project_dir": str(data_root / "alpha"), "ingress_port": 8080}, + {"name": "beta", "project_dir": str(data_root / "beta"), "ingress_port": 8080}, + ] + reject("duplicate project ports", duplicate_ports, "ingress ports must be unique") + + artifact_cases = [] + + def artifact_case(label, field, value, expected): + payload = copy.deepcopy(safe_payload) + payload["artifacts"][0][field] = value + artifact_cases.append((label, payload, expected)) + + artifact_case("unsafe artifact name", "name", "../agent", "safe basename") + artifact_case("overlong artifact name", "name", "a" * 129, "safe basename") + artifact_case("non-HTTPS artifact URL", "url", "http://example.invalid/agent", "HTTPS URL") + artifact_case("uppercase artifact SHA", "sha256", "A" * 64, "64 lowercase hex") + artifact_case("root artifact path", "path", "/", "non-root absolute path") + artifact_case("relative artifact path", "path", "usr/local/bin/agent", "non-root absolute path") + artifact_case("dot artifact path", "path", "/usr/local/../bin/agent", "non-root absolute path") + artifact_case("empty artifact segment", "path", "/usr//local/bin/agent", "non-root absolute path") + artifact_case("control artifact segment", "path", "/usr/local/bad\x7fname", "non-root absolute path") + artifact_case("unsafe artifact mode", "mode", "4755", "mode must match") + artifact_case("unsafe artifact owner", "owner", "root:root", "owner must be") + artifact_case("unsafe artifact group", "group", "root:root", "group must be") + artifact_case("unsafe restart unit", "restart", "../docker.service", "safe .service") + + duplicate_name = copy.deepcopy(safe_payload) + second = copy.deepcopy(safe_artifact) + second["path"] = "/usr/local/bin/rollout-agent-two" + duplicate_name["artifacts"].append(second) + artifact_cases.append(("duplicate artifact name", duplicate_name, "names must be unique")) + + duplicate_path = copy.deepcopy(safe_payload) + second = copy.deepcopy(safe_artifact) + second["name"] = "rollout-agent-two" + duplicate_path["artifacts"].append(second) + artifact_cases.append(("duplicate artifact path", duplicate_path, "target paths must be unique")) + + for label, payload, expected in artifact_cases: + reject(label, payload, expected) + +ansible_tasks = (repo_root / "ansible/roles/cloud_compose/tasks/main.yml").read_text() +ansible_defaults = (repo_root / "ansible/roles/cloud_compose/defaults/main.yml").read_text() +ansible_gate = ansible_tasks.find("Validate project directory and managed artifact host boundaries") +ansible_first_mutation = ansible_tasks.find("Install Debian runtime dependencies") +if ansible_gate < 0 or ansible_first_mutation < 0 or ansible_gate > ansible_first_mutation: + fail("Ansible host-input validation does not precede its first host mutation") +if "files/validate-runtime-inputs.py" not in ansible_tasks: + fail("Ansible does not execute the shared host-input validator") +if "--data-root" in ansible_tasks: + fail("Ansible makes the production project ownership boundary configurable") +if "cmd: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh" not in ansible_tasks: + fail("Ansible bypasses the retryable bootstrap service") +if "cmd: bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh" not in ansible_tasks: + fail("Ansible does not validate bootstrap readiness evidence") +if 'cmd: bash "{{ cloud_compose_home }}/run.sh"' in ansible_tasks: + fail("Ansible still invokes the one-shot bootstrap script directly") +if "cloud_compose_bootstrap_timeout: 14400" not in ansible_defaults: + fail("Ansible bootstrap timeout does not cover the bounded retryable bootstrap wait") +if "cloud_compose_bootstrap_wait_seconds: 10800" not in ansible_defaults: + fail("Ansible bootstrap service wait does not have a bounded default") +for marker in ( + "cloud_compose_bootstrap_wait_seconds is number", + "(cloud_compose_bootstrap_wait_seconds | int) >= 1", + "(cloud_compose_bootstrap_wait_seconds | int) <= 43200", + "(cloud_compose_bootstrap_timeout | int) > (cloud_compose_bootstrap_wait_seconds | int)", + 'CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS: "{{ cloud_compose_bootstrap_wait_seconds | int }}"', +): + if marker not in ansible_tasks: + fail(f"Ansible bounded bootstrap wait marker is missing: {marker!r}") + +salt_state = (repo_root / "salt/cloud-compose/init.sls").read_text() +cloud_smoke_driver = (repo_root / "ci/config-management-cloud-smoke.sh").read_text() +cloud_smoke_inner = (repo_root / "ci/config-management-cloud-smoke-inner.sh").read_text() +cloud_smoke_container = (repo_root / "ci/config-management-cloud-smoke-container.sh").read_text() +remote_diagnostics = (repo_root / "ci/remote/config-management-diagnostics.sh").read_text() +remote_salt_deploy = (repo_root / "ci/remote/config-management-deploy-salt.sh").read_text() +remote_verification = (repo_root / "ci/remote/config-management-verify.sh").read_text() +cloud_smoke_workflow = (repo_root / ".github/workflows/cloud-smoke.yml").read_text() +local_smoke_driver = (repo_root / "ci/config-management-smoke.sh").read_text() +local_smoke_container = (repo_root / "ci/config-management-smoke-container.sh").read_text() +local_smoke_inner = (repo_root / "ci/config-management-smoke-inner.sh").read_text() +local_lifecycle_fixture = ( + repo_root / "ci/fixtures/config-management-lifecycle-lock.sh" +).read_text() +salt_gate = salt_state.find("cloud-compose-host-inputs-valid:") +salt_first_mutation = salt_state.find("cloud-compose-packages:") +if salt_gate < 0 or salt_first_mutation < 0 or salt_gate > salt_first_mutation: + fail("Salt host-input validation does not precede its first host mutation") +if "bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh" not in salt_state: + fail("Salt bypasses the retryable bootstrap service") +if "bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh" not in salt_state: + fail("Salt does not validate bootstrap readiness evidence") +if "home ~ '/run.sh'" in salt_state: + fail("Salt still invokes the one-shot bootstrap script directly") +for marker in ( + "bootstrap_wait_seconds = cc.get('bootstrap_wait_seconds', 10800)", + "bootstrap_wait_seconds < 1 or bootstrap_wait_seconds > 43200", + "CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS: {{ (bootstrap_wait_seconds | string) | json }}", +): + if marker not in salt_state: + fail(f"Salt bounded bootstrap wait marker is missing: {marker!r}") +gate_block = salt_state[salt_gate:salt_first_mutation] +for marker in ( + "cmd.run:", + '/usr/bin/env python3 "$CLOUD_COMPOSE_RUNTIME_VALIDATOR"', + "failhard: True", + "order: 2", +): + if marker not in gate_block: + fail(f"Salt host-input validation gate is missing {marker!r}") +for state_name in ("cloud-compose-packages:", "cloud-compose-docker-group:", "cloud-compose-group:"): + state_start = salt_state.find(state_name) + state_end = salt_state.find("\n\n", state_start) + if "cmd: cloud-compose-host-inputs-valid" not in salt_state[state_start:state_end]: + fail(f"Salt mutating state {state_name} does not require host-input validation") + +for marker in ( + "_cc_compose.init | default(cloud_compose_default_init)", + "_cc_compose.up | default(cloud_compose_default_up)", + "_cc_compose.down | default(cloud_compose_default_down)", + "_cc_compose.rollout | default(cloud_compose_default_rollout)", + "item.value.docker_compose_up | default(_cc_up_commands)", +): + if marker not in ansible_tasks: + fail(f"Ansible explicit-empty lifecycle parity marker is missing: {marker!r}") + +for marker in ( + "_cc_template.package_versions | default({})", + "_cc_template_sitectl_package_versions[item] | default(_cc_sitectl_version)", + "_cc_sitectl_package_version_overrides[item] | default(", + "_cc_sitectl.packages | default(_cc_template.packages)", + "item.value.sitectl_packages | default(_cc_sitectl_packages)", + "(_cc_managed.enabled | default(cloud_compose_managed_runtime_enabled)) is boolean", + "(_cc_vault.agent_enabled | default(false)) is boolean", + "(_cc_disaster_recovery.required | default(false)) is boolean", + "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED", + "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER", +): + if marker not in ansible_tasks: + fail(f"Ansible template package-version parity marker is missing: {marker!r}") + +for marker in ( + "configured_commands = compose.get(lifecycle)", + "configured_commands if lifecycle in compose else default_commands", + "if legacy_key in app", + "elif docker_key in app", +): + if marker not in salt_state: + fail(f"Salt explicit-empty lifecycle parity marker is missing: {marker!r}") + +for marker in ( + "template_sitectl_package_versions = template.get('package_versions', {})", + "sitectl_package_version_overrides.get(package, template_sitectl_package_versions.get(package, sitectl_version))", + "sitectl.get('packages') if 'packages' in sitectl else template.packages", + "app.get('sitectl_packages', sitectl_packages)", + "all_packages = sitectl_packages | list", + "'managed_runtime.enabled': managed_runtime_enabled", + "'vault.agent_enabled': vault.get('agent_enabled', False)", + "runtime_sections.disaster_recovery", + "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED", + "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER", + "run_bootstrap is sameas true", +): + if marker not in salt_state: + fail(f"Salt template package-version parity marker is missing: {marker!r}") + +for marker in ( + 'retry_remote_operation "Ansible deployment" deploy_ansible', + 'retry_remote_operation "Salt deployment" deploy_salt', + 'retry_remote_operation "Remote verification" verify_remote', + '-o ServerAliveInterval=30', + 'for attempt in 1 2 3; do', + 'cloud_compose_bootstrap_timeout: 1500', + 'cloud_compose_bootstrap_wait_seconds: 1200', +): + if marker not in cloud_smoke_inner: + fail(f"hosted adapter smoke retry contract is missing: {marker!r}") +for marker in ( + '/srv/cloud-compose/ci/remote/config-management-deploy-salt.sh', + ' ansible_first_mutation: - fail("Ansible host-input validation does not precede its first host mutation") -if "files/validate-runtime-inputs.py" not in ansible_tasks: - fail("Ansible does not execute the shared host-input validator") -if "--data-root" in ansible_tasks: - fail("Ansible makes the production project ownership boundary configurable") -if "cmd: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh" not in ansible_tasks: - fail("Ansible bypasses the retryable bootstrap service") -if "cmd: bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh" not in ansible_tasks: - fail("Ansible does not validate bootstrap readiness evidence") -if 'cmd: bash "{{ cloud_compose_home }}/run.sh"' in ansible_tasks: - fail("Ansible still invokes the one-shot bootstrap script directly") -if "cloud_compose_bootstrap_timeout: 14400" not in ansible_defaults: - fail("Ansible bootstrap timeout does not cover the bounded retryable bootstrap wait") - -salt_state = (repo_root / "salt/cloud-compose/init.sls").read_text() -cloud_smoke_driver = (repo_root / "ci/config-management-cloud-smoke.sh").read_text() -cloud_smoke_inner = (repo_root / "ci/config-management-cloud-smoke-inner.sh").read_text() -salt_gate = salt_state.find("cloud-compose-host-inputs-valid:") -salt_first_mutation = salt_state.find("cloud-compose-packages:") -if salt_gate < 0 or salt_first_mutation < 0 or salt_gate > salt_first_mutation: - fail("Salt host-input validation does not precede its first host mutation") -if "bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh" not in salt_state: - fail("Salt bypasses the retryable bootstrap service") -if "bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh" not in salt_state: - fail("Salt does not validate bootstrap readiness evidence") -if "home ~ '/run.sh'" in salt_state: - fail("Salt still invokes the one-shot bootstrap script directly") -gate_block = salt_state[salt_gate:salt_first_mutation] -for marker in ( - "cmd.run:", - '/usr/bin/env python3 "$CLOUD_COMPOSE_RUNTIME_VALIDATOR"', - "failhard: True", - "order: 2", -): - if marker not in gate_block: - fail(f"Salt host-input validation gate is missing {marker!r}") -for state_name in ("cloud-compose-packages:", "cloud-compose-docker-group:", "cloud-compose-group:"): - state_start = salt_state.find(state_name) - state_end = salt_state.find("\n\n", state_start) - if "cmd: cloud-compose-host-inputs-valid" not in salt_state[state_start:state_end]: - fail(f"Salt mutating state {state_name} does not require host-input validation") - -for marker in ( - "_cc_compose.init | default(cloud_compose_default_init)", - "_cc_compose.up | default(cloud_compose_default_up)", - "_cc_compose.down | default(cloud_compose_default_down)", - "_cc_compose.rollout | default(cloud_compose_default_rollout)", - "item.value.docker_compose_up | default(_cc_up_commands)", -): - if marker not in ansible_tasks: - fail(f"Ansible explicit-empty lifecycle parity marker is missing: {marker!r}") - -for marker in ( - "_cc_template.package_versions | default({})", - "_cc_template_sitectl_package_versions[item] | default(_cc_sitectl_version)", - "_cc_sitectl_package_version_overrides[item] | default(", - "_cc_sitectl.packages | default(_cc_template.packages)", - "item.value.sitectl_packages | default(_cc_sitectl_packages)", - "(_cc_managed.enabled | default(cloud_compose_managed_runtime_enabled)) is boolean", - "(_cc_vault.agent_enabled | default(false)) is boolean", - "(_cc_disaster_recovery.required | default(false)) is boolean", - "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED", - "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER", -): - if marker not in ansible_tasks: - fail(f"Ansible template package-version parity marker is missing: {marker!r}") - -for marker in ( - "configured_commands = compose.get(lifecycle)", - "configured_commands if lifecycle in compose else default_commands", - "if legacy_key in app", - "elif docker_key in app", -): - if marker not in salt_state: - fail(f"Salt explicit-empty lifecycle parity marker is missing: {marker!r}") - -for marker in ( - "template_sitectl_package_versions = template.get('package_versions', {})", - "sitectl_package_version_overrides.get(package, template_sitectl_package_versions.get(package, sitectl_version))", - "sitectl.get('packages') if 'packages' in sitectl else template.packages", - "app.get('sitectl_packages', sitectl_packages)", - "all_packages = sitectl_packages | list", - "'managed_runtime.enabled': managed_runtime_enabled", - "'vault.agent_enabled': vault.get('agent_enabled', False)", - "runtime_sections.disaster_recovery", - "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED", - "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER", - "run_bootstrap is sameas true", -): - if marker not in salt_state: - fail(f"Salt template package-version parity marker is missing: {marker!r}") - -for marker in ( - 'retry_remote_operation "Ansible deployment" deploy_ansible', - 'retry_remote_operation "Salt deployment" deploy_salt', - 'retry_remote_operation "Remote verification" verify_remote', - '-o ServerAliveInterval=30', - 'for attempt in 1 2 3; do', -): - if marker not in cloud_smoke_inner: - fail(f"hosted adapter smoke retry contract is missing: {marker!r}") - -for text in (cloud_smoke_driver, cloud_smoke_inner): - if "SMOKE_SSH_KEY_B64" in text: - fail("hosted adapter smoke exposes its SSH private key through the environment") -if 'type=bind,src=${key_path},dst=/run/secrets/cloud-compose-ssh-key,readonly' not in cloud_smoke_driver: - fail("hosted adapter smoke does not mount its SSH private key read-only") -if 'git -C "$repo_root" archive --format=tar HEAD' not in cloud_smoke_driver: - fail("hosted adapter smoke does not limit its source stream to committed files") -if 'tar \\\n --exclude="./.git"' in cloud_smoke_driver: - fail("hosted adapter smoke still archives the secret-bearing working tree") -for marker in ( - '[[ -L "$key_path" || ! -f "$key_path" ]]', - 'key_path="$(cd -P -- "$(dirname -- "$key_path")" && pwd)/$(basename -- "$key_path")"', -): - if marker not in cloud_smoke_driver: - fail(f"hosted adapter smoke key-source validation is missing: {marker!r}") -for marker in ( - 'readonly smoke_ssh_key_mount="/run/secrets/cloud-compose-ssh-key"', - 'install -m 0600 "$smoke_ssh_key_mount" /tmp/cloud-compose-ssh/id_ed25519', - '-i /tmp/cloud-compose-ssh/id_ed25519', - 'ansible_ssh_private_key_file: /tmp/cloud-compose-ssh/id_ed25519', -): - if marker not in cloud_smoke_inner: - fail(f"hosted adapter smoke key-file contract is missing: {marker!r}") - -for label, text, owner_marker, mode_marker in ( - ("Ansible", ansible_tasks, "owner: root", 'mode: "0640"'), - ("Salt", salt_state, "- user: root", "- mode: '0640'"), -): - if text.count(owner_marker) < 4 or text.count(mode_marker) < 4: - fail(f"{label} does not keep every root-consumed runtime input root-owned and mode 0640") - -if "no_log: true" not in ansible_tasks[ansible_tasks.find("Write Compose application environment data"):]: - fail("Ansible may expose application environment data in task output") -for state_name in ("cloud-compose-env:", "cloud-compose-application-env:", "cloud-compose-managed-runtime-artifacts:"): - state_start = salt_state.find(state_name) - state_end = salt_state.find("\n\n", state_start) - if "- show_changes: False" not in salt_state[state_start:state_end]: - fail(f"Salt sensitive state {state_name} may expose rendered data in state output") - -terraform_artifacts = (repo_root / "modules/managed-artifacts/main.tf").read_text() -validator_source = validator.read_text() -if 'PRODUCTION_DATA_ROOT = "/mnt/disks/data"' not in validator_source: - fail("configuration-management validator changed the fixed production data boundary") -if "--data-root" in salt_state: - fail("Salt makes the production project ownership boundary configurable") -if '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' not in terraform_artifacts: - fail("Terraform managed-artifact name bound changed without adapter parity") -if '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' not in validator_source: - fail("configuration-management validator is missing the 128-character artifact name bound") - -project_fixture_values = { - "invalid-project-dir-root": "project_dir: /", - "invalid-project-dir-etc": "project_dir: /etc", - "invalid-project-dir-traversal": "project_dir: /mnt/disks/data/../escape", - "invalid-project-dir-symlink": "project_dir: /mnt/disks/data/escape/site", -} -for fixture_name, marker in project_fixture_values.items(): - for relative_path in ( - f"tests/config-management/ansible/{fixture_name}.yml", - f"tests/config-management/salt-pillar/{fixture_name}.sls", - ): - fixture = repo_root / relative_path - if not fixture.is_file() or marker not in fixture.read_text(): - fail(f"project boundary fixture {relative_path} is missing {marker!r}") - -artifact_fixture_markers = ( - "../unsafe-name", - "a" * 129, - "http://example.invalid/unsafe-url", - "A" * 64, - "path: /", - "path: usr/local/bin/relative-path", - "path: /usr/local/../bin/dot-path", - "path: /usr//local/bin/empty-path-segment", - "bad\\u007fname", - 'mode: "4755"', - "owner: root:root", - "group: root:root", - "restart: ../docker.service", - "name: duplicate-name", - "path: /usr/local/bin/duplicate-path", -) -for relative_path in ( - "tests/config-management/ansible/invalid-artifacts.yml", - "tests/config-management/salt-pillar/invalid-artifacts.sls", -): - fixture_text = (repo_root / relative_path).read_text() - for marker in artifact_fixture_markers: - if marker not in fixture_text: - fail(f"managed artifact fixture {relative_path} is missing {marker!r}") - -adapter_fixture_markers = { - "invalid-ingress-port": "ingress_port: 80.5", - "invalid-project-port": "ingress_port: 443.5", - "invalid-lifecycle": "- 42", - "invalid-project-lifecycle": "docker_compose_up: not-a-list", - "invalid-internal-services": "internal_services_enabled: true", - "invalid-disaster-recovery": "driver_path: /etc/cloud-compose/libexec/../untrusted", -} -for fixture_name, marker in adapter_fixture_markers.items(): - for relative_path in ( - f"tests/config-management/ansible/{fixture_name}.yml", - f"tests/config-management/salt-pillar/{fixture_name}.sls", - ): - fixture = repo_root / relative_path - if not fixture.is_file() or marker not in fixture.read_text(): - fail(f"adapter parity fixture {relative_path} is missing {marker!r}") - -for relative_path in ( - "tests/config-management/ansible/smoke.yml", - "tests/config-management/salt-pillar/wp-prod.sls", -): - fixture_text = (repo_root / relative_path).read_text() - for marker in ("init: []", "docker_compose_up: []", "app-down-command"): - if marker not in fixture_text: - fail(f"explicit-empty lifecycle fixture {relative_path} is missing {marker!r}") - -print("Configuration-management input contracts passed") -PY +exec python3 "$repo_root/ci/config-management-input-contract.py" "$repo_root" "$validator" diff --git a/ci/config-management-runtime-state-contract.py b/ci/config-management-runtime-state-contract.py new file mode 100644 index 0000000..6ab03ad --- /dev/null +++ b/ci/config-management-runtime-state-contract.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +import json +import os +import subprocess +from pathlib import Path + + +def load_runtime_env(path: Path) -> dict[str, str]: + result = subprocess.run( + [ + "env", + "-i", + "PATH=/usr/bin:/bin", + f"CLOUD_COMPOSE_ENV_FILE={path}", + "bash", + "--noprofile", + "--norc", + "-c", + "source /home/cloud-compose/profile.sh; env -0", + ], + check=True, + stdout=subprocess.PIPE, + ) + return { + entry.split(b"=", 1)[0].decode(): entry.split(b"=", 1)[1].decode() + for entry in result.stdout.split(b"\0") + if b"=" in entry + } + + +name = os.environ["SMOKE_NAME"] +template = os.environ["SMOKE_TEMPLATE"] +project_dir = os.environ["SMOKE_PROJECT_DIR"] +runtime_env = load_runtime_env(Path("/home/cloud-compose/.env")) +projects = json.loads(Path("/home/cloud-compose/compose-projects.json").read_text()) +project = projects[name] + +assert runtime_env["CLOUD_COMPOSE_PROVIDER"] == "onprem" +assert runtime_env["CLOUD_COMPOSE_APPS"] == name +assert runtime_env["CLOUD_COMPOSE_PRIMARY_APP"] == name +assert runtime_env["SITECTL_PLUGIN"] == template +assert runtime_env["DOCKER_COMPOSE_DIR"] == project_dir +assert f"sitectl-{template}" in runtime_env["SITECTL_PACKAGES"].split() +assert project["docker_compose_repo"] == f"https://github.com/libops/{template}.git" +assert project["project_dir"] == project_dir +assert project["sitectl_plugin"] == template diff --git a/ci/config-management-smoke-assert.py b/ci/config-management-smoke-assert.py index fc777ab..f132cfe 100644 --- a/ci/config-management-smoke-assert.py +++ b/ci/config-management-smoke-assert.py @@ -47,6 +47,8 @@ def assert_runtime_files() -> None: RUNTIME_HOME / "up", RUNTIME_HOME / "down", RUNTIME_HOME / "rollout", + RUNTIME_HOME / "default-lifecycle.sh", + RUNTIME_HOME / "lifecycle-entrypoint.sh", RUNTIME_HOME / "run.sh", RUNTIME_HOME / "start-cloud-compose-bootstrap.sh", BOOTSTRAP_LIBEXEC / "bootstrap-required.sh", @@ -63,6 +65,7 @@ def assert_runtime_files() -> None: for path in [ JQ_PROGRAM_DIR / "diagnostics-validate-compose-projects.jq", JQ_PROGRAM_DIR / "offhost-validate-manifest.jq", + JQ_PROGRAM_DIR / "sitectl-verify-args.jq", ]: assert path.exists(), path @@ -113,6 +116,11 @@ def assert_runtime_files() -> None: assert diagnostics_metadata.st_gid == 0 assert stat.S_IMODE(diagnostics_metadata.st_mode) == 0o755 + lifecycle_metadata = (RUNTIME_HOME / "default-lifecycle.sh").stat() + assert lifecycle_metadata.st_uid == 0 + assert lifecycle_metadata.st_gid == 0 + assert stat.S_IMODE(lifecycle_metadata.st_mode) == 0o755 + jq_programs = list(JQ_PROGRAM_DIR.glob("*.jq")) assert jq_programs for path in jq_programs: diff --git a/ci/config-management-smoke-container.sh b/ci/config-management-smoke-container.sh new file mode 100755 index 0000000..bb5d1cf --- /dev/null +++ b/ci/config-management-smoke-container.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +install -d -m 0755 /work +tar -C /work -xf - +cd /work +exec bash ci/config-management-smoke-inner.sh diff --git a/ci/config-management-smoke-inner.sh b/ci/config-management-smoke-inner.sh index fbd89e0..88ead96 100755 --- a/ci/config-management-smoke-inner.sh +++ b/ci/config-management-smoke-inner.sh @@ -51,6 +51,7 @@ verify_lifecycle_lock_contract() { local lock_dir=/run/lock/cloud-compose local lock_file="$lock_dir/lifecycle.lock" local profile=/home/cloud-compose/profile.sh + local fixture=/work/ci/fixtures/config-management-lifecycle-lock.sh local ready=/tmp/cloud-compose-lifecycle-lock-ready local holder_pid contention_status passwd_sha @@ -62,14 +63,7 @@ verify_lifecycle_lock_contract() { rm -f -- "$ready" CLOUD_COMPOSE_ENV_FILE=/home/cloud-compose/.env \ - bash -c ' - set -euo pipefail - source "$1" - acquire_cloud_compose_lifecycle_lock root-first-contract - touch "$2" - sleep 3 - release_cloud_compose_lifecycle_lock - ' _ "$profile" "$ready" & + "$fixture" hold "$profile" "$ready" & holder_pid=$! for _ in {1..50}; do [[ -e "$ready" ]] && break @@ -82,10 +76,7 @@ verify_lifecycle_lock_contract() { HOME=/home/cloud-compose \ CLOUD_COMPOSE_ENV_FILE=/home/cloud-compose/.env \ CLOUD_COMPOSE_LIFECYCLE_LOCK_TIMEOUT_SECONDS=1 \ - bash -c ' - source "$1" - acquire_cloud_compose_lifecycle_lock contention-contract - ' _ "$profile" >/dev/null 2>&1 + "$fixture" contend "$profile" >/dev/null 2>&1 contention_status=$? set -e [[ "$contention_status" -ne 0 ]] @@ -94,20 +85,13 @@ verify_lifecycle_lock_contract() { runuser -u cloud-compose -- env \ HOME=/home/cloud-compose \ CLOUD_COMPOSE_ENV_FILE=/home/cloud-compose/.env \ - bash -c ' - set -euo pipefail - source "$1" - ( - acquire_cloud_compose_lifecycle_lock subshell-contract - release_cloud_compose_lifecycle_lock - ) - ' _ "$profile" + "$fixture" subshell "$profile" passwd_sha="$(sha256sum /etc/passwd)" mv -- "$lock_file" "${lock_file}.real" ln -s /etc/passwd "$lock_file" if CLOUD_COMPOSE_ENV_FILE=/home/cloud-compose/.env \ - bash -c 'source "$1"; acquire_cloud_compose_lifecycle_lock symlink-contract' _ "$profile" \ + "$fixture" reject-symlink "$profile" \ >/dev/null 2>&1; then echo "Lifecycle lock accepted a symbolic-link target" >&2 exit 1 diff --git a/ci/config-management-smoke.sh b/ci/config-management-smoke.sh index 42e356d..5794d72 100755 --- a/ci/config-management-smoke.sh +++ b/ci/config-management-smoke.sh @@ -6,6 +6,7 @@ repo_root="$(cd "$(dirname "$0")/.." && pwd)" # renovate: datasource=docker depName=python packageName=python versioning=docker CONFIG_MANAGEMENT_IMAGE_DEFAULT="python:3.11-slim@sha256:e031123e3d85762b141ad1cbc56452ba69c6e722ebf2f042cc0dc86c47c0d8b3" image="${CLOUD_COMPOSE_CONFIG_MANAGEMENT_IMAGE:-$CONFIG_MANAGEMENT_IMAGE_DEFAULT}" +container_entrypoint="$repo_root/ci/config-management-smoke-container.sh" require_cmd() { command -v "$1" >/dev/null 2>&1 || { @@ -16,6 +17,10 @@ require_cmd() { require_cmd docker require_cmd tar +[[ -f "$container_entrypoint" && ! -L "$container_entrypoint" ]] || { + echo "Config-management smoke container entrypoint is missing or unsafe" >&2 + exit 1 +} tar \ --exclude="./.git" \ @@ -24,7 +29,8 @@ tar \ -C "$repo_root" \ -cf - . | docker run --rm -i \ + --mount "type=bind,src=${container_entrypoint},dst=/usr/local/libexec/cloud-compose-config-management-smoke,readonly" \ --tmpfs /run \ --tmpfs /tmp \ "$image" \ - bash -lc 'mkdir -p /work && tar -C /work -xf - && cd /work && bash ci/config-management-smoke-inner.sh' + /usr/local/libexec/cloud-compose-config-management-smoke diff --git a/ci/filesystem-prep-contract.sh b/ci/filesystem-prep-contract.sh index a6f49bd..aff3b6a 100644 --- a/ci/filesystem-prep-contract.sh +++ b/ci/filesystem-prep-contract.sh @@ -5,6 +5,11 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" prep_script="$repo_root/rootfs/home/cloud-compose/prepare-filesystem.sh" persist_script="$repo_root/rootfs/home/cloud-compose/persist-filesystems.sh" +gcp_filesystem_boot="$repo_root/rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh" +gcp_filesystem_boothook="$repo_root/templates/gcp-filesystem-boothook.sh.tftpl" +gcp_cloud_init_mime="$repo_root/templates/gcp-cloud-init.mime.tftpl" +gcp_cloud_init_finalize="$repo_root/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh" +linux_cloud_init="$repo_root/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh" tmp="$(mktemp -d)" trap 'rm -rf -- "$tmp"' EXIT @@ -593,22 +598,30 @@ fi if grep -Eq 'fsck[^\n]*\|\|[^\n]*mkfs|fsck[^\n]*mkfs' "$repo_root/templates/cloud-init.yml"; then fail "GCP cloud-init still formats after an fsck failure" fi -grep -Fq 'FILESYSTEM_PREP_SCRIPT_B64' "$repo_root/templates/cloud-init.yml" || \ - fail "GCP cloud-init does not bootstrap the tested filesystem helper" +grep -Fq 'FILESYSTEM_PREP_SCRIPT_B64' "$gcp_filesystem_boothook" || \ + fail "GCP boothook does not bootstrap the tested filesystem helper" +grep -Fq 'bash "$filesystem_boot"' "$gcp_filesystem_boothook" || \ + fail "GCP boothook does not invoke the checked filesystem program" +grep -Fq 'Content-Type: text/cloud-boothook' "$gcp_cloud_init_mime" || \ + fail "GCP user data does not preserve an early every-boot filesystem phase" +if grep -Fq '/var/lib/cloud-compose/bootstrap/gcp-filesystem-boot.sh' \ + "$repo_root/templates/cloud-init.yml"; then + fail "GCP cloud-config invokes filesystem preparation late from runcmd" +fi grep -Fq 'bash "$filesystem_prep" /dev/disk/by-id/google-data /mnt/disks/data \' \ - "$repo_root/templates/cloud-init.yml" || + "$gcp_filesystem_boot" || fail "GCP cloud-init does not request a fresh marker for the data filesystem" -grep -Fq -- '--publish-fresh-marker ${jsonencode(FRESH_FILESYSTEM_IDENTITY)}' \ - "$repo_root/templates/cloud-init.yml" || +grep -Fq -- '--publish-fresh-marker "$fresh_filesystem_identity"' \ + "$gcp_filesystem_boot" || fail "GCP cloud-init does not bind the marker to the rendered data-disk identity" -grep -Fq 'bash "$filesystem_prep" '\''${DATA_DEVICE}'\'' /mnt/disks/data --publish-fresh-marker' \ - "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml" || +grep -Fq 'bash "$filesystem_prep" "$data_device" /mnt/disks/data --publish-fresh-marker' \ + "$linux_cloud_init" || fail "Linux VM cloud-init does not request a fresh marker for the data filesystem" -grep -Fq 'bash /run/cloud-compose-prepare-filesystem /dev/disk/by-id/google-docker-volumes /mnt/disks/volumes' \ - "$repo_root/templates/cloud-init.yml" || +grep -Fq 'bash "$filesystem_prep" /dev/disk/by-id/google-docker-volumes /mnt/disks/volumes' \ + "$gcp_filesystem_boot" || fail "GCP cloud-init does not prepare the volumes filesystem without a fresh marker" -grep -Fq 'bash "$filesystem_prep" '\''${VOLUMES_DEVICE}'\'' /mnt/disks/volumes' \ - "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml" || +grep -Fq 'bash "$filesystem_prep" "$volumes_device" /mnt/disks/volumes' \ + "$linux_cloud_init" || fail "Linux VM cloud-init does not prepare the volumes filesystem without a fresh marker" grep -Fq '(umask 077 && mkdir -- "$marker_dir")' "$prep_script" || fail "the fresh-filesystem marker directory is not created privately" @@ -623,41 +636,42 @@ grep -Fq -- '-L "$fresh_filesystem_pending_label" -- "$device"' "$prep_script" | grep -Fq 'e2label "$device" ""' "$prep_script" || fail "fresh-filesystem pending intent is not cleared after marker publication" if grep -Eq '^[[:space:]]*"?\$filesystem_(prep|persist)"?[[:space:]]' \ - "$repo_root/templates/cloud-init.yml" "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; then + "$gcp_filesystem_boot" "$linux_cloud_init"; then fail "cloud-init directly executes a temporary helper from potentially noexec /run" fi -for cloud_init_template in \ - "$repo_root/templates/cloud-init.yml" \ - "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do - [[ "$(grep -Fc -- '--publish-fresh-marker' "$cloud_init_template")" == "1" ]] || - fail "cloud-init must publish a fresh marker for the data filesystem only" - grep -Fq 'chown root:cloud-compose /mnt/disks/data' "$cloud_init_template" || - fail "cloud-init does not preserve root ownership of the data mount root" - grep -Fq 'chmod 1775 /mnt/disks/data' "$cloud_init_template" || - fail "cloud-init does not make the data mount root sticky and group-writable" - grep -Fq 'chown cloud-compose:cloud-compose /mnt/disks/volumes' "$cloud_init_template" || - fail "cloud-init does not assign the volumes mount root to cloud-compose" - grep -Fq 'chmod 0775 /mnt/disks/volumes' "$cloud_init_template" || - fail "cloud-init does not make the volumes mount root group-writable" +for filesystem_bootstrap in "$gcp_filesystem_boot" "$linux_cloud_init"; do + [[ "$(grep -Fc -- '--publish-fresh-marker' "$filesystem_bootstrap")" == "1" ]] || + fail "$filesystem_bootstrap must publish a fresh marker for the data filesystem only" grep -Fq 'for required_mount in /mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes; do' \ - "$cloud_init_template" || fail "cloud-init does not verify every required mount before initialization" - grep -Fq 'Required cloud-compose mount is unavailable:' "$cloud_init_template" || - fail "cloud-init mount gate does not report the unavailable path" - grep -Fq ' bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \ - "$cloud_init_template" || fail "cloud-init retryable bootstrap startup is outside the fail-closed mount block" - marker_reset_line="$(grep -nF ' rm -f /var/lib/cloud-compose/bootstrap-complete' \ - "$cloud_init_template" | cut -d: -f1)" - run_line="$(grep -nF ' bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \ - "$cloud_init_template" | cut -d: -f1)" + "$filesystem_bootstrap" || fail "$filesystem_bootstrap does not verify every required mount before initialization" + grep -Fq 'Required cloud-compose mount is unavailable:' "$filesystem_bootstrap" || + fail "$filesystem_bootstrap mount gate does not report the unavailable path" +done +for bootstrap_finalize in "$gcp_cloud_init_finalize" "$linux_cloud_init"; do + grep -Fq 'chown root:cloud-compose /mnt/disks/data' "$bootstrap_finalize" || + fail "$bootstrap_finalize does not preserve root ownership of the data mount root" + grep -Fq 'chmod 1775 /mnt/disks/data' "$bootstrap_finalize" || + fail "$bootstrap_finalize does not make the data mount root sticky and group-writable" + grep -Fq 'chown cloud-compose:cloud-compose /mnt/disks/volumes' "$bootstrap_finalize" || + fail "$bootstrap_finalize does not assign the volumes mount root to cloud-compose" + grep -Fq 'chmod 0775 /mnt/disks/volumes' "$bootstrap_finalize" || + fail "$bootstrap_finalize does not make the volumes mount root group-writable" + grep -Fq 'bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \ + "$bootstrap_finalize" || fail "$bootstrap_finalize does not start the retryable bootstrap service" + marker_reset_line="$(grep -nF 'rm -f /var/lib/cloud-compose/bootstrap-complete' \ + "$bootstrap_finalize" | cut -d: -f1)" + run_line="$(grep -nF 'bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \ + "$bootstrap_finalize" | cut -d: -f1)" [[ -n "$marker_reset_line" && -n "$run_line" && "$marker_reset_line" -lt "$run_line" ]] || - fail "cloud-init does not clear stale bootstrap readiness before retryable startup" + fail "$bootstrap_finalize does not clear stale bootstrap readiness before retryable startup" done grep -Fq 'install -m 0600 /dev/null /run/cloud-compose-filesystems-ready' \ - "$repo_root/templates/cloud-init.yml" || fail "GCP bootcmd does not publish filesystem readiness" -grep -Fq 'test -f /run/cloud-compose-filesystems-ready || {' \ - "$repo_root/templates/cloud-init.yml" || fail "GCP runcmd does not require filesystem readiness" -grep -Fq 'test -f /run/cloud-compose-filesystems-ready || {' \ - "$repo_root/modules/gcp/main.tf" || fail "GCP archive installation is not gated by filesystem readiness" + "$gcp_filesystem_boot" || fail "GCP boot program does not publish filesystem readiness" +grep -Fq '[[ ! -f /run/cloud-compose-filesystems-ready ]]' \ + "$gcp_cloud_init_finalize" || fail "GCP finalization does not require filesystem readiness" +grep -Fq 'fail "Cloud Compose filesystems were not prepared; refusing rootfs installation"' \ + "$repo_root/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh" || \ + fail "GCP archive installation is not gated by filesystem readiness" persist_tmp="$tmp/persist" mkdir -p "$persist_tmp/bin" @@ -779,11 +793,11 @@ if CLOUD_COMPOSE_FSTAB_PATH="$fstab" \ fail "filesystem persistence accepted an unsafe device path" fi -grep -Fq 'FILESYSTEM_PERSIST_SCRIPT_B64' "$repo_root/templates/cloud-init.yml" || \ - fail "GCP cloud-init does not bootstrap persistent mount configuration" -grep -Fq 'FILESYSTEM_PERSIST_SCRIPT_B64' "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml" || \ - fail "Linux VM cloud-init does not bootstrap persistent mount configuration" -grep -Fq '/var/lib/cloud-compose/mounted-rootfs/mnt/disks' "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml" || \ +grep -Fq 'FILESYSTEM_PERSIST_SCRIPT_B64' "$gcp_filesystem_boothook" || \ + fail "GCP boothook does not bootstrap persistent mount configuration" +grep -Fq 'bash "$filesystem_persist" "$data_device" "$volumes_device"' "$linux_cloud_init" || \ + fail "Linux VM cloud-init does not invoke persistent mount configuration" +grep -Fq '/var/lib/cloud-compose/mounted-rootfs/mnt/disks' "$linux_cloud_init" || \ fail "Linux VM cloud-init does not copy mounted-root files after mounting durable storage" do_module="$repo_root/modules/digitalocean/main.tf" diff --git a/ci/fixtures/config-management-lifecycle-lock.sh b/ci/fixtures/config-management-lifecycle-lock.sh new file mode 100755 index 0000000..3323baf --- /dev/null +++ b/ci/fixtures/config-management-lifecycle-lock.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -euo pipefail + +action="${1:-}" +profile="${2:-}" + +if [[ "$profile" != "/home/cloud-compose/profile.sh" || ! -f "$profile" || -L "$profile" ]]; then + echo "config-management lifecycle-lock fixture requires the checked runtime profile" >&2 + exit 2 +fi + +# shellcheck disable=SC1090 +source "$profile" + +case "$action" in + hold) + ready="${3:-}" + [[ "$#" -eq 3 && "$ready" == /tmp/cloud-compose-lifecycle-lock-ready ]] || { + echo "usage: config-management-lifecycle-lock.sh hold PROFILE READY" >&2 + exit 2 + } + acquire_cloud_compose_lifecycle_lock root-first-contract + touch -- "$ready" + sleep 3 + release_cloud_compose_lifecycle_lock + ;; + contend) + [[ "$#" -eq 2 ]] || { + echo "usage: config-management-lifecycle-lock.sh contend PROFILE" >&2 + exit 2 + } + acquire_cloud_compose_lifecycle_lock contention-contract + ;; + subshell) + [[ "$#" -eq 2 ]] || { + echo "usage: config-management-lifecycle-lock.sh subshell PROFILE" >&2 + exit 2 + } + ( + acquire_cloud_compose_lifecycle_lock subshell-contract + release_cloud_compose_lifecycle_lock + ) + ;; + reject-symlink) + [[ "$#" -eq 2 ]] || { + echo "usage: config-management-lifecycle-lock.sh reject-symlink PROFILE" >&2 + exit 2 + } + acquire_cloud_compose_lifecycle_lock symlink-contract + ;; + *) + echo "usage: config-management-lifecycle-lock.sh hold|contend|subshell|reject-symlink ..." >&2 + exit 2 + ;; +esac diff --git a/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh b/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh new file mode 100755 index 0000000..efd3c59 --- /dev/null +++ b/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -euo pipefail + +action="${1:-}" +runner="${2:-}" +[[ -f "$runner" && ! -L "$runner" ]] || { + echo "GCP upgrade contract harness requires the checked smoke runner" >&2 + exit 2 +} + +# shellcheck disable=SC1090 +source "$runner" + +case "$action" in + supported-cidrs) + [[ "$#" -eq 2 ]] + valid_direct_vpc_cidr "10.60.0.0/26" + valid_direct_vpc_cidr "172.20.0.0/24" + valid_direct_vpc_cidr "100.64.0.0/26" + valid_direct_vpc_cidr "240.0.0.0/26" + ;; + cidr) + [[ "$#" -eq 3 ]] + valid_direct_vpc_cidr "$3" + ;; + network-ownership) + [[ "$#" -eq 6 ]] + validate_upgrade_network_ownership "$3" "$4" "$5" "$6" + ;; + write-tfvars) + [[ "$#" -eq 3 ]] + write_tfvars "$3" name project us-east5 us-east5-b key 192.0.2.1/32 \ + project network subnet projects/project/roles/startVM \ + projects/project/roles/suspendVM true + ;; + cleanup-failure) + [[ "$#" -eq 3 ]] + repo_root="$3" + : "${CLEANUP_LOG:?CLEANUP_LOG is required}" + require_cmd() { :; } + require_env() { :; } + validate_upgrade_network() { :; } + cleanup_calls=0 + cleanup_resources() { + cleanup_calls=$((cleanup_calls + 1)) + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$@" >>"$CLEANUP_LOG" + [[ "$cleanup_calls" -gt 1 ]] + } + run_upgrade + ;; + *) + echo "usage: gcp-upgrade-smoke-contract-harness.sh supported-cidrs|cidr|network-ownership|write-tfvars|cleanup-failure RUNNER ..." >&2 + exit 2 + ;; +esac diff --git a/ci/fixtures/source-trust-rollout.sh b/ci/fixtures/source-trust-rollout.sh new file mode 100755 index 0000000..996408c --- /dev/null +++ b/ci/fixtures/source-trust-rollout.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +target_ref="${SOURCE_TRUST_ROLLOUT_REF:?SOURCE_TRUST_ROLLOUT_REF is required}" +if [[ ! "$target_ref" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ || + "$target_ref" == -* || "$target_ref" =~ (^|/)\.\.?(/|$) ]]; then + echo "Unsafe source-trust rollout ref: $target_ref" >&2 + exit 2 +fi + +git fetch -- origin "$target_ref" +git checkout --detach FETCH_HEAD diff --git a/ci/gcp-upgrade-smoke-contract.sh b/ci/gcp-upgrade-smoke-contract.sh index 1fcc795..d2a668b 100755 --- a/ci/gcp-upgrade-smoke-contract.sh +++ b/ci/gcp-upgrade-smoke-contract.sh @@ -24,10 +24,34 @@ script="$repo_root/ci/gcp-upgrade-smoke.sh" shared_smoke="$repo_root/ci/cloud-smoke.sh" fixture="$repo_root/tests/smoke/gcp-upgrade/main.tf" variables="$repo_root/tests/smoke/gcp-upgrade/variables.tf" +fixture_prepare="$repo_root/tests/smoke/gcp-upgrade/rootfs/home/cloud-compose/gcp-upgrade-prepare-repository.sh" +fixture_up="$repo_root/tests/smoke/gcp-upgrade/rootfs/etc/cloud-compose/lifecycle.d/gcp-upgrade-up.sh" +context_fixture="$repo_root/tests/smoke/modules/context/main.tf" +remote_services="$repo_root/ci/remote/gcp-upgrade-assert-services-disabled.sh" +remote_metadata="$repo_root/ci/remote/gcp-upgrade-verify-metadata-isolation.sh" +remote_container_metadata="$repo_root/ci/remote/gcp-upgrade-container-metadata-isolation.sh" +remote_sentinels="$repo_root/ci/remote/gcp-upgrade-write-disk-sentinels.sh" +remote_filesystem_size="$repo_root/ci/remote/gcp-upgrade-read-filesystem-size.sh" workflow="$repo_root/.github/workflows/cloud-smoke.yml" docs="$repo_root/docs/runtime-contracts.md" - -for required in "$script" "$shared_smoke" "$fixture" "$variables" "$workflow" "$docs"; do +contract_harness="$repo_root/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh" + +for required in \ + "$script" \ + "$shared_smoke" \ + "$fixture" \ + "$variables" \ + "$fixture_prepare" \ + "$fixture_up" \ + "$context_fixture" \ + "$remote_services" \ + "$remote_metadata" \ + "$remote_container_metadata" \ + "$remote_sentinels" \ + "$remote_filesystem_size" \ + "$contract_harness" \ + "$workflow" \ + "$docs"; do [[ -f "$required" ]] || fail "required file is missing: $required" done @@ -100,16 +124,20 @@ grep -Fq '/mnt/disks/data/.cloud-compose-upgrade-sentinel' "$script" || fail "upgrade runner omits the persistent data-disk sentinel" grep -Fq '/mnt/disks/volumes/.cloud-compose-upgrade-sentinel' "$script" || fail "upgrade runner omits the Docker-volume disk sentinel" -[[ "$(grep -Fc 'sudo tee /mnt/disks/data/.cloud-compose-upgrade-sentinel' "$script")" -eq 1 ]] || - fail "upgrade runner must write the data-disk sentinel exactly once" -[[ "$(grep -Fc 'sudo tee /mnt/disks/volumes/.cloud-compose-upgrade-sentinel' "$script")" -eq 1 ]] || - fail "upgrade runner must write the Docker-volume sentinel exactly once" +grep -Fq 'gcp-upgrade-write-disk-sentinels.sh' "$script" || + fail "upgrade runner does not stage the checked disk-sentinel program" +[[ "$(grep -Fc '>/mnt/disks/data/.cloud-compose-upgrade-sentinel' "$remote_sentinels")" -eq 1 ]] || + fail "checked remote program must write the data-disk sentinel exactly once" +[[ "$(grep -Fc '>/mnt/disks/volumes/.cloud-compose-upgrade-sentinel' "$remote_sentinels")" -eq 1 ]] || + fail "checked remote program must write the Docker-volume sentinel exactly once" +grep -Fq '[[ "$#" -ne 1 || ! "$1" =~ ^[A-Za-z0-9._:-]{1,128}$ ]]' "$remote_sentinels" || + fail "checked disk-sentinel program does not validate its nonce argument" grep -Fq 'read_data_filesystem_size_bytes()' "$script" || fail "upgrade runner does not measure mounted application-data filesystem capacity" -grep -Fq 'filesystem="$(findmnt -n -o FSTYPE --target /mnt/disks/data)"' "$script" || - fail "upgrade runner does not require the application-data mount to remain ext4" -grep -Fq 'df --block-size=1 --output=size -- /mnt/disks/data' "$script" || - fail "upgrade runner does not measure mounted ext4 capacity in bytes" +grep -Fq 'filesystem="$(findmnt -n -o FSTYPE --target /mnt/disks/data)"' "$remote_filesystem_size" || + fail "checked filesystem-size program does not require the application-data mount to remain ext4" +grep -Fq 'df --block-size=1 --output=size -- /mnt/disks/data' "$remote_filesystem_size" || + fail "checked filesystem-size program does not measure mounted ext4 capacity in bytes" grep -Fq '((10#$upgraded_data_filesystem_bytes > 10#$baseline_data_filesystem_bytes))' "$script" || fail "upgrade runner does not prove mounted ext4 grew beyond its baseline capacity" @@ -158,11 +186,28 @@ grep -Fq 'branch = var.wordpress_compose_ref' "$fixture" || grep -Fq 'projects = var.legacy_baseline ? {} : {' "$fixture" || fail "upgrade fixture does not transition from the legacy inputs to the current project map" for unit in internal-services.timer cloud-compose-internal-services.timer; do - grep -Fq "$unit" "$fixture" || fail "upgrade fixture does not disable ${unit}" + grep -Fq "$unit" "$fixture_prepare" || fail "upgrade fixture program does not disable ${unit}" done grep -Fq 'initcmd = [' "$fixture" || fail "upgrade fixture does not disable internal-service timers before bootstrap" -grep -Fq 'git -c safe.directory=\"$project\" -C \"$project\"' "$fixture" || +grep -Fq 'rootfs = "${path.module}/rootfs"' "$fixture" || + fail "upgrade fixture does not package its checked initialization program" +grep -Fq 'bash /home/cloud-compose/gcp-upgrade-prepare-repository.sh' "$fixture" || + fail "upgrade fixture does not invoke its checked initialization program" +grep -Fq '"/etc/cloud-compose/lifecycle.d/gcp-upgrade-up.sh"' "$fixture" || + fail "upgrade fixture does not invoke its checked lifecycle program" +grep -Fq '"/home/cloud-compose/default-lifecycle.sh up"' "$context_fixture" || + fail "hosted smoke context does not invoke the checked default lifecycle program" +grep -Fq 'sitectl compose --context "$context" up -d --remove-orphans' "$fixture_up" || + fail "checked upgrade lifecycle program does not bring up the exact sitectl context" +grep -Fq 'sitectl healthcheck --context "$context" --persist' "$fixture_up" || + fail "checked upgrade lifecycle program does not persist the post-start healthcheck" +[[ -x "$fixture_up" ]] || + fail "checked upgrade lifecycle program is not executable" +if grep -Eq '"sitectl (compose|healthcheck)' "$fixture" "$context_fixture"; then + fail "smoke fixtures still embed raw sitectl lifecycle commands in Terraform" +fi +grep -Fq 'git -c safe.directory="$project" -C "$project"' "$fixture_prepare" || fail "upgrade fixture does not explicitly trust its preserved pinned repository" if grep -Fq 'runcmd = [' "$fixture"; then fail "upgrade fixture defers its timer shutdown until after bootstrap" @@ -176,25 +221,41 @@ baseline_initcmd_line="$(grep -nF 'for CMD in ADDITIONAL_INITCMD' "$tmp/baseline current_bootstrap_line="$( grep -nF 'bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \ - "$repo_root/templates/cloud-init.yml" | + "$repo_root/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh" | cut -d: -f1 || true )" current_initcmd_line="$( - grep -nF 'for CMD in ADDITIONAL_INITCMD' "$repo_root/templates/cloud-init.yml" | + grep -nF 'source "$init_commands_file"' \ + "$repo_root/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh" | cut -d: -f1 || true )" [[ -n "$current_initcmd_line" && -n "$current_bootstrap_line" && "$current_initcmd_line" -lt "$current_bootstrap_line" ]] || - fail "current cloud-init does not execute fixture initcmd before retryable bootstrap" -grep -Fq 'for unit in internal-services.timer internal-services.service cloud-compose-internal-services.timer cloud-compose-internal-services.service' "$script" || - fail "upgrade runner does not assert that both timer generations remain inactive after boot" + fail "current checked-in cloud-init program does not execute fixture initcmd before retryable bootstrap" +for unit in internal-services.timer internal-services.service cloud-compose-internal-services.timer cloud-compose-internal-services.service; do + grep -Fq "$unit" "$remote_services" || + fail "checked service assertion does not cover ${unit}" +done +grep -Fq 'gcp-upgrade-assert-services-disabled.sh' "$script" || + fail "upgrade runner does not invoke the checked service assertion after boot" grep -Fq 'run_direct_vpc_cold_start "$new_home" "$key_path" "$new_output" "$name" "$zone"' "$script" || fail "upgrade runner does not exercise the upgraded Direct VPC cold-start path" grep -Fq 'verify_metadata_isolation "$new_home" "$key_path" "$new_output"' "$script" || fail "upgrade runner does not verify metadata isolation after the replacement boot" -grep -Fq 'nslookup metadata.google.internal' "$script" || +grep -Fq 'nslookup metadata.google.internal' "$remote_container_metadata" || fail "upgrade runner does not prove that Compute internal DNS survives metadata isolation" -grep -Fq 'sudo -u cloud-compose curl' "$script" || +grep -Fq 'for metadata_scheme in http https; do' "$remote_metadata" || fail "upgrade runner does not prove that unprivileged host metadata access is denied" +grep -Fq 'type=bind,src=${container_program},dst=/usr/local/bin/cloud-compose-metadata-isolation,readonly' "$remote_metadata" || + fail "metadata-isolation smoke does not mount its checked container program read-only" +grep -Fq '/bin/sh /usr/local/bin/cloud-compose-metadata-isolation' "$remote_metadata" || + fail "metadata-isolation smoke does not invoke its checked container program directly" +if grep -Eq 'base64[^[:space:]]*[[:space:]]*\|[[:space:]]*(sudo[[:space:]]+)?bash|ssh_cmd.*bash[[:space:]]+-(c|lc)' "$script"; then + fail "upgrade runner still sends an embedded Bash program to the remote host" +fi +grep -Fq 'remote_dir=/home/cloud-compose/.cache/libops-ci' "$script" || + fail "upgrade runner does not stage checked remote programs at a stable path" +grep -Fq 'install -m 0700 /dev/stdin $remote_path' "$script" || + fail "upgrade runner does not install checked remote programs with a private executable mode" grep -Fq -- '--header '\''X-Forwarded-For: 10.0.0.8'\''' "$script" || fail "Direct VPC smoke does not test an attacker-controlled forwarded prefix" grep -Fq -- '--max-time 600' "$script" || @@ -225,24 +286,20 @@ grep -Fq 'valid_direct_vpc_cidr "$subnet_cidr"' "$script" || fail "upgrade runner does not enforce Cloud Run Direct VPC supported IPv4 ranges" grep -Fq '((10#$subnet_prefix <= 26))' "$script" || fail "upgrade runner does not require a /26-or-larger persistent Direct VPC subnet" -bash -c ' - source "$1" - valid_direct_vpc_cidr "10.60.0.0/26" - valid_direct_vpc_cidr "172.20.0.0/24" - valid_direct_vpc_cidr "100.64.0.0/26" - valid_direct_vpc_cidr "240.0.0.0/26" -' _ "$script" || fail "upgrade runner rejected a supported Direct VPC IPv4 range" -if bash -c 'source "$1"; valid_direct_vpc_cidr "203.0.113.0/24"' _ "$script"; then +"$contract_harness" supported-cidrs "$script" || + fail "upgrade runner rejected a supported Direct VPC IPv4 range" +if "$contract_harness" cidr "$script" "203.0.113.0/24"; then fail "upgrade runner accepted an unsupported Direct VPC IPv4 range" fi -bash -c 'source "$1"; validate_upgrade_network_ownership "service-project" "service-project" "ci-network" "ci-subnet"' \ - _ "$script" || fail "upgrade runner rejected safe persistent-network ownership" -if bash -c 'source "$1"; validate_upgrade_network_ownership "service-project" "host-project" "ci-network" "ci-subnet"' \ - _ "$script" >/dev/null 2>&1; then +"$contract_harness" network-ownership "$script" \ + service-project service-project ci-network ci-subnet || + fail "upgrade runner rejected safe persistent-network ownership" +if "$contract_harness" network-ownership "$script" \ + service-project host-project ci-network ci-subnet >/dev/null 2>&1; then fail "upgrade runner accepted Shared VPC input unsupported by the baseline" fi -if bash -c 'source "$1"; validate_upgrade_network_ownership "service-project" "service-project" "cc-g-wp-owned" "ci-subnet"' \ - _ "$script" >/dev/null 2>&1; then +if "$contract_harness" network-ownership "$script" \ + service-project service-project cc-g-wp-owned ci-subnet >/dev/null 2>&1; then fail "upgrade runner placed its persistent network inside the disposable sweep boundary" fi network_validation_line="$(grep -nF ' validate_upgrade_network \' "$script" | cut -d: -f1)" @@ -256,23 +313,18 @@ wordpress_ref='5058610fddc7267ace92d65a5c49713dce570ac3' fail "upgrade fixture variable does not default to and enforce the pinned WordPress commit" grep -Fq 'docker_compose_branch = var.wordpress_compose_ref' "$fixture" || fail "upgrade fixture does not use its pinned WordPress commit" -grep -Fq "git_project checkout --detach \${var.wordpress_compose_ref}" "$fixture" || +grep -Fq 'git_project checkout --detach "$revision"' "$fixture_prepare" || fail "legacy bootstrap does not pre-check out the exact WordPress commit" grep -Eq 'wordpress_project_dir[[:space:]]*=[[:space:]]*"/mnt/disks/data/libops/wp.git/\$\{var\.wordpress_compose_ref\}"' "$fixture" || fail "upgrade fixture does not derive the exact legacy single-project checkout path" -grep -Fq 'project=${local.wordpress_project_dir};' "$fixture" || - fail "upgrade fixture does not pre-check out the shared baseline/current Compose path" +grep -Fq '[[ "$project" == "/mnt/disks/data/libops/wp.git/${revision}" ]]' "$fixture_prepare" || + fail "upgrade fixture program does not constrain the shared baseline/current Compose path" grep -Fq 'project_dir = local.wordpress_project_dir' "$fixture" || fail "baseline and current phases do not share the pinned Compose checkout path" tfvars_dir="$tmp/tfvars" mkdir -p "$tfvars_dir" -bash -c ' - set -euo pipefail - source "$1" - write_tfvars "$2" name project us-east5 us-east5-b key 192.0.2.1/32 \ - project network subnet projects/project/roles/startVM projects/project/roles/suspendVM true -' _ "$script" "$tfvars_dir" +"$contract_harness" write-tfvars "$script" "$tfvars_dir" jq -e '.legacy_baseline == true' "$tfvars_dir/upgrade.auto.tfvars.json" >/dev/null || fail "upgrade runner does not mark the old phase as a legacy baseline" @@ -290,21 +342,7 @@ GCLOUD_POWER_START_ROLE=projects/contract-project/roles/startVM \ GCLOUD_POWER_SUSPEND_ROLE=projects/contract-project/roles/suspendVM \ CLOUD_COMPOSE_GCP_UPGRADE_WORKDIR="$tmp/cleanup-work" \ CLEANUP_LOG="$cleanup_log" \ - bash -c ' - set -euo pipefail - source "$1" - repo_root="$2" - require_cmd() { :; } - require_env() { :; } - validate_upgrade_network() { :; } - cleanup_calls=0 - cleanup_resources() { - cleanup_calls=$((cleanup_calls + 1)) - printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$@" >>"$CLEANUP_LOG" - [[ "$cleanup_calls" -gt 1 ]] - } - run_upgrade - ' _ "$script" "$repo_root" + "$contract_harness" cleanup-failure "$script" "$repo_root" cleanup_status=$? set -e [[ "$cleanup_status" -eq 1 ]] || diff --git a/ci/gcp-upgrade-smoke.sh b/ci/gcp-upgrade-smoke.sh index f05c07e..23a0630 100755 --- a/ci/gcp-upgrade-smoke.sh +++ b/ci/gcp-upgrade-smoke.sh @@ -234,6 +234,22 @@ phase_output() { chmod 0600 "$output_json" } +stage_upgrade_remote_program() { + local home_dir="$1" key_path="$2" host="$3" port="$4" user="$5" + local source_path="$6" program_name="$7" + local remote_dir=/home/cloud-compose/.cache/libops-ci remote_path + + [[ "$program_name" =~ ^[a-z0-9][a-z0-9.-]*\.sh$ ]] || + fail "unsafe remote upgrade program name: $program_name" + [[ -f "$source_path" && ! -L "$source_path" ]] || + fail "remote upgrade program is missing or unsafe: $source_path" + remote_path="$remote_dir/$program_name" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "install -d -m 0700 $remote_dir && install -m 0700 /dev/stdin $remote_path" \ + <"$source_path" || return 1 + printf '%s\n' "$remote_path" +} + dump_remote_logs_from_output() { local home_dir="$1" key_path="$2" output_json="$3" local host port user project_dir @@ -247,7 +263,7 @@ dump_remote_logs_from_output() { run_phase_healthcheck() { local phase="$1" home_dir="$2" key_path="$3" output_json="$4" - local host port user project_dir + local host port user project_dir remote_program host="$(jq -er '.host' "$output_json")" port="$(jq -er '.ssh_port' "$output_json")" @@ -262,16 +278,11 @@ run_phase_healthcheck() { dump_remote_logs "$home_dir" "$key_path" "$host" "$port" "$user" "$project_dir" return 1 fi - ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set -euo pipefail -sudo systemctl disable --now internal-services.timer internal-services.service 2>/dev/null || true -sudo systemctl disable --now cloud-compose-internal-services.timer cloud-compose-internal-services.service 2>/dev/null || true -for unit in internal-services.timer internal-services.service cloud-compose-internal-services.timer cloud-compose-internal-services.service; do - if sudo systemctl is-active --quiet \"\$unit\" 2>/dev/null; then - echo \"Fixture-only internal service remained active: \$unit\" >&2 - exit 1 - fi -done -'" + remote_program="$(stage_upgrade_remote_program \ + "$home_dir" "$key_path" "$host" "$port" "$user" \ + "$repo_root/ci/remote/gcp-upgrade-assert-services-disabled.sh" \ + gcp-upgrade-assert-services-disabled.sh)" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "$remote_program" configure_sitectl_context "$home_dir" "$key_path" "$output_json" if ! run_healthcheck "$home_dir" "$key_path" "$output_json"; then dump_remote_logs "$home_dir" "$key_path" "$host" "$port" "$user" "$project_dir" @@ -357,76 +368,40 @@ run_direct_vpc_cold_start() { verify_metadata_isolation() { local home_dir="$1" key_path="$2" output_json="$3" - local host port user remote_script encoded_script quoted_script + local host port user remote_program container_program quoted_container_program host="$(jq -er '.host' "$output_json")" port="$(jq -er '.ssh_port' "$output_json")" user="$(jq -er '.ssh_user' "$output_json")" - read -r -d '' remote_script <<'EOF' || true -set -euo pipefail - -metadata_address="169.254.169.254" -metadata_header="Metadata-Flavor: Google" -alpine_image="alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce" - -# The default bridge must retain normal and Compute internal DNS through the -# metadata resolver, while metadata HTTP and HTTPS remain unreachable. -docker run --rm --network bridge "$alpine_image" /bin/sh -ec ' - nslookup dl-cdn.alpinelinux.org >/dev/null - nslookup metadata.google.internal | grep -Fq "169.254.169.254" - for metadata_port in 80 443; do - if nc -z -w 3 169.254.169.254 "$metadata_port"; then - echo "Container reached GCP metadata TCP port ${metadata_port}" >&2 - exit 1 - fi - done -' - -# Root retains the narrow access required by key rotation. -curl -fsS --connect-timeout 3 --max-time 10 \ - --header "$metadata_header" \ - "http://${metadata_address}/computeMetadata/v1/instance/id" >/dev/null - -# No unprivileged host process may reach either metadata transport. -if sudo -u cloud-compose curl -kfsS --connect-timeout 3 --max-time 5 \ - --header "$metadata_header" \ - "http://${metadata_address}/computeMetadata/v1/instance/id" >/dev/null 2>&1; then - echo "Unprivileged host process reached GCP metadata HTTP" >&2 - exit 1 -fi -if sudo -u cloud-compose curl -kfsS --connect-timeout 3 --max-time 5 \ - --header "$metadata_header" \ - "https://${metadata_address}/computeMetadata/v1/instance/id" >/dev/null 2>&1; then - echo "Unprivileged host process reached GCP metadata HTTPS" >&2 - exit 1 -fi -EOF - - encoded_script="$(printf '%s' "$remote_script" | base64 | tr -d '\n')" - quoted_script="$(shell_quote "$encoded_script")" + remote_program="$(stage_upgrade_remote_program \ + "$home_dir" "$key_path" "$host" "$port" "$user" \ + "$repo_root/ci/remote/gcp-upgrade-verify-metadata-isolation.sh" \ + gcp-upgrade-verify-metadata-isolation.sh)" + container_program="$(stage_upgrade_remote_program \ + "$home_dir" "$key_path" "$host" "$port" "$user" \ + "$repo_root/ci/remote/gcp-upgrade-container-metadata-isolation.sh" \ + gcp-upgrade-container-metadata-isolation.sh)" + quoted_container_program="$(shell_quote "$container_program")" echo "Verifying GCP DNS continuity and metadata isolation after the replacement boot" ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ - "printf %s ${quoted_script} | base64 -d | sudo bash" + "$remote_program ${quoted_container_program}" } write_disk_sentinels() { local home_dir="$1" key_path="$2" output_json="$3" nonce="$4" - local host port user encoded_nonce quoted_nonce + local host port user remote_program quoted_nonce host="$(jq -er '.host' "$output_json")" port="$(jq -er '.ssh_port' "$output_json")" user="$(jq -er '.ssh_user' "$output_json")" - encoded_nonce="$(printf '%s' "$nonce" | base64 | tr -d '\n')" - quoted_nonce="$(shell_quote "$encoded_nonce")" - - ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set -euo pipefail -sudo findmnt -n /mnt/disks/data >/dev/null -sudo findmnt -n /mnt/disks/volumes >/dev/null -printf %s ${quoted_nonce} | base64 -d | sudo tee /mnt/disks/data/.cloud-compose-upgrade-sentinel >/dev/null -printf %s ${quoted_nonce} | base64 -d | sudo tee /mnt/disks/volumes/.cloud-compose-upgrade-sentinel >/dev/null -sudo sync -'" + remote_program="$(stage_upgrade_remote_program \ + "$home_dir" "$key_path" "$host" "$port" "$user" \ + "$repo_root/ci/remote/gcp-upgrade-write-disk-sentinels.sh" \ + gcp-upgrade-write-disk-sentinels.sh)" + quoted_nonce="$(shell_quote "$nonce")" + ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "$remote_program ${quoted_nonce}" } verify_disk_sentinels() { @@ -437,9 +412,9 @@ verify_disk_sentinels() { port="$(jq -er '.ssh_port' "$output_json")" user="$(jq -er '.ssh_user' "$output_json")" actual_data="$(ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ - "sudo cat /mnt/disks/data/.cloud-compose-upgrade-sentinel")" + "cat /mnt/disks/data/.cloud-compose-upgrade-sentinel")" actual_volumes="$(ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ - "sudo cat /mnt/disks/volumes/.cloud-compose-upgrade-sentinel")" + "cat /mnt/disks/volumes/.cloud-compose-upgrade-sentinel")" [[ "$actual_data" == "$nonce" ]] || fail "the persistent data-disk sentinel did not survive the upgrade" [[ "$actual_volumes" == "$nonce" ]] || fail "the Docker-volume disk sentinel did not survive the upgrade" @@ -447,27 +422,18 @@ verify_disk_sentinels() { read_data_filesystem_size_bytes() { local home_dir="$1" key_path="$2" output_json="$3" - local host port user remote_script encoded_script quoted_script size_bytes + local host port user remote_program size_bytes host="$(jq -er '.host' "$output_json")" port="$(jq -er '.ssh_port' "$output_json")" user="$(jq -er '.ssh_user' "$output_json")" - read -r -d '' remote_script <<'EOF' || true -set -euo pipefail - -filesystem="$(findmnt -n -o FSTYPE --target /mnt/disks/data)" -if [[ "$filesystem" != "ext4" ]]; then - echo "Application-data mount uses ${filesystem:-an unknown filesystem}, expected ext4" >&2 - exit 1 -fi -df --block-size=1 --output=size -- /mnt/disks/data | awk 'NR == 2 { print $1 }' -EOF - - encoded_script="$(printf '%s' "$remote_script" | base64 | tr -d '\n')" - quoted_script="$(shell_quote "$encoded_script")" + remote_program="$(stage_upgrade_remote_program \ + "$home_dir" "$key_path" "$host" "$port" "$user" \ + "$repo_root/ci/remote/gcp-upgrade-read-filesystem-size.sh" \ + gcp-upgrade-read-filesystem-size.sh)" size_bytes="$(ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ - "printf %s ${quoted_script} | base64 -d | sudo bash")" + "$remote_program")" size_bytes="${size_bytes//$'\r'/}" [[ "$size_bytes" =~ ^[1-9][0-9]*$ ]] || fail "mounted application-data ext4 reported an invalid byte capacity: ${size_bytes:-empty}" @@ -740,7 +706,7 @@ run_upgrade() ( chmod 0600 "$old_state_json" "$old_state_list" write_phase_ids old "$old_state_json" "$old_ids" - nonce="$(printf '%s' "${run_id}:${base_sha}:${current_sha}" | sha256sum | awk '{print $1}')" + read -r nonce _ < <(printf '%s' "${run_id}:${base_sha}:${current_sha}" | sha256sum) write_disk_sentinels "$old_home" "$key_path" "$old_output" "$nonce" initialize_phase "$new_root" "$new_data" "$state_path" diff --git a/ci/host-runtime-security.sh b/ci/host-runtime-security.sh index 5b13cd1..998c9b9 100644 --- a/ci/host-runtime-security.sh +++ b/ci/host-runtime-security.sh @@ -243,47 +243,55 @@ if grep -Fq 'rotate-keys' "$repo_root/rootfs/home/cloud-compose/docker-prune.sh" fail "provider-neutral Docker prune still invokes GCP key rotation" fi -for archive_source in \ - "$repo_root/modules/linux-vm-runtime/main.tf" \ - "$repo_root/modules/gcp/main.tf"; do - verify_line="$(grep -n 'sha256sum -c -' "$archive_source" | head -n 1 | cut -d: -f1)" - extract_line="$(grep -n 'tar --no-same-owner -xzf "\$tmp/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" - normalize_line="$(grep -n 'chown -hR 0:0 -- "\$rootfs_dir"' "$archive_source" | head -n 1 | cut -d: -f1)" - copy_line="$(grep -n 'cp -a "\$rootfs_dir"/. /' "$archive_source" | head -n 1 | cut -d: -f1)" - [[ -n "$verify_line" && -n "$extract_line" && -n "$normalize_line" && -n "$copy_line" && - "$verify_line" -lt "$extract_line" && "$extract_line" -lt "$normalize_line" && - "$normalize_line" -lt "$copy_line" ]] || \ - fail "$archive_source does not verify, ownership-safe extract, then normalize the rootfs to root" -done +archive_source="$repo_root/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh" +verify_line="$(grep -n 'sha256sum -c -' "$archive_source" | head -n 1 | cut -d: -f1)" +members_line="$(grep -n 'validate_rootfs_archive "\$stage_root/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" +extract_line="$(grep -n 'tar --no-same-owner --same-permissions -xzf "\$stage_root/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" +contract_line="$(grep -n 'rootfs archive paths, bytes, or canonical metadata do not match this cloud-compose module source' "$archive_source" | head -n 1 | cut -d: -f1)" +copy_line="$(grep -n 'cp -a "\$staged_rootfs"/. /' "$archive_source" | head -n 1 | cut -d: -f1)" +[[ -n "$verify_line" && -n "$members_line" && -n "$extract_line" && -n "$contract_line" && -n "$copy_line" && + "$verify_line" -lt "$members_line" && "$members_line" -lt "$extract_line" && + "$extract_line" -lt "$contract_line" && "$contract_line" -lt "$copy_line" ]] || \ + fail "$archive_source does not verify archive bytes and canonical rootfs metadata before installation" +assert_contains "$archive_source" "stat -c '%a:%h:%F'" +assert_contains "$archive_source" '[[ "$metadata" == "${expected_mode}:1:regular file" ]]' +assert_contains "$archive_source" '[[ "$require_root_owner" != "true" || "$owner" == "0:0" ]]' if [[ -n "$(git -C "$repo_root" ls-files 'rootfs/usr/**')" ]]; then fail "Cloud Compose-owned rootfs programs still target immutable /usr" fi for trusted_program in \ rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh \ + rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh \ + rootfs/etc/cloud-compose/libexec/gcp-cloud-init-post-bootstrap.sh \ + rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh \ rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh \ rootfs/etc/cloud-compose/libexec/build-cos-make.sh \ rootfs/etc/cloud-compose/libexec/bootstrap-security.sh \ + rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh \ + rootfs/etc/cloud-compose/libexec/rootfs-archive.sh \ + rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh \ rootfs/etc/cloud-compose/libexec/run-bootstrap.sh \ rootfs/etc/cloud-compose/libexec/run-root-program.sh \ - rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq; do + rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq \ + rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq; do [[ -f "$repo_root/$trusted_program" ]] || fail "COS-safe trusted program is missing: $trusted_program" done -assert_contains "$repo_root/templates/cloud-init.yml" \ +assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh" \ '/etc/cloud-compose/libexec/harden-bootstrap-paths.sh' -assert_contains "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml" \ +assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh" \ '/etc/cloud-compose/libexec/harden-bootstrap-paths.sh' assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh" \ 'chown root:root "$cloud_compose_home"' assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh" \ "0:1:regular file" -for cloud_init_template in \ - "$repo_root/templates/cloud-init.yml" \ - "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do - assert_contains "$cloud_init_template" 'install -d -m 0755 -o root -g root /etc/cloud-compose/bin' - if grep -Eq '^[[:space:]]*- path: /usr/|install -d[^#]* /usr/local' "$cloud_init_template"; then - fail "$cloud_init_template writes Cloud Compose-owned programs beneath immutable /usr" +for cloud_init_program in \ + "$repo_root/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh" \ + "$repo_root/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh"; do + assert_contains "$cloud_init_program" 'install-diagnostics "$diagnostics_sha256"' + if grep -Eq 'install -d[^#]* /usr/local' "$cloud_init_program"; then + fail "$cloud_init_program writes Cloud Compose-owned programs beneath immutable /usr" fi done diff --git a/ci/lifecycle-program-contract.sh b/ci/lifecycle-program-contract.sh new file mode 100755 index 0000000..6d62224 --- /dev/null +++ b/ci/lifecycle-program-contract.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "${0##*/}" == "sitectl" ]]; then + : "${SITECTL_ARGV_LOG:?SITECTL_ARGV_LOG is required}" + { + printf '%s' "${1:-}" + shift || true + for argument in "$@"; do + printf '\t%s' "$argument" + done + printf '\n' + } >>"$SITECTL_ARGV_LOG" + exit 0 +fi + +lifecycle_program="${1:-/home/cloud-compose/default-lifecycle.sh}" +verify_args_program="${2:-/etc/cloud-compose/jq/sitectl-verify-args.jq}" +if [[ ! -f "$lifecycle_program" ]]; then + echo "lifecycle program contract: missing $lifecycle_program" >&2 + exit 1 +fi + +contract_dir="$(mktemp -d "${TMPDIR:-/tmp}/cloud-compose-lifecycle-program.XXXXXX")" +trap 'rm -rf -- "$contract_dir"' EXIT + +contract_program="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/$(basename -- "${BASH_SOURCE[0]}")" +ln -s -- "$contract_program" "$contract_dir/sitectl" +argv_log="$contract_dir/sitectl.argv" +target_ref="refs/pull/123/head" +commit_sha="0123456789abcdef0123456789abcdef01234567" +context="lifecycle-contract" + +env -u SITECTL_EXECUTABLE \ + PATH="$contract_dir:/usr/bin:/bin" \ + SITECTL_ARGV_LOG="$argv_log" \ + SITECTL_CONTEXT_NAME="$context" \ + SITECTL_ENVIRONMENT=production \ + GIT_COMMIT_SHA="$commit_sha" \ + GIT_REF="$target_ref" \ + GIT_BRANCH=ignored-branch \ + bash "$lifecycle_program" rollout + +expected_ref_call="deploy"$'\t'"--context"$'\t'"$context"$'\t'"--ref"$'\t'"$commit_sha" +grep -Fxq -- "$expected_ref_call" "$argv_log" || { + echo "lifecycle program contract: GIT_COMMIT_SHA did not take precedence for sitectl deploy --ref" >&2 + exit 1 +} +if grep -Fq -- $'deploy\t--context\tlifecycle-contract\t--skip-git' "$argv_log"; then + echo "lifecycle program contract: supplied GIT_REF incorrectly selected --skip-git" >&2 + exit 1 +fi + +: >"$argv_log" +env -u GIT_COMMIT_SHA -u SITECTL_EXECUTABLE \ + PATH="$contract_dir:/usr/bin:/bin" \ + SITECTL_ARGV_LOG="$argv_log" \ + SITECTL_CONTEXT_NAME="$context" \ + SITECTL_ENVIRONMENT=production \ + GIT_REF="$target_ref" \ + GIT_BRANCH=ignored-branch \ + bash "$lifecycle_program" rollout + +expected_ref_call="deploy"$'\t'"--context"$'\t'"$context"$'\t'"--ref"$'\t'"$target_ref" +grep -Fxq -- "$expected_ref_call" "$argv_log" || { + echo "lifecycle program contract: GIT_REF did not reach sitectl deploy --ref" >&2 + exit 1 +} + +: >"$argv_log" +if env -u SITECTL_EXECUTABLE \ + PATH="$contract_dir:/usr/bin:/bin" \ + SITECTL_ARGV_LOG="$argv_log" \ + SITECTL_CONTEXT_NAME="$context" \ + SITECTL_ENVIRONMENT=production \ + GIT_COMMIT_SHA=not-a-commit \ + GIT_REF="$target_ref" \ + bash "$lifecycle_program" rollout; then + echo "lifecycle program contract: malformed GIT_COMMIT_SHA was accepted" >&2 + exit 1 +fi +if [[ -s "$argv_log" ]]; then + echo "lifecycle program contract: malformed GIT_COMMIT_SHA reached sitectl" >&2 + exit 1 +fi + +: >"$argv_log" +env -u GIT_COMMIT_SHA -u GIT_REF -u GIT_BRANCH -u SITECTL_EXECUTABLE \ + PATH="$contract_dir:/usr/bin:/bin" \ + SITECTL_ARGV_LOG="$argv_log" \ + SITECTL_CONTEXT_NAME="$context" \ + SITECTL_ENVIRONMENT=production \ + bash "$lifecycle_program" rollout + +expected_skip_call="deploy"$'\t'"--context"$'\t'"$context"$'\t'"--skip-git" +grep -Fxq -- "$expected_skip_call" "$argv_log" || { + echo "lifecycle program contract: rollout without a ref did not select --skip-git" >&2 + exit 1 +} + +: >"$argv_log" +env -u SITECTL_EXECUTABLE \ + PATH="$contract_dir:/usr/bin:/bin" \ + SITECTL_ARGV_LOG="$argv_log" \ + SITECTL_CONTEXT_NAME="$context" \ + SITECTL_ENVIRONMENT=preview \ + SITECTL_VERIFY_ARGS_JSON='["--label","value with spaces",""]' \ + CLOUD_COMPOSE_SITECTL_VERIFY_ARGS_PROGRAM="$verify_args_program" \ + bash "$lifecycle_program" up + +expected_verify_call="verify"$'\t'"--context"$'\t'"$context"$'\t'"--label"$'\t'"value with spaces"$'\t' +grep -Fxq -- "$expected_verify_call" "$argv_log" || { + echo "lifecycle program contract: verify argument boundaries were not preserved" >&2 + exit 1 +} + +echo "Lifecycle program contract passed" diff --git a/ci/package-rootfs.sh b/ci/package-rootfs.sh index 6e3af96..07ba931 100755 --- a/ci/package-rootfs.sh +++ b/ci/package-rootfs.sh @@ -6,6 +6,7 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd -- "$script_dir/.." && pwd)" output_dir="${1:-$repo_root/dist}" asset_name="cloud-compose-rootfs.tar.gz" +contract_asset_name="cloud-compose-rootfs.contract.sha256" if [[ "$output_dir" != /* ]]; then output_dir="$PWD/$output_dir" @@ -14,13 +15,21 @@ install -d -m 0755 -- "$output_dir" tmp="$(mktemp -d "${TMPDIR:-/tmp}/cloud-compose-rootfs.XXXXXX")" trap 'rm -rf -- "$tmp"' EXIT +cp -a -- "$repo_root/rootfs" "$tmp/rootfs" +find "$tmp/rootfs" -depth -type d -empty -delete +find "$tmp/rootfs" -type d -exec chmod 0755 -- {} + +find "$tmp/rootfs" -type f -exec chmod 0644 -- {} + +find "$tmp/rootfs" -type f -name '*.sh' -exec chmod 0755 -- {} + +bash "$repo_root/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh" \ + contract "$tmp/rootfs" >"$tmp/$contract_asset_name" + LC_ALL=C tar \ --sort=name \ --mtime='UTC 1970-01-01' \ --owner=0 \ --group=0 \ --numeric-owner \ - -C "$repo_root" \ + -C "$tmp" \ -cf - rootfs | gzip -n -9 >"$tmp/$asset_name" ( @@ -29,3 +38,4 @@ LC_ALL=C tar \ ) install -m 0644 "$tmp/$asset_name" "$output_dir/$asset_name" install -m 0644 "$tmp/${asset_name}.sha256" "$output_dir/${asset_name}.sha256" +install -m 0644 "$tmp/$contract_asset_name" "$output_dir/$contract_asset_name" diff --git a/ci/remote/config-management-deploy-salt.sh b/ci/remote/config-management-deploy-salt.sh new file mode 100755 index 0000000..d8ce9a5 --- /dev/null +++ b/ci/remote/config-management-deploy-salt.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 4 ]]; then + echo "usage: config-management-deploy-salt.sh NAME TEMPLATE ENVIRONMENT PROJECT_DIR" >&2 + exit 2 +fi + +smoke_name="$1" +smoke_template="$2" +smoke_environment="$3" +smoke_project_dir="$4" + +for identifier in "$smoke_name" "$smoke_template" "$smoke_environment"; do + [[ "$identifier" =~ ^[a-z][a-z0-9-]*$ ]] || { + echo "Unsafe config-management smoke identifier: $identifier" >&2 + exit 2 + } +done +[[ "$smoke_project_dir" == /mnt/disks/data/libops/* && + "$smoke_project_dir" != *'//'* && + ! "$smoke_project_dir" =~ (^|/)\.\.?(/|$) ]] || { + echo "Unsafe config-management smoke project directory: $smoke_project_dir" >&2 + exit 2 +} + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends python3-venv ca-certificates + +python3 -m venv /opt/cloud-compose-salt-smoke +/opt/cloud-compose-salt-smoke/bin/python -m pip install --no-cache-dir \ + salt==3007.1 \ + tornado==6.4.2 \ + looseversion==1.3.0 \ + PyYAML==6.0.2 \ + packaging==24.2 \ + msgpack==1.1.0 \ + distro==1.9.0 \ + Jinja2==3.1.4 + +mkdir -p \ + /tmp/cloud-compose-salt/etc \ + /tmp/cloud-compose-salt/cache \ + /tmp/cloud-compose-salt/pki \ + /srv/cloud-compose/.smoke-pillar +cat >/tmp/cloud-compose-salt/etc/minion </srv/cloud-compose/.smoke-pillar/top.sls </srv/cloud-compose/.smoke-pillar/cloud-compose.sls </tmp/cloud-compose-salt-show-sls.txt + +/opt/cloud-compose-salt-smoke/bin/salt-call \ + --local \ + --retcode-passthrough \ + --config-dir=/tmp/cloud-compose-salt/etc \ + state.apply cloud-compose diff --git a/ci/remote/config-management-diagnostics.sh b/ci/remote/config-management-diagnostics.sh new file mode 100755 index 0000000..6f020b0 --- /dev/null +++ b/ci/remote/config-management-diagnostics.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set +e + +echo "--- cloud-init status ---" +cloud-init status --long +echo "--- /var/log/cloud-init-output.log ---" +tail -n 300 /var/log/cloud-init-output.log +echo "--- bootstrap unit state ---" +systemctl show --no-pager \ + --property=ActiveState,SubState,Result,NRestarts,ExecMainCode,ExecMainStatus \ + cloud-compose-bootstrap.service +echo "--- cloud-compose unit state ---" +systemctl show --no-pager \ + --property=ActiveState,SubState,Result,NRestarts,ExecMainCode,ExecMainStatus \ + cloud-compose.service +echo "--- cloud-compose bootstrap journal ---" +journalctl -u cloud-compose-bootstrap.service --no-pager -n 300 +echo "--- cloud-compose application journal ---" +journalctl -u cloud-compose.service --no-pager -n 300 +echo "--- active bootstrap processes ---" +ps -eo pid,ppid,stat,etime,args --forest +echo "--- docker ps ---" +docker ps -a +echo "--- compose manifest ---" +cat /home/cloud-compose/compose-projects.json diff --git a/ci/remote/config-management-verify.sh b/ci/remote/config-management-verify.sh new file mode 100755 index 0000000..192b31c --- /dev/null +++ b/ci/remote/config-management-verify.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 6 ]]; then + echo "usage: config-management-verify.sh NAME TEMPLATE ENVIRONMENT PROJECT_DIR LIFECYCLE_CONTRACT RUNTIME_STATE_CONTRACT" >&2 + exit 2 +fi + +export SMOKE_NAME="$1" +export SMOKE_TEMPLATE="$2" +export SMOKE_ENVIRONMENT="$3" +export SMOKE_PROJECT_DIR="$4" +readonly lifecycle_program_contract="$5" +readonly runtime_state_contract="$6" + +for contract_program in "$lifecycle_program_contract" "$runtime_state_contract"; do + [[ "$contract_program" == /tmp/cloud-compose-hosted-contract.*/* && + -f "$contract_program" && ! -L "$contract_program" ]] || { + echo "Remote contract program is missing or unsafe: $contract_program" >&2 + exit 1 + } +done + +for program in \ + /home/cloud-compose/init \ + /home/cloud-compose/up \ + /home/cloud-compose/down \ + /home/cloud-compose/rollout \ + /home/cloud-compose/default-lifecycle.sh \ + /home/cloud-compose/run.sh \ + /home/cloud-compose/start-cloud-compose-bootstrap.sh \ + /etc/cloud-compose/libexec/bootstrap-required.sh \ + /etc/cloud-compose/libexec/bootstrap-security.sh \ + /etc/cloud-compose/libexec/run-bootstrap.sh \ + /etc/cloud-compose/libexec/require-bootstrap-ready.sh \ + /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh \ + /etc/cloud-compose/libexec/run-root-program.sh; do + test -x "$program" +done + +python3 -m json.tool /home/cloud-compose/compose-projects.json >/dev/null +python3 -m json.tool /home/cloud-compose/application-env.json >/dev/null +python3 "$runtime_state_contract" +bash "$lifecycle_program_contract" /home/cloud-compose/default-lifecycle.sh + +test -d "$SMOKE_PROJECT_DIR/.git" +systemctl is-active --quiet cloud-compose +runuser -u cloud-compose -- env HOME=/home/cloud-compose \ + /home/cloud-compose/smoke-healthcheck.sh "$SMOKE_NAME" diff --git a/ci/remote/gcp-upgrade-assert-services-disabled.sh b/ci/remote/gcp-upgrade-assert-services-disabled.sh new file mode 100755 index 0000000..6a48104 --- /dev/null +++ b/ci/remote/gcp-upgrade-assert-services-disabled.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -euo pipefail + +for unit in \ + internal-services.timer \ + internal-services.service \ + cloud-compose-internal-services.timer \ + cloud-compose-internal-services.service; do + if systemctl is-active --quiet "$unit" 2>/dev/null; then + echo "Fixture-only internal service remained active: $unit" >&2 + exit 1 + fi +done diff --git a/ci/remote/gcp-upgrade-container-metadata-isolation.sh b/ci/remote/gcp-upgrade-container-metadata-isolation.sh new file mode 100755 index 0000000..3fa1b83 --- /dev/null +++ b/ci/remote/gcp-upgrade-container-metadata-isolation.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -eu + +nslookup dl-cdn.alpinelinux.org >/dev/null +nslookup metadata.google.internal | grep -Fq "169.254.169.254" +for metadata_port in 80 443; do + if nc -z -w 3 169.254.169.254 "$metadata_port"; then + echo "Container reached GCP metadata TCP port ${metadata_port}" >&2 + exit 1 + fi +done diff --git a/ci/remote/gcp-upgrade-read-filesystem-size.sh b/ci/remote/gcp-upgrade-read-filesystem-size.sh new file mode 100755 index 0000000..c2bbbd6 --- /dev/null +++ b/ci/remote/gcp-upgrade-read-filesystem-size.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +filesystem="$(findmnt -n -o FSTYPE --target /mnt/disks/data)" +if [[ "$filesystem" != "ext4" ]]; then + echo "Application-data mount uses ${filesystem:-an unknown filesystem}, expected ext4" >&2 + exit 1 +fi +{ + read -r _header + read -r size_bytes + [[ "$size_bytes" =~ ^[1-9][0-9]*$ ]] + printf '%s\n' "$size_bytes" +} < <(df --block-size=1 --output=size -- /mnt/disks/data) diff --git a/ci/remote/gcp-upgrade-verify-metadata-isolation.sh b/ci/remote/gcp-upgrade-verify-metadata-isolation.sh new file mode 100755 index 0000000..903be2e --- /dev/null +++ b/ci/remote/gcp-upgrade-verify-metadata-isolation.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 1 ]]; then + echo "usage: gcp-upgrade-verify-metadata-isolation.sh CONTAINER_PROGRAM" >&2 + exit 2 +fi + +container_program="$1" +metadata_address="169.254.169.254" +metadata_header="Metadata-Flavor: Google" +alpine_image="alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce" + +[[ "$container_program" == /home/cloud-compose/.cache/libops-ci/gcp-upgrade-container-metadata-isolation.sh && + -f "$container_program" && ! -L "$container_program" ]] || { + echo "Container metadata-isolation program is missing or unsafe" >&2 + exit 1 +} + +docker run --rm --network bridge \ + --mount "type=bind,src=${container_program},dst=/usr/local/bin/cloud-compose-metadata-isolation,readonly" \ + "$alpine_image" \ + /bin/sh /usr/local/bin/cloud-compose-metadata-isolation + +for metadata_scheme in http https; do + if curl -kfsS --connect-timeout 3 --max-time 5 \ + --header "$metadata_header" \ + "${metadata_scheme}://${metadata_address}/computeMetadata/v1/instance/id" \ + >/dev/null 2>&1; then + echo "Unprivileged host process reached GCP metadata ${metadata_scheme}" >&2 + exit 1 + fi +done diff --git a/ci/remote/gcp-upgrade-write-disk-sentinels.sh b/ci/remote/gcp-upgrade-write-disk-sentinels.sh new file mode 100755 index 0000000..c5b35d2 --- /dev/null +++ b/ci/remote/gcp-upgrade-write-disk-sentinels.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 1 || ! "$1" =~ ^[A-Za-z0-9._:-]{1,128}$ ]]; then + echo "usage: gcp-upgrade-write-disk-sentinels.sh SAFE_NONCE" >&2 + exit 2 +fi + +findmnt -n /mnt/disks/data >/dev/null +findmnt -n /mnt/disks/volumes >/dev/null +printf '%s' "$1" >/mnt/disks/data/.cloud-compose-upgrade-sentinel +printf '%s' "$1" >/mnt/disks/volumes/.cloud-compose-upgrade-sentinel +sync diff --git a/ci/rootfs-archive-path-contract.awk b/ci/rootfs-archive-path-contract.awk new file mode 100644 index 0000000..9a2783f --- /dev/null +++ b/ci/rootfs-archive-path-contract.awk @@ -0,0 +1,3 @@ +$0 !~ /^rootfs\// { bad = 1 } +/(^|\/)\.\.($|\/)/ { bad = 1 } +END { exit bad } diff --git a/ci/rootfs-package-contract.sh b/ci/rootfs-package-contract.sh index 19a1a6d..0c8ad1a 100755 --- a/ci/rootfs-package-contract.sh +++ b/ci/rootfs-package-contract.sh @@ -10,16 +10,42 @@ bash "$script_dir/package-rootfs.sh" "$tmp/one" bash "$script_dir/package-rootfs.sh" "$tmp/two" cmp "$tmp/one/cloud-compose-rootfs.tar.gz" "$tmp/two/cloud-compose-rootfs.tar.gz" cmp "$tmp/one/cloud-compose-rootfs.tar.gz.sha256" "$tmp/two/cloud-compose-rootfs.tar.gz.sha256" +cmp "$tmp/one/cloud-compose-rootfs.contract.sha256" "$tmp/two/cloud-compose-rootfs.contract.sha256" ( cd "$tmp/one" sha256sum -c cloud-compose-rootfs.tar.gz.sha256 ) -tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | awk ' - $0 !~ /^rootfs\// { bad = 1 } - /(^|\/)\.\.($|\/)/ { bad = 1 } - END { exit bad } -' +contract_sha256="$(<"$tmp/one/cloud-compose-rootfs.contract.sha256")" +[[ "$contract_sha256" =~ ^[0-9a-f]{64}$ ]] || { + echo "Rootfs contract sidecar is not one lowercase SHA-256 digest" >&2 + exit 1 +} +mkdir "$tmp/extracted" +tar -xzf "$tmp/one/cloud-compose-rootfs.tar.gz" -C "$tmp/extracted" +actual_contract_sha256="$( + bash "$script_dir/../rootfs/etc/cloud-compose/libexec/rootfs-archive.sh" \ + contract "$tmp/extracted/rootfs" +)" +[[ "$actual_contract_sha256" == "$contract_sha256" ]] || { + echo "Rootfs contract sidecar does not match the packaged rootfs" >&2 + exit 1 +} +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ + awk -f "$script_dir/rootfs-archive-path-contract.awk" tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/home/cloud-compose/run.sh' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/home/cloud-compose/default-lifecycle.sh' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/home/cloud-compose/lifecycle-entrypoint.sh' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq' >/dev/null +for bootstrap_program in \ + gcp-cloud-init-finalize.sh \ + gcp-cloud-init-post-bootstrap.sh \ + gcp-filesystem-boot.sh \ + linux-vm-cloud-init.sh \ + rootfs-archive.sh \ + run-lifecycle-program.sh; do + tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ + grep -Fx "rootfs/etc/cloud-compose/libexec/$bootstrap_program" >/dev/null +done tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ grep -Fx 'rootfs/etc/cloud-compose/libexec/run-bootstrap.sh' >/dev/null tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \ diff --git a/ci/source-trust-contract.sh b/ci/source-trust-contract.sh index 5c3c5cb..60a6b0d 100644 --- a/ci/source-trust-contract.sh +++ b/ci/source-trust-contract.sh @@ -59,6 +59,8 @@ export COMPOSE_PROJECTS_FILE="$projects_file" export COMPOSE_APPS_ENV_DIR="$tmp/apps" export COMPOSE_APPS_STATE_DIR="$tmp/state" export CLOUD_COMPOSE_DATA_ROOT="$tmp" +export CLOUD_COMPOSE_LIFECYCLE_PROGRAM_DIR="$repo_root/ci/fixtures" +readonly source_trust_rollout_program="$repo_root/ci/fixtures/source-trust-rollout.sh" retry_until_success() { "$@" @@ -135,34 +137,33 @@ git -C "$source_repo" commit -m feature >/dev/null feature_commit="$(git -C "$source_repo" rev-parse HEAD)" git -C "$source_repo" push origin feature >/dev/null git -C "$source_repo" checkout main >/dev/null -jq '.branch.rollout_commands = [ - "git fetch -- origin feature", - "git checkout --detach FETCH_HEAD" - ] | .branch.up_commands = ["true"]' \ +jq --arg rollout_program "$source_trust_rollout_program" \ + '.branch.rollout_commands = [$rollout_program] | .branch.up_commands = ["true"]' \ "$projects_file" >"$projects_file.tmp" mv "$projects_file.tmp" "$projects_file" +export SOURCE_TRUST_ROLLOUT_REF=feature run_compose_app_lifecycle branch rollout assert_head "$branch_checkout" "$feature_commit" [[ "$(<"$COMPOSE_APPS_STATE_DIR/branch.deployed-head")" == "$feature_commit" ]] || \ fail "feature rollout HEAD was not recorded" run_compose_app_lifecycle branch up assert_head "$branch_checkout" "$feature_commit" -jq '.branch.rollout_commands = [ - "git fetch -- origin main", - "git checkout --detach FETCH_HEAD" - ]' "$projects_file" >"$projects_file.tmp" +jq --arg rollout_program "$source_trust_rollout_program" \ + '.branch.rollout_commands = [$rollout_program]' \ + "$projects_file" >"$projects_file.tmp" mv "$projects_file.tmp" "$projects_file" +export SOURCE_TRUST_ROLLOUT_REF=main run_compose_app_lifecycle branch rollout assert_head "$branch_checkout" "$commit_four" # Full bootstrap/source preparation may restore the configured baseline after # a recorded rollout. It must not grant the same reset authority to an # unrecorded local-ahead commit. -jq '.branch.rollout_commands = [ - "git fetch -- origin feature", - "git checkout --detach FETCH_HEAD" - ]' "$projects_file" >"$projects_file.tmp" +jq --arg rollout_program "$source_trust_rollout_program" \ + '.branch.rollout_commands = [$rollout_program]' \ + "$projects_file" >"$projects_file.tmp" mv "$projects_file.tmp" "$projects_file" +export SOURCE_TRUST_ROLLOUT_REF=feature run_compose_app_lifecycle branch rollout assert_head "$branch_checkout" "$feature_commit" clone_or_update_compose_app branch @@ -281,20 +282,125 @@ if grep -Eq 'source = "https://github\.com/.*/archive/(refs/)?(heads|tags)/' "$g fi linux_runtime="$repo_root/modules/linux-vm-runtime/main.tf" -grep -Fq "archive_url_b64='\${base64encode(local.rootfs_archive_url)}'" "$linux_runtime" || \ - fail "Linux rootfs archive URL is not rendered as base64 shell data" -if grep -Fq 'archive_url=${jsonencode(local.rootfs_archive_url)}' "$linux_runtime"; then - fail "Linux rootfs archive URL is still rendered as executable shell syntax" -fi -grep -Fq 'archive_additional_rootfs_commands' "$linux_runtime" || \ - fail "Additional rootfs content is not reapplied after archive extraction" +linux_runtime_outputs="$repo_root/modules/linux-vm-runtime/outputs.tf" +linux_runtime_variables="$repo_root/modules/linux-vm-runtime/variables.tf" +linux_runtime_tests="$repo_root/modules/linux-vm-runtime/runtime_inputs.tftest.hcl" +gcp_cloud_init="$repo_root/templates/cloud-init.yml" +linux_cloud_init="$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml" +archive_program="$repo_root/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh" + +grep -Eq 'ROOTFS_ARCHIVE_URL_B64[[:space:]]*=[[:space:]]*base64encode\(local\.rootfs_archive_url\)' "$linux_runtime" || \ + fail "Linux rootfs archive URL is not transported as base64 data" +grep -Fq 'count = local.rootfs_archive_url != "" && local.rootfs_test_source_archive_prefix == "" ? 1 : 0' "$linux_runtime" || \ + fail "Linux production archive mode does not keep the release sidecar mandatory" +grep -Fq 'variable "rootfs_test_source_archive_prefix"' "$linux_runtime_variables" || \ + fail "Linux hosted smoke source mode is not unmistakably test-only" +grep -Fq 'regex("^cloud-compose-[0-9a-f]{40}$"' "$linux_runtime_variables" || \ + fail "Linux hosted smoke source mode does not require one exact lowercase commit SHA" +grep -Fq 'https://github.com/libops/cloud-compose/archive/${trimprefix(local.rootfs_test_source_archive_prefix, "cloud-compose-")}.tar.gz' "$linux_runtime_outputs" || \ + fail "Linux hosted smoke source mode does not bind its URL to the exact prefix commit" +for negative_contract in \ + rejects_source_archive_from_another_commit \ + rejects_tag_named_source_archive_prefix \ + rejects_arbitrary_test_source_archive_url; do + grep -Fq "run \"${negative_contract}\"" "$linux_runtime_tests" || \ + fail "Linux hosted smoke source mode lacks ${negative_contract} coverage" +done +for production_surface in \ + "$repo_root/variables.tf" \ + "$repo_root/modules/gcp/variables.tf" \ + "$repo_root/providers/gcp/variables.tf" \ + "$repo_root/providers/do/variables.tf" \ + "$repo_root/providers/linode/variables.tf"; do + if grep -Fq 'rootfs_test_source_archive_prefix' "$production_surface"; then + fail "$production_surface exposes the test-only hosted source mode" + fi +done +grep -Fq 'ROOTFS_ARCHIVE_URL_B64' "$linux_cloud_init" || \ + fail "Linux cloud-init does not pass rootfs archive URL data to its checked-in entrypoint" +grep -Fq 'prepare-linux-test-source' "$linux_cloud_init" || \ + fail "Linux hosted smoke cannot invoke the checked source-archive fixture path" +grep -Fq 'rootfs_overlay_staging_path' "$linux_runtime" || \ + fail "Additional rootfs content is not staged for reapplication after archive extraction" for runtime_module in "$linux_runtime" "$gcp_module"; do - grep -Fq -- "curl -fsSL --proto '=https' --proto-redir '=https' --tlsv1.2" "$runtime_module" || \ - fail "rootfs archive download is not restricted to HTTPS with TLS 1.2 or newer in $runtime_module" - grep -Fq -- '--connect-timeout 10 --max-time 300 -o "$tmp/rootfs.tar.gz" -- "$archive_url"' "$runtime_module" || \ - fail "rootfs archive download is not bounded or separated from curl options in $runtime_module" - grep -Fq 'rootfs_dir="$(find "$tmp" -mindepth 1 -maxdepth 3 -type d -name rootfs -print -quit)"' "$runtime_module" || \ - fail "rootfs archive discovery does not accept the documented depth range in $runtime_module" + grep -Fq 'ROOTFS_ARCHIVE_SCRIPT_B64' "$runtime_module" || \ + fail "$runtime_module does not transfer the checked-in rootfs archive program" + grep -Fq 'rootfs_contract_sha256 = sha256(join("", [' "$runtime_module" || \ + fail "$runtime_module does not bind archive contents to its exact bundled rootfs" + grep -Fq 'cloud-compose-rootfs.contract.sha256' "$runtime_module" || \ + fail "$runtime_module does not derive the immutable rootfs contract sidecar" + grep -Fq 'data "http" "rootfs_contract"' "$runtime_module" || \ + fail "$runtime_module does not verify the release contract during planning" + if grep -Fq -- "curl -fsSL --proto '=https'" "$runtime_module"; then + fail "$runtime_module still embeds the rootfs archive shell implementation" + fi +done +for cloud_init_template in "$linux_cloud_init" "$gcp_cloud_init"; do + grep -Fq '/var/lib/cloud-compose/bootstrap/rootfs-archive.sh' "$cloud_init_template" || \ + fail "$cloud_init_template does not install or invoke the checked-in rootfs archive program" +done +grep -Fq -- "curl -fsSL --proto '=https' --proto-redir '=https' --tlsv1.2" "$archive_program" || \ + fail "rootfs archive download is not restricted to HTTPS with TLS 1.2 or newer" +grep -Fq -- '--connect-timeout 10 --max-time 300' "$archive_program" || \ + fail "rootfs archive download is not bounded" +grep -Fq -- '-o "$stage_root/rootfs.tar.gz" -- "$archive_url"' "$archive_program" || \ + fail "rootfs archive URL is not separated from curl options" +grep -Fq 'validate_rootfs_archive "$stage_root/rootfs.tar.gz"' "$archive_program" || \ + fail "rootfs archive members are not validated before extraction" +grep -Fq 'validate_rootfs_test_source_archive "$stage_root/rootfs.tar.gz" "$test_source_prefix"' "$archive_program" || \ + fail "hosted smoke source-archive members are not validated before extraction" +grep -Fq 'https://github.com/libops/cloud-compose/archive/${source_commit}.tar.gz' "$archive_program" || \ + fail "hosted smoke source archives are not tied to one exact libops/cloud-compose commit" +grep -Fq '[[ "$member_type" == "-" || "$member_type" == "d" ]]' "$archive_program" || \ + fail "rootfs archive validation does not reject links before extraction" +grep -Fq 'rootfs archive paths, bytes, or canonical metadata do not match this cloud-compose module source' "$archive_program" || \ + fail "rootfs archive extraction does not reject module/archive content mismatches" +grep -Fq "stat -c '%a:%h:%F'" "$archive_program" || \ + fail "rootfs archive contract does not reject noncanonical modes or hard links" +for embedded_source in "$linux_runtime" "$gcp_module" "$linux_cloud_init" "$gcp_cloud_init"; do + if grep -Eq 'sha256sum -c -|tar --no-same-owner|rootfs_dir=\"\$\(find' "$embedded_source"; then + fail "$embedded_source still embeds the substantive rootfs archive program" + fi +done + +cloud_smoke="$repo_root/ci/cloud-smoke.sh" +cloud_smoke_workflow="$repo_root/.github/workflows/cloud-smoke.yml" +for fixture in "$repo_root/tests/smoke/do/main.tf" "$repo_root/tests/smoke/linode/main.tf"; do + grep -Fq 'rootfs_test_source_archive_prefix = "cloud-compose-${var.cloud_compose_source_ref}"' "$fixture" || \ + fail "$fixture does not select the explicit exact-commit source-archive fixture mode" +done +grep -Fq 'source = "../../../modules/digitalocean"' "$repo_root/tests/smoke/do/main.tf" || \ + fail "DigitalOcean hosted smoke does not keep source-archive mode below the public provider entrypoint" +grep -Fq 'source = "../../../modules/linode"' "$repo_root/tests/smoke/linode/main.tf" || \ + fail "Linode hosted smoke does not keep source-archive mode below the public provider entrypoint" +for example_name in digitalocean linode; do + example_main="$repo_root/examples/$example_name/main.tf" + example_variables="$repo_root/examples/$example_name/variables.tf" + source_ref_block="$(sed -n '/^variable "cloud_compose_source_ref" {/,/^}/p' "$example_variables")" + source_sha_block="$(sed -n '/^variable "cloud_compose_source_sha256" {/,/^}/p' "$example_variables")" + if grep -Fq 'default' <<<"$source_ref_block" || grep -Fq 'default' <<<"$source_sha_block"; then + fail "$example_name runnable example still defaults to an obsolete cloud-compose release" + fi + grep -Fq 'releases/download/${var.cloud_compose_source_ref}/cloud-compose-rootfs.tar.gz' "$example_main" || \ + fail "$example_name runnable example does not derive its canonical archive from the required exact release" + grep -Fq 'rootfs_archive_sha256 = var.cloud_compose_source_sha256' "$example_main" || \ + fail "$example_name runnable example does not require the matching archive checksum" +done +grep -Fq 'run "rejects_rootfs_release_from_another_module_version"' "$linux_runtime_tests" || \ + fail "runnable provider examples lack plan-time module/archive mismatch coverage" +grep -Fq '"$source_ref" != "$checkout_sha"' "$cloud_smoke" || \ + fail "hosted smoke does not bind its downloadable source archive to the checked-out commit" +grep -Fq 'CLOUD_COMPOSE_SOURCE_REF: ${{ github.sha }}' "$cloud_smoke_workflow" || \ + fail "hosted smoke does not select the exact tested merge commit" +grep -Fq 'contents: read' "$cloud_smoke_workflow" || \ + fail "hosted smoke lacks read-only repository permission" +if grep -Fq 'contents: write' "$cloud_smoke_workflow"; then + fail "untrusted pull-request smoke code has repository write permission" +fi +for cloud_init_template in "$linux_cloud_init" "$gcp_cloud_init"; do + if grep -Eq '^[[:space:]]*-[[:space:]]*[|>][+-]?[[:space:]]*$' "$cloud_init_template"; then + fail "$cloud_init_template still embeds a shell program instead of invoking a checked-in file" + fi done rollout_installer="$repo_root/rootfs/home/cloud-compose/deploy-rollout.sh" diff --git a/ci/systemd-contract.sh b/ci/systemd-contract.sh index aa6f000..f95bb49 100755 --- a/ci/systemd-contract.sh +++ b/ci/systemd-contract.sh @@ -45,6 +45,8 @@ assert_contains "$unit_dir/cloud-compose.service" 'After=network-online.target d assert_contains "$unit_dir/cloud-compose.service" 'ExecStartPre=/bin/bash /home/cloud-compose/assert-app-initialized.sh' assert_contains "$unit_dir/cloud-compose.service" 'Restart=on-failure' assert_contains "$unit_dir/cloud-compose.service" 'RestartSec=30s' +assert_contains "$unit_dir/cloud-compose.service" 'StartLimitIntervalSec=6h' +assert_contains "$unit_dir/cloud-compose.service" 'StartLimitBurst=3' if grep -Fq 'ConditionPathExists=' "$unit_dir/cloud-compose-bootstrap.service"; then fail "bootstrap still trusts an unvalidated marker path condition" fi @@ -52,6 +54,8 @@ assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecCondition=/bin/ assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecStart=/bin/bash /etc/cloud-compose/libexec/run-bootstrap.sh' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'Restart=on-failure' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'RestartSec=30s' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'StartLimitIntervalSec=8h' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'StartLimitBurst=3' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'TimeoutStartSec=2h' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'UMask=0022' assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'StandardOutput=journal' diff --git a/ci/terraform-validate.sh b/ci/terraform-validate.sh index 17e0156..66a38eb 100644 --- a/ci/terraform-validate.sh +++ b/ci/terraform-validate.sh @@ -31,13 +31,13 @@ validate_public_provider_graph() { case "$rel" in . | providers/gcp) - expected_sources=$'hashicorp/cloudinit\nhashicorp/google\nhashicorp/time' + expected_sources=$'hashicorp/cloudinit\nhashicorp/google\nhashicorp/http\nhashicorp/time' ;; providers/do) - expected_sources='digitalocean/digitalocean' + expected_sources=$'digitalocean/digitalocean\nhashicorp/http' ;; providers/linode) - expected_sources='linode/linode' + expected_sources=$'hashicorp/http\nlinode/linode' ;; *) return 0 diff --git a/ci/verify-rootfs-release.sh b/ci/verify-rootfs-release.sh new file mode 100755 index 0000000..5497890 --- /dev/null +++ b/ci/verify-rootfs-release.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "$script_dir/.." && pwd)" +readonly archive_name=cloud-compose-rootfs.tar.gz +readonly archive_checksum_name=cloud-compose-rootfs.tar.gz.sha256 +readonly contract_name=cloud-compose-rootfs.contract.sha256 + +fail() { + echo "rootfs release verification: $*" >&2 + exit 1 +} + +resolve_tag_for_commit() { + local commit="$1" attempt tag + + [[ "$commit" =~ ^[0-9a-f]{40}$ ]] || fail "commit must be a full lowercase Git SHA" + for ((attempt = 1; attempt <= 60; attempt++)); do + git fetch --force --tags origin >/dev/null 2>&1 || true + tag="$(git tag --points-at "$commit" --sort=-version:refname | head -n 1)" + if [[ -n "$tag" ]]; then + printf '%s\n' "$tag" + return 0 + fi + sleep 10 + done + fail "no release tag appeared for commit $commit" +} + +download_assets() { + local tag="$1" destination="$2" attempt + + for ((attempt = 1; attempt <= 60; attempt++)); do + if gh release download "$tag" \ + --repo "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" \ + --dir "$destination" \ + --clobber \ + --pattern "$archive_name" \ + --pattern "$archive_checksum_name" \ + --pattern "$contract_name" >/dev/null 2>&1 && + [[ -f "$destination/$archive_name" && + -f "$destination/$archive_checksum_name" && + -f "$destination/$contract_name" ]]; then + return 0 + fi + sleep 10 + done + fail "release $tag did not publish all three immutable rootfs assets" +} + +verify_assets() { + local tag="$1" asset_dir="$2" extract_dir="$3" + local checksum_line expected_archive_sha256 checksum_filename actual_archive_sha256 + local contract_sha256 actual_contract_sha256 + + checksum_line="$(<"$asset_dir/$archive_checksum_name")" + [[ "$checksum_line" =~ ^([0-9a-f]{64})\ \ cloud-compose-rootfs\.tar\.gz$ ]] || + fail "release $tag has a malformed archive checksum asset" + expected_archive_sha256="${BASH_REMATCH[1]}" + checksum_filename="${checksum_line#* }" + [[ "$checksum_filename" == "$archive_name" ]] || + fail "release $tag checksum names an unexpected archive" + actual_archive_sha256="$(sha256sum -- "$asset_dir/$archive_name" | cut -d' ' -f1)" + [[ "$actual_archive_sha256" == "$expected_archive_sha256" ]] || + fail "release $tag archive bytes do not match its checksum" + + [[ "$(wc -c <"$asset_dir/$contract_name")" -eq 65 ]] || + fail "release $tag rootfs contract must be one digest plus a newline" + contract_sha256="$(<"$asset_dir/$contract_name")" + [[ "$contract_sha256" =~ ^[0-9a-f]{64}$ ]] || + fail "release $tag rootfs contract is not a lowercase SHA-256 digest" + + bash "$repo_root/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh" \ + validate-archive "$asset_dir/$archive_name" || + fail "release $tag archive member topology is unsafe" + + install -d -m 0755 "$extract_dir" + tar --no-same-owner --same-permissions \ + -xzf "$asset_dir/$archive_name" -C "$extract_dir" + actual_contract_sha256="$( + bash "$repo_root/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh" \ + contract "$extract_dir/rootfs" + )" + [[ "$actual_contract_sha256" == "$contract_sha256" ]] || + fail "release $tag archive does not match its canonical rootfs contract" +} + +verify_source_assets() { + local tag="$1" asset_dir="$2" expected_dir="$3" + + bash "$repo_root/ci/package-rootfs.sh" "$expected_dir" + for asset in "$archive_name" "$archive_checksum_name" "$contract_name"; do + cmp -- "$expected_dir/$asset" "$asset_dir/$asset" || + fail "release $tag asset $asset does not match the tagged source" + done +} + +main() { + local selector="${1:-}" value="${2:-}" tag tmp + + [[ "$#" -eq 2 ]] || fail "usage: verify-rootfs-release.sh tag TAG|commit SHA" + case "$selector" in + tag) + [[ "$value" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+([+-][0-9A-Za-z.-]+)?$ ]] || + fail "tag is not a semantic-version release tag" + tag="$value" + ;; + commit) + tag="$(resolve_tag_for_commit "$value")" + ;; + *) fail "usage: verify-rootfs-release.sh tag TAG|commit SHA" ;; + esac + + tmp="$(mktemp -d "${RUNNER_TEMP:-/tmp}/cloud-compose-rootfs-release.XXXXXX")" + trap 'rm -rf -- "$tmp"' EXIT + install -d -m 0755 "$tmp/assets" + download_assets "$tag" "$tmp/assets" + verify_source_assets "$tag" "$tmp/assets" "$tmp/expected" + verify_assets "$tag" "$tmp/assets" "$tmp/extracted" + echo "Verified rootfs release assets for $tag" +} + +main "$@" diff --git a/docs/examples.md b/docs/examples.md index dbad7ea..075a732 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -22,7 +22,7 @@ module path and supplies the template: ```hcl module "app" { - source = "github.com/libops/cloud-compose//providers/do?ref=1.0.0" + source = "github.com/libops/cloud-compose//providers/do?ref=REPLACE_WITH_EXACT_RELEASE" name = "cc-wp" template = "wp" @@ -31,12 +31,19 @@ module "app" { cloud_compose_keys = var.operator_ssh_keys } } + runtime = { + rootfs_archive_url = var.cloud_compose_rootfs_archive_url + rootfs_archive_sha256 = var.cloud_compose_rootfs_archive_sha256 + } } ``` -The examples pin the reviewed `1.0.0` release. Replace that ref only with the -exact release or full commit your organization has reviewed; omitting `?ref=` -makes future plans consume a moving module source. +Replace the source placeholder with an exact release that publishes all three +canonical rootfs assets, and set the URL and SHA-256 from that same release. +The runnable DigitalOcean and Linode examples intentionally provide no release +default. Terraform fetches the adjacent rootfs contract during planning and +rejects a module/archive mismatch; omitting `?ref=` would make future plans +consume a moving module source. ## GCP foundation and application states diff --git a/docs/non-gcp-providers.md b/docs/non-gcp-providers.md index f22bce6..3c781b4 100644 --- a/docs/non-gcp-providers.md +++ b/docs/non-gcp-providers.md @@ -262,7 +262,7 @@ repo, `sitectl` plugin, and plugin package defaults: ```hcl module "wp" { - source = "github.com/libops/cloud-compose//providers/do?ref=1.0.0" + source = "github.com/libops/cloud-compose//providers/do?ref=REPLACE_WITH_EXACT_RELEASE" name = "cc-wp" template = "wp" @@ -273,18 +273,20 @@ module "wp" { } } runtime = { - rootfs_archive_url = "https://github.com/libops/cloud-compose/releases/download/1.0.0/cloud-compose-rootfs.tar.gz" + rootfs_archive_url = var.cloud_compose_rootfs_archive_url rootfs_archive_sha256 = var.cloud_compose_rootfs_sha256 } } ``` -The `1.0.0` ref is intentional. Replace it only with an exact reviewed release -or full commit. DigitalOcean limits Droplet `user_data` to 64 KiB, and the full -managed runtime no longer fits safely inline. Use the integrity-pinned archive -pair shown above; Terraform rejects an oversized inline payload before calling -the Droplet API. The default DigitalOcean SSH firewall sources include the -public internet; production callers should set +Replace the source placeholder with the exact reviewed release that supplied +`cloud_compose_rootfs_archive_url` and its SHA-256. The runnable example leaves +both release inputs required because older releases do not publish the adjacent +contract now required at plan time. DigitalOcean limits Droplet `user_data` to +64 KiB, and the full managed runtime no longer fits safely inline. Terraform +rejects both an oversized inline payload and a mismatched module/archive before +calling the Droplet API. The default DigitalOcean SSH firewall sources include +the public internet; production callers should set `digitalocean.firewall.ssh_source_addresses` to institutional or operator CIDRs instead of inheriting that default. @@ -313,7 +315,7 @@ before extracting any archive content. ```hcl module "drupal" { - source = "github.com/libops/cloud-compose//providers/linode?ref=1.0.0" + source = "github.com/libops/cloud-compose//providers/linode?ref=REPLACE_WITH_EXACT_RELEASE" name = "cc-drupal" template = "drupal" @@ -327,7 +329,7 @@ module "drupal" { } } runtime = { - rootfs_archive_url = "https://github.com/libops/cloud-compose/releases/download/1.0.0/cloud-compose-rootfs.tar.gz" + rootfs_archive_url = var.cloud_compose_rootfs_archive_url rootfs_archive_sha256 = var.cloud_compose_rootfs_sha256 } } @@ -335,13 +337,18 @@ module "drupal" { Linode also defaults SSH ingress to the public internet. Set `linode.firewall.ssh_source_ipv4` and `ssh_source_ipv6` to reviewed operator -CIDRs. Replace the `1.0.0` module ref only with an exact release or full commit -your organization has reviewed. - -Use the canonical `cloud-compose-rootfs.tar.gz` and adjacent `.sha256` release -assets. They are built reproducibly from the tagged `rootfs/` tree; GitHub's -generated repository source archives are not a stable long-term checksum -boundary because GitHub may regenerate their outer compression. CI may use a -commit archive only within the same run while testing an unreleased commit. -Download the release asset through the same URL consumers will use and verify -its published checksum before planning the deployment. +CIDRs. Replace the source placeholder only with the exact reviewed release that +supplied the rootfs archive URL and checksum. The runnable example intentionally +has no release default. + +Use the canonical `cloud-compose-rootfs.tar.gz`, adjacent archive `.sha256`, +and `cloud-compose-rootfs.contract.sha256` release assets. They are built +reproducibly from the tagged `rootfs/` tree. Terraform fetches the contract +sidecar while planning and rejects an archive from a different module release +before changing a VM; boot independently verifies both the pinned archive +bytes and the canonical rootfs contract. GitHub's generated repository source +archives are not a stable long-term checksum boundary because GitHub may +regenerate their outer compression. CI may use a commit archive only within the +same run while testing an unreleased commit. Promote a release only after its +`Verify rootfs release assets` job is green, then use the exact release asset +URL and published archive checksum in the deployment. diff --git a/docs/rollout.md b/docs/rollout.md index c4b35e2..a023561 100644 --- a/docs/rollout.md +++ b/docs/rollout.md @@ -28,16 +28,16 @@ module "site" { runtime = { compose = { rollout = [ - "TARGET_REF=\"$${GIT_REF:-$${GIT_BRANCH:-}}\"", - "if [ -n \"$TARGET_REF\" ]; then sitectl deploy --context \"$${SITECTL_CONTEXT_NAME}\" --ref \"$TARGET_REF\"; else sitectl deploy --context \"$${SITECTL_CONTEXT_NAME}\" --skip-git; fi", - "sitectl healthcheck --context \"$${SITECTL_CONTEXT_NAME}\" --persist", - "if [ \"$${SITECTL_ENVIRONMENT}\" != \"production\" ]; then sitectl verify --context \"$${SITECTL_CONTEXT_NAME}\" $${SITECTL_VERIFY_ARGS:-}; fi", + "/home/cloud-compose/default-lifecycle.sh rollout", ] } } } ``` +The `runtime.compose.rollout` entry above is the built-in default and may be +omitted. It is shown to make the single-program execution boundary explicit. + The example pins the reviewed cloud-compose `1.0.0` release. Replace that ref only with an exact reviewed release or full commit. `gcp.rollout.release_url` should point to a pinned Linux binary built from the generic rollout service; @@ -124,9 +124,13 @@ For bin-packed hosts, set `rollout_arg1` to the exact app key from `CLOUD_COMPOSE_APP` only for the rollout lifecycle and validates it against the manifest before running any command. Omit it to retain the primary-app default. -The generated rollout script runs from the checked-out compose repository after -sourcing `/home/cloud-compose/profile.sh`. The default contract prefers -`GIT_REF`, then `GIT_BRANCH`. When either is present, `sitectl deploy --ref` +The root-owned default lifecycle program runs from the checked-out compose +repository after the host dispatcher loads the validated app environment. For +controller-compatible callers that additionally provide `GIT_COMMIT_SHA`, the +default contract requires an exact lowercase 40-character commit SHA and +prefers it over `GIT_REF`, then `GIT_BRANCH`. The authenticated rollout service +currently supplies the latter two fields. When a source selector is present, +`sitectl deploy --ref` fetches that exact remote ref into a dedicated local ref, verifies it resolves to a commit, and checks it out detached before invoking the active plugin's component lifecycle. This supports branch names, advertised commit IDs, and diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 1febd43..e160352 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -143,11 +143,14 @@ repository contents but does not prove who authored them; protect the selected repository and review/sign commits according to your downstream governance. First boot runs through `cloud-compose-bootstrap.service`. Both that bootstrap -and `cloud-compose.service` retry failures after 30 seconds, so a transient -registry, Vault, or Compose failure converges without an operator restarting -cloud-init. Application initialization is serialized by the normal lifecycle -lock and recorded for the current boot before the app starts; a bootstrap retry -reuses that successful initialization instead of repeating it. Durable +and `cloud-compose.service` retry transient failures after 30 seconds, with a +three-attempt systemd start limit. A terminal unit failure stops the caller's +wait immediately; a five-minute heartbeat records the unit state during a long +but healthy convergence. An operator can inspect the failure and start the +unit again to open a new bounded retry window. Application initialization is +serialized by the normal lifecycle lock and recorded for the current boot +before the app starts; a bootstrap retry reuses that successful initialization +instead of repeating it. Durable readiness is published only after the Compose `up` unit reaches its successful oneshot state. The cloud-init caller waits for that result, so configured post-initialization commands retain their ordering. Inspect the bounded system @@ -164,6 +167,25 @@ diagnostics command, and checked-in `jq` programs live below that stateless tree while its `/usr` filesystem is immutable. Application services retain their unprivileged execution model. +Terraform and cloud-init are transport and orchestration boundaries, not the +home of bootstrap implementations. The GCP and provider-neutral templates +transfer checked-in programs from `rootfs/etc/cloud-compose/libexec` and invoke +them by path with data-only arguments. GCP packages its checked filesystem +program in a `text/cloud-boothook` MIME part so durable mounts are validated at +the original early, every-boot phase before cloud-config writes files or starts +services. Archive-mode Linux user data separately transports the small current +bootstrap driver and verifies its Terraform-rendered SHA-256 before execution; +filesystem helpers and the rest of the runtime still come from the verified +rootfs archive. This preserves metadata headroom without requiring an older +rootfs archive to contain the new driver. Files destined for `/mnt/disks` are +staged until the checked-in filesystem program has mounted and validated the +durable disks. GCP `initcmd` and `runcmd` values are likewise written as +root-controlled program files before they are sourced at their documented +points in the bootstrap sequence. Keep substantive shell, Python, `jq`, and +similar programs in reviewed files; do not interpolate them into Terraform +heredocs, cloud-init command strings, Compose commands, or configuration- +management task bodies. + ## Sitectl During init, the VM creates a `sitectl` context for every app. The default `up` @@ -196,6 +218,17 @@ invokes `sitectl verify`; spaces in one value never become additional arguments. Newlines, carriage returns, and NUL bytes are rejected instead of being flattened into an ambiguous shell scalar. +Each configured lifecycle list value names an independent checked program; it +is not parsed as shell source. The built-in `init`, `up`, `down`, and `rollout` +defaults each contain exactly `/home/cloud-compose/default-lifecycle.sh ACTION`, +where `ACTION` matches the lifecycle field. A custom entry must be one +argument-free, root-controlled program immediately below +`/etc/cloud-compose/lifecycle.d`; pass data through the documented lifecycle +environment. `true` and `false` remain explicit no-op and failure sentinels. +Multi-step work, state such as local variables or traps, and any quoting belong +inside that reviewed program file. The constrained executor invokes its argv +directly and never evaluates a manifest value with a shell parser. + ## Vault `cloud-compose` defines the Vault contract and leaves product-specific Vault @@ -667,14 +700,34 @@ All Terraform entrypoints, including GCP, DigitalOcean, and Linode, support the same verified rootfs archive contract. When `runtime.rootfs_archive_url` is used, it must be an HTTPS URL without whitespace and `runtime.rootfs_archive_sha256` is mandatory with a 64-character SHA-256 -digest. Boot restricts curl and redirects to HTTPS with TLS 1.2 or newer, -downloads to a temporary path, and verifies the complete file before extracting -its `rootfs` directory. Both archive installers normalize the verified tree to -UID/GID `0:0` before loading a helper or copying it onto the host, so Git forge -source-archive ownership cannot leak into the privileged runtime. The GCP path -stages the caller's packaged rootfs overlay and reapplies it after the archive, -so consumer overrides still win. Use an immutable archive URL; a moving branch -and a pinned checksum intentionally fail as soon as the branch content changes. +digest. Terraform derives the adjacent +`cloud-compose-rootfs.contract.sha256` URL and reads it during planning. That +sidecar must contain exactly the canonical contract digest for the module's +checked-in `rootfs`: every directory, file path, file byte, root ownership, +mode, and single-link file topology. A missing, malformed, older, or otherwise +mismatched sidecar rejects the plan before Terraform can replace a VM. + +Boot restricts curl and redirects to HTTPS with TLS 1.2 or newer, downloads to +a root-only temporary path, verifies the complete archive checksum, rejects +links and unsupported filesystem objects, and re-verifies the same canonical +rootfs contract before copying anything onto the host. Packaged directories are +root-owned mode `0755`; checked `*.sh` programs are mode `0755`; other files are +mode `0644`; regular files must have one link. The GCP path stages the caller's +packaged rootfs overlay and reapplies it after the archive, so consumer +overrides still win. A single checked-in archive program implements download, +verification, and installation for every Terraform provider rather than +duplicating provider-specific shell bodies. Use the immutable assets from the +same release as the Terraform module. Keeping an older archive while advancing +the module fails safely during planning and leaves the existing workload +untouched. + +Each release publishes three assets: +`cloud-compose-rootfs.tar.gz`, its archive-byte `.sha256`, and +`cloud-compose-rootfs.contract.sha256`. The release workflow downloads the +published assets again, validates the archive bytes and canonical tree, and +does not complete its release gate until all three agree. Downstream catalogs +or automation must promote a cloud-compose release only after the +`Verify rootfs release assets` job is green. ## Backups diff --git a/examples/digitalocean/variables.tf b/examples/digitalocean/variables.tf index d9b71af..fc89289 100644 --- a/examples/digitalocean/variables.tf +++ b/examples/digitalocean/variables.tf @@ -30,8 +30,8 @@ variable "cloud_compose_ssh_keys" { variable "cloud_compose_source_ref" { type = string - default = "1.0.0" - description = "Exact cloud-compose release tag whose canonical rootfs asset is fetched by DigitalOcean cloud-init." + nullable = false + description = "Required exact cloud-compose release tag whose canonical rootfs asset is fetched by DigitalOcean cloud-init. It must match the release containing this example/module source." validation { condition = can(regex("^v?[0-9]+\\.[0-9]+\\.[0-9]+$", trimspace(var.cloud_compose_source_ref))) @@ -41,11 +41,12 @@ variable "cloud_compose_source_ref" { variable "cloud_compose_source_sha256" { type = string + nullable = false description = "SHA-256 of the canonical rootfs asset published for cloud_compose_source_ref." validation { - condition = can(regex("^[0-9a-fA-F]{64}$", trimspace(var.cloud_compose_source_sha256))) - error_message = "cloud_compose_source_sha256 must be a 64-character SHA-256 digest." + condition = can(regex("^[0-9a-f]{64}$", trimspace(var.cloud_compose_source_sha256))) + error_message = "cloud_compose_source_sha256 must be a lowercase 64-character SHA-256 digest." } } diff --git a/examples/linode/variables.tf b/examples/linode/variables.tf index d65d708..cb1c596 100644 --- a/examples/linode/variables.tf +++ b/examples/linode/variables.tf @@ -43,8 +43,8 @@ variable "cloud_compose_ssh_keys" { variable "cloud_compose_source_ref" { type = string - default = "1.0.0" - description = "Exact cloud-compose release tag whose canonical rootfs asset is fetched by Linode cloud-init." + nullable = false + description = "Required exact cloud-compose release tag whose canonical rootfs asset is fetched by Linode cloud-init. It must match the release containing this example/module source." validation { condition = can(regex("^v?[0-9]+\\.[0-9]+\\.[0-9]+$", trimspace(var.cloud_compose_source_ref))) @@ -54,11 +54,12 @@ variable "cloud_compose_source_ref" { variable "cloud_compose_source_sha256" { type = string + nullable = false description = "SHA-256 of the canonical rootfs asset published for cloud_compose_source_ref." validation { - condition = can(regex("^[0-9a-fA-F]{64}$", trimspace(var.cloud_compose_source_sha256))) - error_message = "cloud_compose_source_sha256 must be a 64-character SHA-256 digest." + condition = can(regex("^[0-9a-f]{64}$", trimspace(var.cloud_compose_source_sha256))) + error_message = "cloud_compose_source_sha256 must be a lowercase 64-character SHA-256 digest." } } diff --git a/internal/contracttest/provider_boundary_test.go b/internal/contracttest/provider_boundary_test.go index 04d82e8..3a83457 100644 --- a/internal/contracttest/provider_boundary_test.go +++ b/internal/contracttest/provider_boundary_test.go @@ -73,6 +73,7 @@ func TestRootEntrypointLoadsOnlyGCPProviders(t *testing.T) { "digitalocean/digitalocean", "hashicorp/cloudinit", "hashicorp/google", + "hashicorp/http", "hashicorp/time", "linode/linode", "Unexpected Terraform provider graph", diff --git a/internal/contracttest/rollout_parity_test.go b/internal/contracttest/rollout_parity_test.go index a774967..c521e7b 100644 --- a/internal/contracttest/rollout_parity_test.go +++ b/internal/contracttest/rollout_parity_test.go @@ -11,10 +11,7 @@ import ( ) var expectedRollout = []string{ - `TARGET_REF="${GIT_REF:-${GIT_BRANCH:-}}"`, - `if [ -n "$TARGET_REF" ]; then sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --ref "$TARGET_REF"; else sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --skip-git; fi`, - `sitectl healthcheck --context "${SITECTL_CONTEXT_NAME}" --persist`, - `if [ "${SITECTL_ENVIRONMENT}" != "production" ]; then sitectl verify --context "${SITECTL_CONTEXT_NAME}" ${SITECTL_VERIFY_ARGS:-}; fi`, + `/home/cloud-compose/default-lifecycle.sh rollout`, } type rolloutSource struct { diff --git a/modules/digitalocean/main.tf b/modules/digitalocean/main.tf index bf78b55..663bee2 100644 --- a/modules/digitalocean/main.tf +++ b/modules/digitalocean/main.tf @@ -13,35 +13,36 @@ locals { module "runtime" { source = "../linux-vm-runtime" - name = var.name - provider_name = "digitalocean" - region = local.do.region - data_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.data_volume_name}" - volumes_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.docker_volumes_volume_name}" - ssh_users = merge(local.runtime.users, local.do.ssh.users) - cloud_compose_ssh_keys = local.do.ssh.cloud_compose_keys - rootfs = local.runtime.rootfs - rootfs_archive_url = local.runtime.rootfs_archive_url - rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 - offhost_backup_required = local.runtime.disaster_recovery.required - offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path - ingress_port = local.compose.ingress_port - primary_compose_project = local.compose.primary - sitectl_ingress = local.compose.ingress - docker_compose_repo = local.compose.repo - docker_compose_branch = local.compose.branch - compose_projects = local.compose.projects - docker_compose_init = local.compose.init - docker_compose_up = local.compose.up - docker_compose_down = local.compose.down - docker_compose_rollout = local.compose.rollout - rollout_enabled = local.do.rollout.enabled - rollout_release_url = local.do.rollout.release_url - rollout_release_sha256 = local.do.rollout.release_sha256 - rollout_port = local.do.rollout.port - rollout_jwks_uri = local.do.rollout.jwks_uri - rollout_jwt_audience = local.do.rollout.jwt_audience - rollout_custom_claims = local.do.rollout.custom_claims + name = var.name + provider_name = "digitalocean" + region = local.do.region + data_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.data_volume_name}" + volumes_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.docker_volumes_volume_name}" + ssh_users = merge(local.runtime.users, local.do.ssh.users) + cloud_compose_ssh_keys = local.do.ssh.cloud_compose_keys + rootfs = local.runtime.rootfs + rootfs_archive_url = local.runtime.rootfs_archive_url + rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 + rootfs_test_source_archive_prefix = local.runtime.rootfs_test_source_archive_prefix + offhost_backup_required = local.runtime.disaster_recovery.required + offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path + ingress_port = local.compose.ingress_port + primary_compose_project = local.compose.primary + sitectl_ingress = local.compose.ingress + docker_compose_repo = local.compose.repo + docker_compose_branch = local.compose.branch + compose_projects = local.compose.projects + docker_compose_init = local.compose.init + docker_compose_up = local.compose.up + docker_compose_down = local.compose.down + docker_compose_rollout = local.compose.rollout + rollout_enabled = local.do.rollout.enabled + rollout_release_url = local.do.rollout.release_url + rollout_release_sha256 = local.do.rollout.release_sha256 + rollout_port = local.do.rollout.port + rollout_jwks_uri = local.do.rollout.jwks_uri + rollout_jwt_audience = local.do.rollout.jwt_audience + rollout_custom_claims = local.do.rollout.custom_claims sitectl_packages = local.sitectl.packages sitectl_version = local.sitectl.version diff --git a/modules/digitalocean/runtime_inputs.tftest.hcl b/modules/digitalocean/runtime_inputs.tftest.hcl index 19c9150..6b7eebc 100644 --- a/modules/digitalocean/runtime_inputs.tftest.hcl +++ b/modules/digitalocean/runtime_inputs.tftest.hcl @@ -1,4 +1,12 @@ mock_provider "digitalocean" {} +mock_provider "http" { + mock_data "http" { + defaults = { + response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + status_code = 200 + } + } +} run "merges_provider_neutral_and_provider_specific_ssh_users" { command = plan diff --git a/modules/digitalocean/variables.tf b/modules/digitalocean/variables.tf index 26a6104..2def7c6 100644 --- a/modules/digitalocean/variables.tf +++ b/modules/digitalocean/variables.tf @@ -74,7 +74,9 @@ variable "runtime" { rootfs = optional(string, "") rootfs_archive_url = optional(string, "") rootfs_archive_sha256 = optional(string, "") - users = optional(map(list(string)), {}) + # Internal hosted-smoke escape hatch. Public provider entrypoints must not expose it. + rootfs_test_source_archive_prefix = optional(string, "") + users = optional(map(list(string)), {}) disaster_recovery = optional(object({ required = optional(bool, false) diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index 0833c86..99a459b 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/google" version = "~> 7.0" } + http = { + source = "hashicorp/http" + version = "~> 3.6" + } time = { source = "hashicorp/time" version = "~> 0.14" @@ -22,6 +26,8 @@ locals { additional_rootfs = var.rootfs != "" ? var.rootfs : "" rootfs_archive_url = trimspace(var.rootfs_archive_url) rootfs_archive_sha256 = lower(trimspace(var.rootfs_archive_sha256)) + rootfs_archive_asset_url = split("#", split("?", local.rootfs_archive_url)[0])[0] + rootfs_contract_sidecar_url = local.rootfs_archive_url == "" ? "" : replace(local.rootfs_archive_asset_url, "/[^/]+$/", "/cloud-compose-rootfs.contract.sha256") rootfs_overlay_staging_path = "/var/lib/cloud-compose/rootfs-overlay" project_number = tostring(data.google_project.service.number) @@ -210,8 +216,21 @@ locals { selected_subnetwork_project_matches = local.network_project_id == local.subnetwork_project_id selected_subnetwork_region_matches = local.subnetwork_region == var.region - # Archive mode fetches the packaged base rootfs at boot. A consumer-provided - # rootfs is staged separately and applied after extraction so it still wins. + # Archive mode fetches the packaged base rootfs at boot. A content contract + # binds that archive to this exact module source before anything is copied. + rootfs_contract_files = sort(tolist(fileset(local.rootFs, "**"))) + rootfs_contract_directories = sort(distinct(flatten([ + for file in local.rootfs_contract_files : dirname(file) == "." ? [] : [ + for index in range(length(split("/", dirname(file)))) : + join("/", slice(split("/", dirname(file)), 0, index + 1)) + ] + ]))) + rootfs_contract_sha256 = sha256(join("", [ + for entry in concat( + [for directory in local.rootfs_contract_directories : "d\t0:0:755\t${directory}\n"], + [for file in local.rootfs_contract_files : "f\t${filesha256("${local.rootFs}/${file}")}\t0:0:${endswith(file, ".sh") ? "755" : "644"}\t${file}\n"], + ) : entry + ])) base_files = local.rootfs_archive_url == "" ? fileset(local.rootFs, "**") : [] # Get files from additional rootfs if path is provided @@ -227,7 +246,7 @@ locals { }, { for file in local.additional_files : file => { - destination = local.rootfs_archive_url == "" ? "/${file}" : "${local.rootfs_overlay_staging_path}/${file}" + destination = local.rootfs_archive_url != "" ? "${local.rootfs_overlay_staging_path}/${file}" : "/${file}" source = "${local.additional_rootfs}/${file}" } } @@ -251,47 +270,39 @@ EOT - path: "/home/cloud-compose/${name}" owner: "root:root" permissions: "0755" - encoding: gzip+base64 - content: ${jsonencode(base64gzip(<<-EOS - #!/usr/bin/env bash - - set -eou pipefail - - source /home/cloud-compose/profile.sh - exec bash /home/cloud-compose/compose-dispatch.sh "${name}" - EOS -))} + encoding: b64 + content: ${filebase64("${local.rootFs}/home/cloud-compose/lifecycle-entrypoint.sh")} EOT -]) -compose_projects_file = <<-EOT + ]) + compose_projects_file = <<-EOT - path: "/home/cloud-compose/compose-projects.json" owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(jsonencode(local.validated_compose_projects)))} EOT -managed_runtime_artifact_lines = [ - for artifact in module.managed_artifacts.artifacts : join("\t", [ - artifact.name, - artifact.url, - artifact.sha256, - artifact.path, - try(artifact.mode, "0755"), - try(artifact.owner, "root"), - try(artifact.group, "root"), - try(artifact.restart, ""), - ]) -] -managed_runtime_artifacts_content = length(local.managed_runtime_artifact_lines) == 0 ? "\n" : "${join("\n", local.managed_runtime_artifact_lines)}\n" -managed_runtime_artifacts_file = <<-EOT + managed_runtime_artifact_lines = [ + for artifact in module.managed_artifacts.artifacts : join("\t", [ + artifact.name, + artifact.url, + artifact.sha256, + artifact.path, + try(artifact.mode, "0755"), + try(artifact.owner, "root"), + try(artifact.group, "root"), + try(artifact.restart, ""), + ]) + ] + managed_runtime_artifacts_content = length(local.managed_runtime_artifact_lines) == 0 ? "\n" : "${join("\n", local.managed_runtime_artifact_lines)}\n" + managed_runtime_artifacts_file = <<-EOT - path: "/home/cloud-compose/managed-runtime-artifacts.tsv" owner: "root:root" permissions: "0640" encoding: gzip+base64 content: ${jsonencode(base64gzip(local.managed_runtime_artifacts_content))} EOT -vault_agent_template_stanzas = join("\n", [ - for template in var.vault_agent_templates : <<-EOT + vault_agent_template_stanzas = join("\n", [ + for template in var.vault_agent_templates : <<-EOT template { destination = ${jsonencode(template.destination)} contents = <&2 - exit 1 - } - archive_url="$(printf '%s' '${base64encode(local.rootfs_archive_url)}' | base64 -d)" - archive_sha256=${jsonencode(local.rootfs_archive_sha256)} - overlay_dir=${jsonencode(local.rootfs_overlay_staging_path)} - case "$archive_url" in - https://*) ;; - *) echo "rootfs archive URL must use HTTPS" >&2; exit 1 ;; - esac - case "$archive_url" in - *[[:space:]]*) echo "rootfs archive URL must not contain whitespace" >&2; exit 1 ;; - esac - tmp="$(mktemp -d)" - trap 'rm -rf "$tmp"' EXIT - curl -fsSL --proto '=https' --proto-redir '=https' --tlsv1.2 \ - --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 900 \ - --connect-timeout 10 --max-time 300 -o "$tmp/rootfs.tar.gz" -- "$archive_url" - if ! command -v sha256sum >/dev/null 2>&1; then - echo "sha256sum is required to verify $archive_url" >&2 - exit 1 - fi - printf '%s %s\n' "$archive_sha256" "$tmp/rootfs.tar.gz" | sha256sum -c - - tar --no-same-owner -xzf "$tmp/rootfs.tar.gz" -C "$tmp" - rootfs_dir="$(find "$tmp" -mindepth 1 -maxdepth 3 -type d -name rootfs -print -quit)" - if [ -z "$rootfs_dir" ] || [ ! -d "$rootfs_dir" ]; then - echo "rootfs directory not found in $archive_url" >&2 - exit 1 - fi - chown -hR 0:0 -- "$rootfs_dir" - cp -a "$rootfs_dir"/. / - if [ -d "$overlay_dir" ]; then - cp -a "$overlay_dir"/. / - rm -rf "$overlay_dir" - fi - EOT -rootfs_archive_command = local.rootfs_archive_url != "" ? local.rootfs_archive_command_raw : "" -use_overlay = length(var.volume_names) > 0 -prod_disk_url = var.overlay_source_instance != "" ? format("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s-docker-volumes", var.project_id, var.zone, var.overlay_source_instance) : "" -rollout_runcmd = var.rollout_enabled ? [ - "bash /home/cloud-compose/deploy-rollout.sh >> /home/cloud-compose/run.log 2>&1", -] : [] -cloud_init_yaml = templatefile("${path.module}/../../templates/cloud-init.yml", { - DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"), - FILESYSTEM_PREP_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/prepare-filesystem.sh"), - FILESYSTEM_PERSIST_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/persist-filesystems.sh"), - FRESH_FILESYSTEM_IDENTITY = "v1:gcp-disk-id:${google_compute_disk.data.disk_id}", - WRITE_FILES_CONTENT = local.write_files_content, - DOCKER_COMPOSE_SCRIPTS = local.docker_compose_scripts, - COMPOSE_PROJECTS_FILE = local.compose_projects_file, - ENV_FILE_CONTENT = local.env_file_content, - APPLICATION_ENV_FILE_CONTENT = local.application_env_file_content, - VAULT_AGENT_FILES = local.vault_agent_files, - MANAGED_RUNTIME_ARTIFACTS_FILE = local.managed_runtime_artifacts_file, - ROOTFS_ARCHIVE_COMMAND = local.rootfs_archive_command, - USE_OVERLAY = local.use_overlay, - DOCKER_VOLUME_OVERLAYS = var.volume_names, - CLOUD_COMPOSE_SSH_KEYS = try(var.users["cloud-compose"], []), - SSH_USERS = { for username, ssh_keys in var.users : username => ssh_keys if username != "cloud-compose" }, - ADDITIONAL_INITCMD = var.initcmd, - ADDITIONAL_RUNCMD = concat(local.rollout_runcmd, var.runcmd), -}) - -vm_service_account_email = var.service_account_email != "" ? data.google_service_account.vm[0].email : google_service_account.cloud-compose[0].email -vm_service_account_id = var.service_account_email != "" ? data.google_service_account.vm[0].name : google_service_account.cloud-compose[0].id -vm_service_account_name = var.service_account_email != "" ? data.google_service_account.vm[0].name : google_service_account.cloud-compose[0].name - -app_service_account_email = var.app_service_account_email != "" ? data.google_service_account.app[0].email : google_service_account.app[0].email -app_service_account_id = var.app_service_account_email != "" ? data.google_service_account.app[0].name : google_service_account.app[0].id -app_service_account_name = var.app_service_account_email != "" ? data.google_service_account.app[0].name : google_service_account.app[0].name -app_service_account_managed = var.app_service_account_email == "" - -app_credentials_enabled = var.app_credentials_enabled -internal_services_enabled = var.libops_internal_services_enabled || var.power_management_enabled -internal_services_compose_profiles = var.power_management_enabled ? "lightsout" : "" -# Production snapshots are crash-consistent (`guest_flush = false`). MariaDB -# logical dumps run before the snapshot window and provide application-level -# consistency without coupling disk snapshots to a guest-agent implementation. -scheduled_snapshots_enabled = var.production && var.run_snapshots -# have prod snapshot begin near the initial run so non-prod overlays can -# discover a production snapshot; non-production plans avoid snapshot resources. -snapshot_start_time = local.scheduled_snapshots_enabled ? formatdate("h:00", time_static.snapshot_time_static[0].rfc3339) : "00:00" + initcmd_content = length(var.initcmd) > 0 ? "${join("\n", var.initcmd)}\n" : "" + runcmd_content = length(var.runcmd) > 0 ? "${join("\n", var.runcmd)}\n" : "" + post_bootstrap_required = var.rollout_enabled || length(var.runcmd) > 0 + use_overlay = length(var.volume_names) > 0 + prod_disk_url = var.overlay_source_instance != "" ? format("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s-docker-volumes", var.project_id, var.zone, var.overlay_source_instance) : "" + gcp_filesystem_boothook = templatefile("${path.module}/../../templates/gcp-filesystem-boothook.sh.tftpl", { + FILESYSTEM_PREP_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/prepare-filesystem.sh"), + FILESYSTEM_PERSIST_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/persist-filesystems.sh"), + GCP_FILESYSTEM_BOOT_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/libexec/gcp-filesystem-boot.sh"), + FRESH_FILESYSTEM_IDENTITY = "v1:gcp-disk-id:${google_compute_disk.data.disk_id}", + USE_OVERLAY = local.use_overlay, + }) + cloud_config_yaml = templatefile("${path.module}/../../templates/cloud-init.yml", { + ROOTFS_ARCHIVE_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/libexec/rootfs-archive.sh"), + GCP_CLOUD_INIT_FINALIZE_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh"), + GCP_CLOUD_INIT_POST_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/libexec/gcp-cloud-init-post-bootstrap.sh"), + DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"), + DIAGNOSTICS_SCRIPT_SHA256 = filesha256("${local.rootFs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"), + INIT_COMMANDS_B64 = base64encode(local.initcmd_content), + RUNCMD_B64 = base64encode(local.runcmd_content), + WRITE_FILES_CONTENT = local.write_files_content, + DOCKER_COMPOSE_SCRIPTS = local.docker_compose_scripts, + COMPOSE_PROJECTS_FILE = local.compose_projects_file, + ENV_FILE_CONTENT = local.env_file_content, + APPLICATION_ENV_FILE_CONTENT = local.application_env_file_content, + VAULT_AGENT_FILES = local.vault_agent_files, + MANAGED_RUNTIME_ARTIFACTS_FILE = local.managed_runtime_artifacts_file, + ROOTFS_ARCHIVE_ENABLED = local.rootfs_archive_url != "", + ROOTFS_ARCHIVE_URL_B64 = base64encode(local.rootfs_archive_url), + ROOTFS_ARCHIVE_SHA256 = local.rootfs_archive_sha256, + ROOTFS_CONTRACT_SHA256 = local.rootfs_contract_sha256, + ROOTFS_OVERLAY_DIR = local.rootfs_overlay_staging_path, + ROLLOUT_ENABLED = tostring(var.rollout_enabled), + POST_BOOTSTRAP_REQUIRED = local.post_bootstrap_required, + CLOUD_COMPOSE_SSH_KEYS = try(var.users["cloud-compose"], []), + SSH_USERS = { for username, ssh_keys in var.users : username => ssh_keys if username != "cloud-compose" }, + }) + cloud_init_yaml = templatefile("${path.module}/../../templates/gcp-cloud-init.mime.tftpl", { + FILESYSTEM_BOOTHOOK = local.gcp_filesystem_boothook, + CLOUD_CONFIG = local.cloud_config_yaml, + }) + + vm_service_account_email = var.service_account_email != "" ? data.google_service_account.vm[0].email : google_service_account.cloud-compose[0].email + vm_service_account_id = var.service_account_email != "" ? data.google_service_account.vm[0].name : google_service_account.cloud-compose[0].id + vm_service_account_name = var.service_account_email != "" ? data.google_service_account.vm[0].name : google_service_account.cloud-compose[0].name + + app_service_account_email = var.app_service_account_email != "" ? data.google_service_account.app[0].email : google_service_account.app[0].email + app_service_account_id = var.app_service_account_email != "" ? data.google_service_account.app[0].name : google_service_account.app[0].id + app_service_account_name = var.app_service_account_email != "" ? data.google_service_account.app[0].name : google_service_account.app[0].name + app_service_account_managed = var.app_service_account_email == "" + + app_credentials_enabled = var.app_credentials_enabled + internal_services_enabled = var.libops_internal_services_enabled || var.power_management_enabled + internal_services_compose_profiles = var.power_management_enabled ? "lightsout" : "" + # Production snapshots are crash-consistent (`guest_flush = false`). MariaDB + # logical dumps run before the snapshot window and provide application-level + # consistency without coupling disk snapshots to a guest-agent implementation. + scheduled_snapshots_enabled = var.production && var.run_snapshots + # have prod snapshot begin near the initial run so non-prod overlays can + # discover a production snapshot; non-production plans avoid snapshot resources. + snapshot_start_time = local.scheduled_snapshots_enabled ? formatdate("h:00", time_static.snapshot_time_static[0].rfc3339) : "00:00" } data "google_project" "service" { project_id = var.project_id } +data "http" "rootfs_contract" { + count = local.rootfs_archive_url != "" ? 1 : 0 + + url = local.rootfs_contract_sidecar_url + request_timeout_ms = 30000 + + lifecycle { + postcondition { + condition = ( + self.status_code == 200 && + can(regex("^[0-9a-f]{64}\\n?$", self.response_body)) && + trimspace(self.response_body) == local.rootfs_contract_sha256 + ) + error_message = "The immutable rootfs release sidecar must contain exactly this module source's canonical rootfs contract digest. Publish or select a matching archive before replacing a VM." + } + } +} + data "google_service_account" "vm" { count = var.service_account_email != "" ? 1 : 0 @@ -929,6 +933,13 @@ resource "google_compute_instance" "cloud-compose" { ) error_message = "rootfs_archive_url and a 64-character rootfs_archive_sha256 must be supplied together." } + precondition { + condition = ( + local.rootfs_archive_url == "" || + trimspace(try(data.http.rootfs_contract[0].response_body, "")) == local.rootfs_contract_sha256 + ) + error_message = "The rootfs archive release contract must match this module before a VM is replaced." + } precondition { condition = contains(keys(local.compose_projects), local.primary_compose_project_key) error_message = "primary_compose_project must be one of the compose_projects keys." diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index 50554ff..a91dacb 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -1,4 +1,12 @@ mock_provider "cloudinit" {} +mock_provider "http" { + mock_data "http" { + defaults = { + response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + status_code = 200 + } + } +} mock_provider "google" { mock_resource "google_compute_disk" { override_during = plan @@ -90,10 +98,15 @@ run "disables_privileged_services_by_default" { local.host_env.CLOUD_COMPOSE_FRESH_FILESYSTEM_IDENTITY == "v1:gcp-disk-id:987654321012345678" && strcontains( local.cloud_init_yaml, - "--publish-fresh-marker \"v1:gcp-disk-id:987654321012345678\"", - ) + jsonencode("v1:gcp-disk-id:987654321012345678"), + ) && + strcontains( + local.cloud_init_yaml, + filebase64("${path.module}/../../rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh"), + ) && + strcontains(local.cloud_init_yaml, "Content-Type: text/cloud-boothook") ) - error_message = "GCP cloud-init and the root runtime environment must carry the same immutable data-disk identity." + error_message = "GCP's early every-boot filesystem program and root runtime environment must carry the same immutable data-disk identity." } assert { @@ -553,19 +566,26 @@ run "renders_verified_archive_before_downstream_overlay" { assert { condition = ( - strcontains(local.cloud_init_yaml, "sha256sum -c -") && + strcontains( + local.cloud_init_yaml, + filebase64("${path.module}/../../rootfs/etc/cloud-compose/libexec/rootfs-archive.sh"), + ) && + strcontains(local.cloud_init_yaml, local.rootfs_contract_sha256) && + data.http.rootfs_contract[0].url == "https://example.invalid/cloud-compose-rootfs.contract.sha256" && strcontains(local.cloud_init_yaml, base64encode(var.rootfs_archive_url)) && !strcontains(local.cloud_init_yaml, var.rootfs_archive_url) ) - error_message = "The GCP archive URL must be transported as literal base64 data and verified before extraction." + error_message = "The GCP archive program must be transferred as a checked file and receive literal URL data plus the exact module rootfs contract." } assert { - condition = can(regex( - "(?s)sha256sum -c -.*tar --no-same-owner -xzf \\\"\\$tmp/rootfs\\.tar\\.gz\\\".*chown -hR 0:0 -- \\\"\\$rootfs_dir\\\".*cp -a \\\"\\$rootfs_dir\\\"/\\. /.*cp -a \\\"\\$overlay_dir\\\"/\\. /", - local.cloud_init_yaml, - )) - error_message = "The verified base archive must be extracted ownership-safely, normalized to root, and installed before the consumer rootfs overlay." + condition = ( + strcontains(local.cloud_init_yaml, "[bash, /var/lib/cloud-compose/bootstrap/rootfs-archive.sh, install") && + strcontains(local.cloud_init_yaml, "[bash, /var/lib/cloud-compose/bootstrap/gcp-cloud-init-finalize.sh") && + length(split("/var/lib/cloud-compose/bootstrap/rootfs-archive.sh, install", local.cloud_init_yaml)[0]) < + length(split("/var/lib/cloud-compose/bootstrap/gcp-cloud-init-finalize.sh", local.cloud_init_yaml)[0]) + ) + error_message = "The verified base archive and downstream overlay must be installed before GCP application initialization." } assert { @@ -600,6 +620,28 @@ run "renders_verified_archive_before_downstream_overlay" { } } +run "rejects_rootfs_release_from_another_module_version" { + command = plan + + variables { + name = "gcp-contract" + project_id = "test-project" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs_archive_url = "https://example.invalid/cloud-compose-rootfs.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + + override_data { + target = data.http.rootfs_contract[0] + values = { + response_body = "0000000000000000000000000000000000000000000000000000000000000000\n" + status_code = 200 + } + } + + expect_failures = [data.http.rootfs_contract[0]] +} + run "renders_embedded_offhost_backup_driver_executable" { command = plan diff --git a/modules/gcp/variables.tf b/modules/gcp/variables.tf index df05258..f9f33f6 100644 --- a/modules/gcp/variables.tf +++ b/modules/gcp/variables.tf @@ -262,7 +262,7 @@ variable "docker_compose_branch" { variable "docker_compose_init" { type = list(string) default = [ - "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --yolo --default" + "/home/cloud-compose/default-lifecycle.sh init" ] nullable = false description = "After cloning the docker compose git repo, any initialization that needs to happen before the docker compose project can start. One command per list value" @@ -271,9 +271,7 @@ variable "docker_compose_init" { variable "docker_compose_up" { type = list(string) default = [ - "sitectl compose --context \"$${SITECTL_CONTEXT_NAME}\" up -d --remove-orphans", - "sitectl healthcheck --context \"$${SITECTL_CONTEXT_NAME}\" --persist", - "if [ \"$${SITECTL_ENVIRONMENT}\" != \"production\" ]; then sitectl verify --context \"$${SITECTL_CONTEXT_NAME}\" $${SITECTL_VERIFY_ARGS:-}; fi" + "/home/cloud-compose/default-lifecycle.sh up" ] nullable = false description = "Command to start the docker compose project" @@ -282,7 +280,7 @@ variable "docker_compose_up" { variable "docker_compose_down" { type = list(string) default = [ - "sitectl compose --context \"$${SITECTL_CONTEXT_NAME}\" down" + "/home/cloud-compose/default-lifecycle.sh down" ] nullable = false description = "Command to stop the docker compose project" @@ -291,13 +289,10 @@ variable "docker_compose_down" { variable "docker_compose_rollout" { type = list(string) default = [ - "TARGET_REF=\"$${GIT_REF:-$${GIT_BRANCH:-}}\"", - "if [ -n \"$TARGET_REF\" ]; then sitectl deploy --context \"$${SITECTL_CONTEXT_NAME}\" --ref \"$TARGET_REF\"; else sitectl deploy --context \"$${SITECTL_CONTEXT_NAME}\" --skip-git; fi", - "sitectl healthcheck --context \"$${SITECTL_CONTEXT_NAME}\" --persist", - "if [ \"$${SITECTL_ENVIRONMENT}\" != \"production\" ]; then sitectl verify --context \"$${SITECTL_CONTEXT_NAME}\" $${SITECTL_VERIFY_ARGS:-}; fi" + "/home/cloud-compose/default-lifecycle.sh rollout" ] nullable = false - description = "Commands used by rollout triggers. GIT_REF/GIT_BRANCH selects a source ref; without one, sitectl reconciles the current checkout." + description = "Commands used by rollout triggers. A validated GIT_COMMIT_SHA takes precedence over GIT_REF/GIT_BRANCH; without one, sitectl reconciles the current checkout." } variable "sitectl_packages" { @@ -591,7 +586,7 @@ variable "rootfs" { variable "rootfs_archive_url" { type = string default = "" - description = "Optional HTTPS tar.gz URL containing a rootfs directory to fetch during boot instead of embedding the packaged rootfs. Must be set with rootfs_archive_sha256." + description = "Optional immutable HTTPS cloud-compose rootfs release archive to fetch during boot instead of embedding the packaged rootfs. Must be set with rootfs_archive_sha256; planning also requires the adjacent cloud-compose-rootfs.contract.sha256 asset to match this module source." validation { condition = ( diff --git a/modules/linode/main.tf b/modules/linode/main.tf index eed23d8..6c219bd 100644 --- a/modules/linode/main.tf +++ b/modules/linode/main.tf @@ -16,35 +16,36 @@ locals { module "runtime" { source = "../linux-vm-runtime" - name = var.name - provider_name = "linode" - region = local.linode.region - data_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.data_volume_label}" - volumes_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.docker_volumes_volume_label}" - ssh_users = merge(local.runtime.users, local.linode.ssh.users) - cloud_compose_ssh_keys = local.linode.ssh.cloud_compose_keys - rootfs = local.runtime.rootfs - rootfs_archive_url = local.runtime.rootfs_archive_url - rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 - offhost_backup_required = local.runtime.disaster_recovery.required - offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path - ingress_port = local.compose.ingress_port - primary_compose_project = local.compose.primary - sitectl_ingress = local.compose.ingress - docker_compose_repo = local.compose.repo - docker_compose_branch = local.compose.branch - compose_projects = local.compose.projects - docker_compose_init = local.compose.init - docker_compose_up = local.compose.up - docker_compose_down = local.compose.down - docker_compose_rollout = local.compose.rollout - rollout_enabled = local.linode.rollout.enabled - rollout_release_url = local.linode.rollout.release_url - rollout_release_sha256 = local.linode.rollout.release_sha256 - rollout_port = local.linode.rollout.port - rollout_jwks_uri = local.linode.rollout.jwks_uri - rollout_jwt_audience = local.linode.rollout.jwt_audience - rollout_custom_claims = local.linode.rollout.custom_claims + name = var.name + provider_name = "linode" + region = local.linode.region + data_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.data_volume_label}" + volumes_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.docker_volumes_volume_label}" + ssh_users = merge(local.runtime.users, local.linode.ssh.users) + cloud_compose_ssh_keys = local.linode.ssh.cloud_compose_keys + rootfs = local.runtime.rootfs + rootfs_archive_url = local.runtime.rootfs_archive_url + rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256 + rootfs_test_source_archive_prefix = local.runtime.rootfs_test_source_archive_prefix + offhost_backup_required = local.runtime.disaster_recovery.required + offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path + ingress_port = local.compose.ingress_port + primary_compose_project = local.compose.primary + sitectl_ingress = local.compose.ingress + docker_compose_repo = local.compose.repo + docker_compose_branch = local.compose.branch + compose_projects = local.compose.projects + docker_compose_init = local.compose.init + docker_compose_up = local.compose.up + docker_compose_down = local.compose.down + docker_compose_rollout = local.compose.rollout + rollout_enabled = local.linode.rollout.enabled + rollout_release_url = local.linode.rollout.release_url + rollout_release_sha256 = local.linode.rollout.release_sha256 + rollout_port = local.linode.rollout.port + rollout_jwks_uri = local.linode.rollout.jwks_uri + rollout_jwt_audience = local.linode.rollout.jwt_audience + rollout_custom_claims = local.linode.rollout.custom_claims sitectl_packages = local.sitectl.packages sitectl_version = local.sitectl.version diff --git a/modules/linode/runtime_inputs.tftest.hcl b/modules/linode/runtime_inputs.tftest.hcl index 1962318..672df96 100644 --- a/modules/linode/runtime_inputs.tftest.hcl +++ b/modules/linode/runtime_inputs.tftest.hcl @@ -1,4 +1,12 @@ mock_provider "linode" {} +mock_provider "http" { + mock_data "http" { + defaults = { + response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + status_code = 200 + } + } +} run "merges_provider_neutral_and_provider_specific_ssh_users" { command = plan diff --git a/modules/linode/variables.tf b/modules/linode/variables.tf index 2ce975e..71fed27 100644 --- a/modules/linode/variables.tf +++ b/modules/linode/variables.tf @@ -88,7 +88,9 @@ variable "runtime" { rootfs = optional(string, "") rootfs_archive_url = optional(string, "") rootfs_archive_sha256 = optional(string, "") - users = optional(map(list(string)), {}) + # Internal hosted-smoke escape hatch. Public provider entrypoints must not expose it. + rootfs_test_source_archive_prefix = optional(string, "") + users = optional(map(list(string)), {}) disaster_recovery = optional(object({ required = optional(bool, false) diff --git a/modules/linux-vm-runtime/main.tf b/modules/linux-vm-runtime/main.tf index fc5e168..9130e30 100644 --- a/modules/linux-vm-runtime/main.tf +++ b/modules/linux-vm-runtime/main.tf @@ -1,8 +1,12 @@ locals { - rootfs = "${path.module}/../../rootfs" - additional_rootfs = var.rootfs != "" ? var.rootfs : "" - rootfs_archive_url = trimspace(var.rootfs_archive_url) - rootfs_archive_sha256 = lower(trimspace(var.rootfs_archive_sha256)) + rootfs = "${path.module}/../../rootfs" + additional_rootfs = var.rootfs != "" ? var.rootfs : "" + rootfs_archive_url = trimspace(var.rootfs_archive_url) + rootfs_archive_sha256 = lower(trimspace(var.rootfs_archive_sha256)) + rootfs_test_source_archive_prefix = trimspace(var.rootfs_test_source_archive_prefix) + rootfs_archive_asset_url = split("#", split("?", local.rootfs_archive_url)[0])[0] + rootfs_contract_sidecar_url = local.rootfs_archive_url == "" || local.rootfs_test_source_archive_prefix != "" ? "" : replace(local.rootfs_archive_asset_url, "/[^/]+$/", "/cloud-compose-rootfs.contract.sha256") + rootfs_overlay_staging_path = "/var/lib/cloud-compose/rootfs-overlay" single_compose_project = { (var.name) = { @@ -82,30 +86,33 @@ locals { flatten([for _, app in local.compose_projects : app.sitectl_packages]) )) - base_files = local.rootfs_archive_url == "" ? fileset(local.rootfs, "**") : [] - additional_files = local.additional_rootfs != "" ? fileset(local.additional_rootfs, "**") : [] - embedded_additional_files = local.rootfs_archive_url == "" ? local.additional_files : [] + rootfs_contract_files = sort(tolist(fileset(local.rootfs, "**"))) + rootfs_contract_directories = sort(distinct(flatten([ + for file in local.rootfs_contract_files : dirname(file) == "." ? [] : [ + for index in range(length(split("/", dirname(file)))) : + join("/", slice(split("/", dirname(file)), 0, index + 1)) + ] + ]))) + rootfs_contract_sha256 = sha256(join("", [ + for entry in concat( + [for directory in local.rootfs_contract_directories : "d\t0:0:755\t${directory}\n"], + [for file in local.rootfs_contract_files : "f\t${filesha256("${local.rootfs}/${file}")}\t0:0:${endswith(file, ".sh") ? "755" : "644"}\t${file}\n"], + ) : entry + ])) + base_files = local.rootfs_archive_url == "" ? fileset(local.rootfs, "**") : [] + additional_files = local.additional_rootfs != "" ? fileset(local.additional_rootfs, "**") : [] all_files = merge( { for file in local.base_files : file => "${local.rootfs}/${file}" }, - { for file in local.embedded_additional_files : file => "${local.additional_rootfs}/${file}" } + { for file in local.additional_files : file => "${local.additional_rootfs}/${file}" } ) rootfs_file_permissions = { for file in setunion(local.base_files, local.additional_files) : file => endswith(file, ".sh") || "/${file}" == var.offhost_backup_driver_path ? "0755" : "0644" } - archive_additional_rootfs_commands = join("\n", [ - for file in local.additional_files : <<-EOT - destination="$(printf '%s' '${base64encode("/${file}")}' | base64 -d)" - install -d "$(dirname "$destination")" - printf '%s' '${filebase64("${local.additional_rootfs}/${file}")}' | base64 -d >"$destination" - chmod ${local.rootfs_file_permissions[file]} "$destination" - EOT - ]) - write_files_content = join("\n", [ for file, fullpath in local.all_files : <<-EOT - - path: ${jsonencode(startswith(file, "mnt/disks/") ? "/var/lib/cloud-compose/mounted-rootfs/${file}" : "/${file}")} + - path: ${jsonencode(local.rootfs_archive_url != "" ? "${local.rootfs_overlay_staging_path}/${file}" : startswith(file, "mnt/disks/") ? "/var/lib/cloud-compose/mounted-rootfs/${file}" : "/${file}")} owner: "root:root" permissions: ${jsonencode(local.rootfs_file_permissions[file])} encoding: gzip+base64 @@ -118,21 +125,13 @@ EOT - path: "/home/cloud-compose/${name}" owner: "root:root" permissions: "0755" - encoding: gzip+base64 - content: ${jsonencode(base64gzip(<<-EOS - #!/usr/bin/env bash - - set -eou pipefail - - source /home/cloud-compose/profile.sh - exec bash /home/cloud-compose/compose-dispatch.sh "${name}" - EOS -))} + encoding: b64 + content: ${filebase64("${local.rootfs}/home/cloud-compose/lifecycle-entrypoint.sh")} EOT -]) + ]) -compose_projects_content = jsonencode(local.validated_compose_projects) -compose_projects_file = <<-EOT + compose_projects_content = jsonencode(local.validated_compose_projects) + compose_projects_file = <<-EOT - path: "/home/cloud-compose/compose-projects.json" owner: "root:root" permissions: "0640" @@ -140,20 +139,20 @@ compose_projects_file = <<-EOT content: ${jsonencode(base64gzip(local.compose_projects_content))} EOT -managed_runtime_artifact_lines = [ - for artifact in module.managed_artifacts.artifacts : join("\t", [ - artifact.name, - artifact.url, - artifact.sha256, - artifact.path, - try(artifact.mode, "0755"), - try(artifact.owner, "root"), - try(artifact.group, "root"), - try(artifact.restart, ""), - ]) -] -managed_runtime_artifacts_content = length(local.managed_runtime_artifact_lines) == 0 ? "\n" : "${join("\n", local.managed_runtime_artifact_lines)}\n" -managed_runtime_artifacts_file = <<-EOT + managed_runtime_artifact_lines = [ + for artifact in module.managed_artifacts.artifacts : join("\t", [ + artifact.name, + artifact.url, + artifact.sha256, + artifact.path, + try(artifact.mode, "0755"), + try(artifact.owner, "root"), + try(artifact.group, "root"), + try(artifact.restart, ""), + ]) + ] + managed_runtime_artifacts_content = length(local.managed_runtime_artifact_lines) == 0 ? "\n" : "${join("\n", local.managed_runtime_artifact_lines)}\n" + managed_runtime_artifacts_file = <<-EOT - path: "/home/cloud-compose/managed-runtime-artifacts.tsv" owner: "root:root" permissions: "0640" @@ -161,8 +160,8 @@ managed_runtime_artifacts_file = <<-EOT content: ${jsonencode(base64gzip(local.managed_runtime_artifacts_content))} EOT -vault_agent_template_stanzas = join("\n", [ - for template in var.vault_agent_templates : <<-EOT + vault_agent_template_stanzas = join("\n", [ + for template in var.vault_agent_templates : <<-EOT template { destination = ${jsonencode(template.destination)} contents = <&2; exit 1 ;; - esac - case "$archive_url" in - *[[:space:]]*) echo "rootfs archive URL must not contain whitespace" >&2; exit 1 ;; - esac - if ! command -v curl >/dev/null 2>&1 || ! command -v tar >/dev/null 2>&1; then - if command -v apt-get >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y ca-certificates curl tar - elif command -v dnf >/dev/null 2>&1; then - dnf install -y ca-certificates curl tar - elif command -v rpm-ostree >/dev/null 2>&1; then - rpm-ostree install --apply-live ca-certificates curl tar - else - echo "No supported package manager found to install curl and tar" >&2 - exit 1 - fi - fi - for required_command in curl tar sha256sum; do - if ! command -v "$required_command" >/dev/null 2>&1; then - echo "$required_command is required to install the verified rootfs archive" >&2 - exit 1 - fi - done - tmp="$(mktemp -d)" - trap 'rm -rf "$tmp"' EXIT - curl -fsSL --proto '=https' --proto-redir '=https' --tlsv1.2 \ - --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 900 \ - --connect-timeout 10 --max-time 300 -o "$tmp/rootfs.tar.gz" -- "$archive_url" - printf '%s %s\n' "$archive_sha256" "$tmp/rootfs.tar.gz" | sha256sum -c - - tar --no-same-owner -xzf "$tmp/rootfs.tar.gz" -C "$tmp" - rootfs_dir="$(find "$tmp" -mindepth 1 -maxdepth 3 -type d -name rootfs -print -quit)" - if [ -z "$rootfs_dir" ] || [ ! -d "$rootfs_dir" ]; then - echo "rootfs directory not found in verified archive $archive_url" >&2 - exit 1 - fi - chown -hR 0:0 -- "$rootfs_dir" - filesystem_prep_source="$rootfs_dir/home/cloud-compose/prepare-filesystem.sh" - filesystem_persist_source="$rootfs_dir/home/cloud-compose/persist-filesystems.sh" - if [ ! -f "$filesystem_prep_source" ] || [ ! -f "$filesystem_persist_source" ]; then - echo "verified rootfs archive is missing filesystem preparation scripts" >&2 - exit 1 - fi - install -m 0600 -- "$filesystem_prep_source" "$filesystem_prep" - install -m 0600 -- "$filesystem_persist_source" "$filesystem_persist" - EOT +data "http" "rootfs_contract" { + count = local.rootfs_archive_url != "" && local.rootfs_test_source_archive_prefix == "" ? 1 : 0 -rootfs_archive_install_command_raw = <<-EOT - if [ -z "$${rootfs_dir:-}" ] || [ ! -d "$rootfs_dir" ]; then - echo "verified rootfs directory is unavailable during installation" >&2 - exit 1 - fi - cp -a "$rootfs_dir"/. / - EOT + url = local.rootfs_contract_sidecar_url + request_timeout_ms = 30000 -rootfs_archive_prepare_command = local.rootfs_archive_url != "" ? local.rootfs_archive_prepare_command_raw : "" -rootfs_archive_install_command = local.rootfs_archive_url != "" ? local.rootfs_archive_install_command_raw : "" - -cloud_init = templatefile("${path.module}/templates/cloud-init.yml", { - CLOUD_COMPOSE_SSH_KEYS = var.cloud_compose_ssh_keys - SSH_USERS = var.ssh_users - DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootfs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh") - DATA_DEVICE = var.data_device - VOLUMES_DEVICE = var.volumes_device - WRITE_FILES_CONTENT = local.write_files_content - DOCKER_COMPOSE_SCRIPTS = local.docker_compose_scripts - COMPOSE_PROJECTS_FILE = local.compose_projects_file - ENV_FILE_CONTENT = local.env_file_content - APPLICATION_ENV_FILE_CONTENT = local.application_env_file_content - VAULT_AGENT_FILES = local.vault_agent_files - MANAGED_RUNTIME_ARTIFACTS_FILE = local.managed_runtime_artifacts_file - ROLLOUT_RUNCMD = var.rollout_enabled ? "bash /home/cloud-compose/deploy-rollout.sh >> /home/cloud-compose/run.log 2>&1" : "" - ROOTFS_ARCHIVE_ENABLED = local.rootfs_archive_url != "" - ROOTFS_ARCHIVE_PREPARE_COMMAND = local.rootfs_archive_prepare_command - ROOTFS_ARCHIVE_INSTALL_COMMAND = local.rootfs_archive_install_command - ARCHIVE_ADDITIONAL_ROOTFS = local.archive_additional_rootfs_commands - FILESYSTEM_PREP_SCRIPT_B64 = local.rootfs_archive_url == "" ? filebase64("${local.rootfs}/home/cloud-compose/prepare-filesystem.sh") : "" - FILESYSTEM_PERSIST_SCRIPT_B64 = local.rootfs_archive_url == "" ? filebase64("${local.rootfs}/home/cloud-compose/persist-filesystems.sh") : "" -}) + lifecycle { + postcondition { + condition = ( + self.status_code == 200 && + can(regex("^[0-9a-f]{64}\\n?$", self.response_body)) && + trimspace(self.response_body) == local.rootfs_contract_sha256 + ) + error_message = "The immutable rootfs release sidecar must contain exactly this module source's canonical rootfs contract digest. Publish or select a matching archive before replacing a VM." + } + } } module "sitectl_runtime" { diff --git a/modules/linux-vm-runtime/outputs.tf b/modules/linux-vm-runtime/outputs.tf index d021d89..4b2f973 100644 --- a/modules/linux-vm-runtime/outputs.tf +++ b/modules/linux-vm-runtime/outputs.tf @@ -9,6 +9,21 @@ output "cloud_init" { ) error_message = "rootfs_archive_url and a 64-character rootfs_archive_sha256 must be supplied together." } + precondition { + condition = ( + local.rootfs_archive_url == "" || + local.rootfs_test_source_archive_prefix != "" || + trimspace(try(data.http.rootfs_contract[0].response_body, "")) == local.rootfs_contract_sha256 + ) + error_message = "The rootfs archive release contract must match this module before cloud-init is rendered." + } + precondition { + condition = ( + local.rootfs_test_source_archive_prefix == "" || + local.rootfs_archive_url == "https://github.com/libops/cloud-compose/archive/${trimprefix(local.rootfs_test_source_archive_prefix, "cloud-compose-")}.tar.gz" + ) + error_message = "The test-only rootfs source archive must use the exact libops/cloud-compose commit named by rootfs_test_source_archive_prefix." + } } output "compose_projects" { diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index 8aa38fb..f5afb43 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -1,3 +1,12 @@ +mock_provider "http" { + mock_data "http" { + defaults = { + response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + status_code = 200 + } + } +} + run "renders_safe_ssh_values" { command = plan @@ -28,11 +37,14 @@ run "renders_safe_ssh_values" { assert { condition = ( - strcontains(output.cloud_init, "path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh") && + strcontains(output.cloud_init, "path: \"/etc/cloud-compose/bin/cloud-compose-diagnostics.sh\"") && strcontains(output.cloud_init, "NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state") && strcontains(output.cloud_init, "NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status") && strcontains(output.cloud_init, "NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump") && - strcontains(output.cloud_init, "install -m 0755 -o root -g root") + strcontains( + local.write_files_content, + "- path: \"/etc/cloud-compose/libexec/linux-vm-cloud-init.sh\"", + ) ) error_message = "Cloud-init must install one root-owned diagnostics program with exact passwordless sudo commands." } @@ -219,48 +231,39 @@ run "renders_verified_rootfs_archive" { offhost_backup_driver_path = "/etc/cloud-compose/libexec/custom-offhost-driver" } - assert { - condition = strcontains(output.cloud_init, "sha256sum -c -") - error_message = "Archive cloud-init must verify SHA-256 before extraction." - } - assert { condition = ( - strcontains(output.cloud_init, "tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"") && - strcontains(output.cloud_init, "chown -hR 0:0 -- \"$rootfs_dir\"") && - strcontains(output.cloud_init, "install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"") && - strcontains(output.cloud_init, "bash \"$filesystem_prep\"") && - strcontains(output.cloud_init, "bash \"$filesystem_persist\"") && - strcontains(output.cloud_init, "cp -a \"$rootfs_dir\"/. /") && - length(split("sha256sum -c -", output.cloud_init)[0]) < length(split("tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) && - length(split("tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) < length(split("chown -hR 0:0 -- \"$rootfs_dir\"", output.cloud_init)[0]) && - length(split("chown -hR 0:0 -- \"$rootfs_dir\"", output.cloud_init)[0]) < length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) && - length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) < length(split("bash \"$filesystem_prep\"", output.cloud_init)[0]) && - length(split("bash \"$filesystem_persist\"", output.cloud_init)[0]) < length(split("cp -a \"$rootfs_dir\"/. /", output.cloud_init)[0]) + strcontains( + output.cloud_init, + base64gzip(file("${path.module}/../../rootfs/etc/cloud-compose/libexec/rootfs-archive.sh")), + ) && + length(data.http.rootfs_contract) == 1 && + strcontains(output.cloud_init, local.rootfs_contract_sha256) && + data.http.rootfs_contract[0].url == "https://example.invalid/cloud-compose-rootfs.contract.sha256" && + strcontains(output.cloud_init, "[bash, /var/lib/cloud-compose/bootstrap/rootfs-archive.sh, prepare-linux") && + strcontains(output.cloud_init, "[bash, /var/lib/cloud-compose/bootstrap/linux-vm-cloud-init.sh") ) - error_message = "Archive cloud-init must verify, extract, and normalize root ownership before loading its helpers, then install the rootfs only after filesystem preparation and persistence." + error_message = "Archive-mode Linux cloud-init must transfer compressed checked bootstrap programs, bind the archive to the exact module rootfs, and invoke stable paths." } assert { condition = ( - !strcontains(output.cloud_init, filebase64("${path.module}/../../rootfs/home/cloud-compose/prepare-filesystem.sh")) && - !strcontains(output.cloud_init, filebase64("${path.module}/../../rootfs/home/cloud-compose/persist-filesystems.sh")) + !strcontains( + local.write_files_content, + base64gzip(file("${path.module}/../../rootfs/home/cloud-compose/prepare-filesystem.sh")), + ) && + !strcontains( + local.write_files_content, + base64gzip(file("${path.module}/../../rootfs/home/cloud-compose/persist-filesystems.sh")), + ) ) error_message = "Archive-backed cloud-init must not embed the filesystem helper payloads." } assert { condition = ( - strcontains(output.cloud_init, "verified rootfs archive is missing filesystem preparation scripts") && - strcontains(output.cloud_init, "verified rootfs directory is unavailable during installation") - ) - error_message = "Archive-backed cloud-init must fail closed when the verified rootfs or its filesystem helpers are missing." - } - - assert { - condition = ( - strcontains(output.cloud_init, "archive_url_b64=") && - !strcontains(output.cloud_init, "$(id)") + strcontains(output.cloud_init, base64encode(var.rootfs_archive_url)) && + !strcontains(output.cloud_init, var.rootfs_archive_url) ) error_message = "Archive URLs must be rendered as base64 data rather than executable shell syntax." } @@ -270,18 +273,123 @@ run "renders_verified_rootfs_archive" { local.rootfs_file_permissions["etc/cloud-compose/libexec/custom-offhost-driver"] == "0755" && local.rootfs_file_permissions["etc/cloud-compose/unrelated-config"] == "0644" && strcontains( - split(base64encode("/etc/cloud-compose/libexec/custom-offhost-driver"), local.archive_additional_rootfs_commands)[1], - "chmod 0755 \"$destination\"", + local.write_files_content, + "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose/libexec/custom-offhost-driver\"\n owner: \"root:root\"\n permissions: \"0755\"", ) && strcontains( - split(base64encode("/etc/cloud-compose/unrelated-config"), local.archive_additional_rootfs_commands)[1], - "chmod 0644 \"$destination\"", + local.write_files_content, + "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose/unrelated-config\"\n owner: \"root:root\"\n permissions: \"0644\"", ) ) error_message = "Archive-backed Linux VM overlays must make only the configured off-host backup driver executable." } } +run "renders_exact_current_source_archive_for_hosted_smoke" { + command = plan + + variables { + name = "contract-test" + provider_name = "linode" + region = "us-east" + data_device = "/dev/test-data" + volumes_device = "/dev/test-volumes" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" + } + + assert { + condition = ( + length(data.http.rootfs_contract) == 0 && + strcontains(output.cloud_init, "prepare-linux-test-source") && + strcontains(output.cloud_init, jsonencode(var.rootfs_test_source_archive_prefix)) && + strcontains(output.cloud_init, local.rootfs_contract_sha256) + ) + error_message = "Hosted smoke source-archive mode must skip the unavailable release sidecar while binding the exact source rootfs to this module contract." + } +} + +run "rejects_source_archive_from_another_commit" { + command = plan + + variables { + name = "contract-test" + provider_name = "linode" + region = "us-east" + data_device = "/dev/test-data" + volumes_device = "/dev/test-volumes" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/2222222222222222222222222222222222222222.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" + } + + expect_failures = [output.cloud_init] +} + +run "rejects_tag_named_source_archive_prefix" { + command = plan + + variables { + name = "contract-test" + provider_name = "linode" + region = "us-east" + data_device = "/dev/test-data" + volumes_device = "/dev/test-volumes" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/refs/tags/v1.2.3.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-v1.2.3" + } + + expect_failures = [var.rootfs_test_source_archive_prefix] +} + +run "rejects_arbitrary_test_source_archive_url" { + command = plan + + variables { + name = "contract-test" + provider_name = "linode" + region = "us-east" + data_device = "/dev/test-data" + volumes_device = "/dev/test-volumes" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs_archive_url = "https://example.invalid/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" + } + + expect_failures = [output.cloud_init] +} + +run "rejects_rootfs_release_from_another_module_version" { + command = plan + + variables { + name = "contract-test" + provider_name = "linode" + region = "us-east" + data_device = "/dev/test-data" + volumes_device = "/dev/test-volumes" + docker_compose_repo = "https://github.com/libops/wp.git" + rootfs_archive_url = "https://example.invalid/cloud-compose-rootfs.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + + override_data { + target = data.http.rootfs_contract[0] + values = { + response_body = "0000000000000000000000000000000000000000000000000000000000000000\n" + status_code = 200 + } + } + + expect_failures = [data.http.rootfs_contract[0]] +} + run "embeds_filesystem_helpers_without_archive" { command = plan @@ -298,11 +406,17 @@ run "embeds_filesystem_helpers_without_archive" { assert { condition = ( - strcontains(output.cloud_init, filebase64("${path.module}/../../rootfs/home/cloud-compose/prepare-filesystem.sh")) && - strcontains(output.cloud_init, filebase64("${path.module}/../../rootfs/home/cloud-compose/persist-filesystems.sh")) && - !strcontains(output.cloud_init, "archive_url_b64=") + strcontains( + local.write_files_content, + base64gzip(file("${path.module}/../../rootfs/home/cloud-compose/prepare-filesystem.sh")), + ) && + strcontains( + local.write_files_content, + base64gzip(file("${path.module}/../../rootfs/home/cloud-compose/persist-filesystems.sh")), + ) && + !strcontains(local.write_files_content, "/var/lib/cloud-compose/rootfs-overlay/") ) - error_message = "Inline cloud-init must retain the embedded filesystem-helper bootstrap when no archive is configured." + error_message = "Non-archive cloud-init must transfer the checked-in filesystem helpers without using the archive overlay." } assert { diff --git a/modules/linux-vm-runtime/templates/cloud-init.yml b/modules/linux-vm-runtime/templates/cloud-init.yml index 7cac7c0..3d812f5 100644 --- a/modules/linux-vm-runtime/templates/cloud-init.yml +++ b/modules/linux-vm-runtime/templates/cloud-init.yml @@ -29,10 +29,22 @@ users: %{ endfor ~} write_files: +- path: /var/lib/cloud-compose/bootstrap/rootfs-archive.sh + owner: root:root + permissions: "0700" + encoding: gzip+base64 + content: ${ROOTFS_ARCHIVE_SCRIPT_B64} +%{ if ROOTFS_ARCHIVE_ENABLED ~} +- path: /var/lib/cloud-compose/bootstrap/linux-vm-cloud-init.sh + owner: root:root + permissions: "0700" + encoding: gzip+base64 + content: ${LINUX_CLOUD_INIT_SCRIPT_B64} +%{ endif ~} - path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh owner: root:root permissions: "0600" - encoding: b64 + encoding: gzip+base64 content: ${DIAGNOSTICS_SCRIPT_B64} ${WRITE_FILES_CONTENT} ${DOCKER_COMPOSE_SCRIPTS} @@ -43,78 +55,13 @@ ${VAULT_AGENT_FILES} ${MANAGED_RUNTIME_ARTIFACTS_FILE} runcmd: -- | - set -eu - ensure_cloud_compose_user() { - if ! getent group docker >/dev/null 2>&1; then - groupadd docker - fi - if ! id -u cloud-compose >/dev/null 2>&1; then - useradd --create-home --shell /bin/bash --groups docker cloud-compose - else - usermod --append --groups docker cloud-compose - fi - install -d -m 0700 -o cloud-compose -g cloud-compose /home/cloud-compose/.ssh - cat >/home/cloud-compose/.ssh/authorized_keys <<'EOF' -%{ for key in CLOUD_COMPOSE_SSH_KEYS ~} - ${key} -%{ endfor ~} - EOF - chown cloud-compose:cloud-compose /home/cloud-compose/.ssh/authorized_keys - chmod 0600 /home/cloud-compose/.ssh/authorized_keys - install -d -m 0755 /etc/sudoers.d - printf '%s\n' \ - 'cloud-compose ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service' \ - 'cloud-compose ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state' \ - 'cloud-compose ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status' \ - 'cloud-compose ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump' \ - >/etc/sudoers.d/90-cloud-compose - chmod 0440 /etc/sudoers.d/90-cloud-compose - } - - ensure_cloud_compose_user - filesystem_prep=/run/cloud-compose-prepare-filesystem - filesystem_persist=/run/cloud-compose-persist-filesystems -%{ if ROOTFS_ARCHIVE_ENABLED } - ${indent(2, ROOTFS_ARCHIVE_PREPARE_COMMAND)} -%{ else } - printf '%s' '${FILESYSTEM_PREP_SCRIPT_B64}' | base64 -d >"$filesystem_prep" - printf '%s' '${FILESYSTEM_PERSIST_SCRIPT_B64}' | base64 -d >"$filesystem_persist" - chmod 0600 "$filesystem_prep" "$filesystem_persist" -%{ endif } - bash "$filesystem_prep" '${DATA_DEVICE}' /mnt/disks/data --publish-fresh-marker - bash "$filesystem_prep" '${VOLUMES_DEVICE}' /mnt/disks/volumes - mkdir -p /mnt/disks/data/docker/volumes - if ! mountpoint -q /mnt/disks/data/docker/volumes; then - mount --bind /mnt/disks/volumes /mnt/disks/data/docker/volumes - fi - for required_mount in /mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes; do - if ! mountpoint -q -- "$required_mount"; then - echo "Required cloud-compose mount is unavailable: $required_mount" >&2 - exit 1 - fi - done - bash "$filesystem_persist" '${DATA_DEVICE}' '${VOLUMES_DEVICE}' -%{ if ROOTFS_ARCHIVE_ENABLED } - ${indent(2, ROOTFS_ARCHIVE_INSTALL_COMMAND)} - ${indent(2, ARCHIVE_ADDITIONAL_ROOTFS)} -%{ else } - if [ -d /var/lib/cloud-compose/mounted-rootfs/mnt/disks ]; then - cp -a /var/lib/cloud-compose/mounted-rootfs/mnt/disks/. /mnt/disks/ - fi -%{ endif } - install -d -m 0755 -o root -g root /etc/cloud-compose/bin - install -m 0755 -o root -g root \ - /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh \ - /etc/cloud-compose/bin/cloud-compose-diagnostics.sh - chown root:cloud-compose /mnt/disks/data - chmod 1775 /mnt/disks/data - chown cloud-compose:cloud-compose /mnt/disks/volumes - chmod 0775 /mnt/disks/volumes - install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops -%{ if ROLLOUT_RUNCMD != "" ~} - ${ROLLOUT_RUNCMD} +%{ if ROOTFS_ARCHIVE_ENABLED ~} +%{ if ROOTFS_TEST_SOURCE_ARCHIVE_PREFIX != "" ~} +- [bash, /var/lib/cloud-compose/bootstrap/rootfs-archive.sh, prepare-linux-test-source, ${jsonencode(ROOTFS_ARCHIVE_URL_B64)}, ${jsonencode(ROOTFS_ARCHIVE_SHA256)}, ${jsonencode(ROOTFS_CONTRACT_SHA256)}, ${jsonencode(LINUX_CLOUD_INIT_SCRIPT_SHA256)}, ${jsonencode(ROOTFS_TEST_SOURCE_ARCHIVE_PREFIX)}] +%{ else ~} +- [bash, /var/lib/cloud-compose/bootstrap/rootfs-archive.sh, prepare-linux, ${jsonencode(ROOTFS_ARCHIVE_URL_B64)}, ${jsonencode(ROOTFS_ARCHIVE_SHA256)}, ${jsonencode(ROOTFS_CONTRACT_SHA256)}, ${jsonencode(LINUX_CLOUD_INIT_SCRIPT_SHA256)}] +%{ endif ~} +- [bash, /var/lib/cloud-compose/bootstrap/linux-vm-cloud-init.sh, ${jsonencode(DATA_DEVICE)}, ${jsonencode(VOLUMES_DEVICE)}, "true", ${jsonencode(ROLLOUT_ENABLED)}, ${jsonencode(DIAGNOSTICS_SCRIPT_SHA256)}] +%{ else ~} +- [bash, /etc/cloud-compose/libexec/linux-vm-cloud-init.sh, ${jsonencode(DATA_DEVICE)}, ${jsonencode(VOLUMES_DEVICE)}, "false", ${jsonencode(ROLLOUT_ENABLED)}, ${jsonencode(DIAGNOSTICS_SCRIPT_SHA256)}] %{ endif ~} - rm -f /var/lib/cloud-compose/bootstrap-complete - /etc/cloud-compose/libexec/harden-bootstrap-paths.sh - bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh diff --git a/modules/linux-vm-runtime/variables.tf b/modules/linux-vm-runtime/variables.tf index 2ec8c3c..c01d590 100644 --- a/modules/linux-vm-runtime/variables.tf +++ b/modules/linux-vm-runtime/variables.tf @@ -79,7 +79,7 @@ variable "rootfs" { variable "rootfs_archive_url" { type = string default = "" - description = "Optional HTTPS tar.gz URL containing a rootfs directory to fetch during boot instead of embedding the packaged rootfs in cloud-init. Must be set with rootfs_archive_sha256." + description = "Optional immutable HTTPS cloud-compose rootfs release archive to fetch during boot instead of embedding the packaged rootfs in cloud-init. Must be set with rootfs_archive_sha256; planning also requires the adjacent cloud-compose-rootfs.contract.sha256 asset to match this module source." validation { condition = ( @@ -96,6 +96,20 @@ variable "rootfs_archive_sha256" { description = "Required 64-character SHA-256 checksum when rootfs_archive_url is set." } +variable "rootfs_test_source_archive_prefix" { + type = string + default = "" + description = "Test-only GitHub source-archive prefix for hosted smoke tests of an unreleased exact commit. Empty keeps the production canonical release archive and adjacent sidecar contract mandatory." + + validation { + condition = ( + trimspace(var.rootfs_test_source_archive_prefix) == "" || + can(regex("^cloud-compose-[0-9a-f]{40}$", trimspace(var.rootfs_test_source_archive_prefix))) + ) + error_message = "rootfs_test_source_archive_prefix must be empty or cloud-compose- followed by one exact lowercase 40-character commit SHA." + } +} + variable "offhost_backup_required" { type = bool default = false @@ -209,7 +223,7 @@ variable "compose_projects" { variable "docker_compose_init" { type = list(string) default = [ - "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --yolo --default" + "/home/cloud-compose/default-lifecycle.sh init" ] nullable = false description = "Commands run after a compose repository is cloned." @@ -218,9 +232,7 @@ variable "docker_compose_init" { variable "docker_compose_up" { type = list(string) default = [ - "sitectl compose --context \"$${SITECTL_CONTEXT_NAME}\" up -d --remove-orphans", - "sitectl healthcheck --context \"$${SITECTL_CONTEXT_NAME}\" --persist", - "if [ \"$${SITECTL_ENVIRONMENT}\" != \"production\" ]; then sitectl verify --context \"$${SITECTL_CONTEXT_NAME}\" $${SITECTL_VERIFY_ARGS:-}; fi" + "/home/cloud-compose/default-lifecycle.sh up" ] nullable = false description = "Commands used to bring a compose project up." @@ -229,7 +241,7 @@ variable "docker_compose_up" { variable "docker_compose_down" { type = list(string) default = [ - "sitectl compose --context \"$${SITECTL_CONTEXT_NAME}\" down" + "/home/cloud-compose/default-lifecycle.sh down" ] nullable = false description = "Commands used to stop a compose project." @@ -238,13 +250,10 @@ variable "docker_compose_down" { variable "docker_compose_rollout" { type = list(string) default = [ - "TARGET_REF=\"$${GIT_REF:-$${GIT_BRANCH:-}}\"", - "if [ -n \"$TARGET_REF\" ]; then sitectl deploy --context \"$${SITECTL_CONTEXT_NAME}\" --ref \"$TARGET_REF\"; else sitectl deploy --context \"$${SITECTL_CONTEXT_NAME}\" --skip-git; fi", - "sitectl healthcheck --context \"$${SITECTL_CONTEXT_NAME}\" --persist", - "if [ \"$${SITECTL_ENVIRONMENT}\" != \"production\" ]; then sitectl verify --context \"$${SITECTL_CONTEXT_NAME}\" $${SITECTL_VERIFY_ARGS:-}; fi" + "/home/cloud-compose/default-lifecycle.sh rollout" ] nullable = false - description = "Commands used by rollout triggers. GIT_REF/GIT_BRANCH selects a source ref; without one, sitectl reconciles the current checkout." + description = "Commands used by rollout triggers. A validated GIT_COMMIT_SHA takes precedence over GIT_REF/GIT_BRANCH; without one, sitectl reconciles the current checkout." } variable "rollout_enabled" { diff --git a/modules/linux-vm-runtime/versions.tf b/modules/linux-vm-runtime/versions.tf index 12ad22a..1135eed 100644 --- a/modules/linux-vm-runtime/versions.tf +++ b/modules/linux-vm-runtime/versions.tf @@ -1,3 +1,10 @@ terraform { required_version = ">= 1.3.0" + + required_providers { + http = { + source = "hashicorp/http" + version = "~> 3.6" + } + } } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index 3fc8276..b61494a 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -1,4 +1,12 @@ mock_provider "digitalocean" {} +mock_provider "http" { + mock_data "http" { + defaults = { + response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + status_code = 200 + } + } +} run "custom_package_set_merges_only_applicable_template_versions" { command = plan diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index 14be098..84936d0 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -1,4 +1,12 @@ mock_provider "linode" {} +mock_provider "http" { + mock_data "http" { + defaults = { + response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + status_code = 200 + } + } +} run "custom_package_set_merges_only_applicable_template_versions" { command = plan diff --git a/rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq b/rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq new file mode 100644 index 0000000..fc24963 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq @@ -0,0 +1,5 @@ +if type == "array" and all(.[]; type == "string") then + .[] | ("x" + @base64) +else + error("sitectl verify arguments must be an array of strings") +end diff --git a/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh b/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh index 27e77c2..192f8aa 100755 --- a/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh +++ b/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh @@ -54,6 +54,7 @@ cloud_compose_secure_runtime_home() { "$CLOUD_COMPOSE_RUNTIME_HOME/host-conf.sh" "$CLOUD_COMPOSE_RUNTIME_HOME/host-init.sh" "$CLOUD_COMPOSE_RUNTIME_HOME/converge-app-filesystems.sh" + "$CLOUD_COMPOSE_RUNTIME_HOME/default-lifecycle.sh" "$CLOUD_COMPOSE_RUNTIME_HOME/prepare-app-sources.sh" "$CLOUD_COMPOSE_RUNTIME_HOME/rotate-keys-daily.sh" "$CLOUD_COMPOSE_RUNTIME_HOME/vault-agent-init.sh" diff --git a/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh b/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh new file mode 100755 index 0000000..63eed17 --- /dev/null +++ b/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -eu + +if [[ "$#" -ne 2 ]]; then + echo "usage: gcp-cloud-init-finalize.sh INIT_COMMANDS_FILE DIAGNOSTICS_SHA256" >&2 + exit 2 +fi + +init_commands_file="$1" +diagnostics_sha256="$2" + +bash /var/lib/cloud-compose/bootstrap/rootfs-archive.sh \ + install-diagnostics "$diagnostics_sha256" +if [[ ! -f /run/cloud-compose-filesystems-ready ]]; then + echo "Cloud Compose filesystems were not prepared; refusing application initialization" >&2 + exit 1 +fi +if [[ -s "$init_commands_file" ]]; then + # Operator-provided initialization commands are stored as a root-controlled + # program instead of being interpolated into the cloud-init shell body. + # shellcheck disable=SC1090 + source "$init_commands_file" +fi +chown root:cloud-compose /mnt/disks/data +chmod 1775 /mnt/disks/data +chown cloud-compose:cloud-compose /mnt/disks/volumes +chmod 0775 /mnt/disks/volumes +install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops +rm -f /var/lib/cloud-compose/bootstrap-complete +/etc/cloud-compose/libexec/harden-bootstrap-paths.sh +bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh diff --git a/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-post-bootstrap.sh b/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-post-bootstrap.sh new file mode 100755 index 0000000..16449d4 --- /dev/null +++ b/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-post-bootstrap.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -eu + +if [[ "$#" -ne 2 ]]; then + echo "usage: gcp-cloud-init-post-bootstrap.sh ROLLOUT_ENABLED RUNCMD_FILE" >&2 + exit 2 +fi + +rollout_enabled="$1" +runcmd_file="$2" +case "$rollout_enabled" in + true | false) ;; + *) + echo "ROLLOUT_ENABLED must be true or false" >&2 + exit 2 + ;; +esac + +bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh +if [[ "$rollout_enabled" == "true" ]]; then + bash /home/cloud-compose/deploy-rollout.sh >>/home/cloud-compose/run.log 2>&1 +fi +if [[ -s "$runcmd_file" ]]; then + # shellcheck disable=SC1090 + source "$runcmd_file" +fi diff --git a/rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh b/rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh new file mode 100755 index 0000000..8d5a373 --- /dev/null +++ b/rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 4 ]]; then + echo "usage: gcp-filesystem-boot.sh FRESH_FILESYSTEM_IDENTITY USE_OVERLAY PREP_PROGRAM PERSIST_PROGRAM" >&2 + exit 2 +fi + +fresh_filesystem_identity="$1" +use_overlay="$2" +filesystem_prep="$3" +filesystem_persist="$4" + +case "$use_overlay" in + true | false) ;; + *) + echo "USE_OVERLAY must be true or false" >&2 + exit 2 + ;; +esac + +rm -f /run/cloud-compose-filesystems-ready +bash "$filesystem_prep" /dev/disk/by-id/google-data /mnt/disks/data \ + --publish-fresh-marker "$fresh_filesystem_identity" +bash "$filesystem_prep" /dev/disk/by-id/google-docker-volumes /mnt/disks/volumes +mkdir -p /mnt/disks/data/docker/volumes +if ! mountpoint -q /mnt/disks/data/docker/volumes; then + mount --bind /mnt/disks/volumes /mnt/disks/data/docker/volumes +fi +for required_mount in /mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes; do + if ! mountpoint -q -- "$required_mount"; then + echo "Required cloud-compose mount is unavailable: $required_mount" >&2 + exit 1 + fi +done + +if [[ "$use_overlay" == "true" ]]; then + mkdir -p /mnt/disks/prod-readonly + if ! mountpoint -q /mnt/disks/prod-readonly; then + mount -o ro "$(readlink -f /dev/disk/by-id/google-prod-volumes)" \ + /mnt/disks/prod-readonly + fi + bash "$filesystem_persist" \ + /dev/disk/by-id/google-data \ + /dev/disk/by-id/google-docker-volumes \ + /dev/disk/by-id/google-prod-volumes +else + bash "$filesystem_persist" \ + /dev/disk/by-id/google-data \ + /dev/disk/by-id/google-docker-volumes +fi +install -m 0600 /dev/null /run/cloud-compose-filesystems-ready diff --git a/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh b/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh new file mode 100755 index 0000000..7d921dd --- /dev/null +++ b/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 5 ]]; then + echo "usage: linux-vm-cloud-init.sh DATA_DEVICE VOLUMES_DEVICE ROOTFS_ARCHIVE_ENABLED ROLLOUT_ENABLED DIAGNOSTICS_SHA256" >&2 + exit 2 +fi + +data_device="$1" +volumes_device="$2" +rootfs_archive_enabled="$3" +rollout_enabled="$4" +diagnostics_sha256="$5" +readonly bootstrap_dir=/var/lib/cloud-compose/bootstrap +readonly archive_program="$bootstrap_dir/rootfs-archive.sh" +readonly overlay_dir=/var/lib/cloud-compose/rootfs-overlay +readonly filesystem_prep=/run/cloud-compose-prepare-filesystem +readonly filesystem_persist=/run/cloud-compose-persist-filesystems + +for boolean_name in rootfs_archive_enabled rollout_enabled; do + case "${!boolean_name}" in + true | false) ;; + *) + echo "${boolean_name^^} must be true or false" >&2 + exit 2 + ;; + esac +done +if ! id -u cloud-compose >/dev/null 2>&1 || ! getent group docker >/dev/null 2>&1; then + echo "cloud-init did not create the cloud-compose user and docker group" >&2 + exit 1 +fi +cloud_compose_in_docker_group=false +for account_group in $(id -nG cloud-compose); do + if [[ "$account_group" == "docker" ]]; then + cloud_compose_in_docker_group=true + break + fi +done +if [[ "$cloud_compose_in_docker_group" != "true" ]]; then + echo "cloud-init did not add cloud-compose to the docker group" >&2 + exit 1 +fi + +if [[ "$rootfs_archive_enabled" == "false" ]]; then + install -m 0600 -- /home/cloud-compose/prepare-filesystem.sh "$filesystem_prep" + install -m 0600 -- /home/cloud-compose/persist-filesystems.sh "$filesystem_persist" +fi + +bash "$filesystem_prep" "$data_device" /mnt/disks/data --publish-fresh-marker +bash "$filesystem_prep" "$volumes_device" /mnt/disks/volumes +mkdir -p /mnt/disks/data/docker/volumes +if ! mountpoint -q /mnt/disks/data/docker/volumes; then + mount --bind /mnt/disks/volumes /mnt/disks/data/docker/volumes +fi +for required_mount in /mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes; do + if ! mountpoint -q -- "$required_mount"; then + echo "Required cloud-compose mount is unavailable: $required_mount" >&2 + exit 1 + fi +done +bash "$filesystem_persist" "$data_device" "$volumes_device" + +if [[ "$rootfs_archive_enabled" == "true" ]]; then + bash "$archive_program" install-staged "$overlay_dir" +elif [[ -d /var/lib/cloud-compose/mounted-rootfs/mnt/disks ]]; then + cp -a /var/lib/cloud-compose/mounted-rootfs/mnt/disks/. /mnt/disks/ +fi + +bash "$archive_program" install-diagnostics "$diagnostics_sha256" +chown root:cloud-compose /mnt/disks/data +chmod 1775 /mnt/disks/data +chown cloud-compose:cloud-compose /mnt/disks/volumes +chmod 0775 /mnt/disks/volumes +install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops + +if [[ "$rollout_enabled" == "true" ]]; then + bash /home/cloud-compose/deploy-rollout.sh >>/home/cloud-compose/run.log 2>&1 +fi +rm -f /var/lib/cloud-compose/bootstrap-complete +/etc/cloud-compose/libexec/harden-bootstrap-paths.sh +bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh diff --git a/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh b/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh new file mode 100755 index 0000000..79955f1 --- /dev/null +++ b/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh @@ -0,0 +1,349 @@ +#!/usr/bin/env bash + +set -euo pipefail + +readonly stage_root=/run/cloud-compose-rootfs-stage +readonly staged_rootfs="$stage_root/rootfs" +readonly bootstrap_dir=/var/lib/cloud-compose/bootstrap +readonly filesystem_prep=/run/cloud-compose-prepare-filesystem +readonly filesystem_persist=/run/cloud-compose-persist-filesystems + +fail() { + echo "rootfs archive: $*" >&2 + exit 1 +} + +canonical_file_mode() { + case "$1" in + *.sh) printf '755\n' ;; + *) printf '644\n' ;; + esac +} + +require_safe_contract_path() { + local relative_path="$1" + + [[ -n "$relative_path" && "$relative_path" != /* && + "$relative_path" != *$'\t'* && "$relative_path" != *$'\r'* && + "$relative_path" != *$'\n'* && "$relative_path" != *'//'* + && ! "$relative_path" =~ (^|/)\.\.?(/|$) ]] || + fail "rootfs contract contains an unsafe path" +} + +validate_rootfs_archive() { + local archive_path="$1" + local member_path normalized_path member_listing member_type + local rootfs_present=false + + [[ -f "$archive_path" && ! -L "$archive_path" ]] || + fail "rootfs archive must be a regular file" + while IFS= read -r member_path; do + normalized_path="${member_path%/}" + require_safe_contract_path "$normalized_path" + [[ "$normalized_path" == rootfs || "$normalized_path" == rootfs/* ]] || + fail "rootfs archive contains a path outside rootfs" + if [[ "$normalized_path" == rootfs ]]; then + rootfs_present=true + fi + done < <(LC_ALL=C tar --quoting-style=literal -tzf "$archive_path") + [[ "$rootfs_present" == "true" ]] || + fail "rootfs archive does not contain its rootfs directory" + + while IFS= read -r member_listing; do + member_type="${member_listing:0:1}" + [[ "$member_type" == "-" || "$member_type" == "d" ]] || + fail "rootfs archive contains a link or unsupported filesystem object" + done < <(LC_ALL=C tar --numeric-owner --quoting-style=escape -tvzf "$archive_path") +} + +validate_rootfs_test_source_archive() { + local archive_path="$1" test_source_prefix="$2" + local member_path normalized_path member_listing member_type + local rootfs_present=false + + [[ -f "$archive_path" && ! -L "$archive_path" ]] || + fail "test-only rootfs source archive must be a regular file" + [[ "$test_source_prefix" =~ ^cloud-compose-([0-9a-f]{40})$ ]] || + fail "test-only rootfs source archive prefix must identify one exact cloud-compose commit" + while IFS= read -r member_path; do + normalized_path="${member_path%/}" + require_safe_contract_path "$normalized_path" + [[ "$normalized_path" == "$test_source_prefix" || + "$normalized_path" == "$test_source_prefix/"* ]] || + fail "test-only rootfs source archive contains a path outside its exact commit prefix" + if [[ "$normalized_path" == "$test_source_prefix/rootfs" ]]; then + rootfs_present=true + fi + done < <(LC_ALL=C tar --quoting-style=literal -tzf "$archive_path") + [[ "$rootfs_present" == "true" ]] || + fail "test-only rootfs source archive does not contain ${test_source_prefix}/rootfs" + + while IFS= read -r member_listing; do + member_type="${member_listing:0:1}" + [[ "$member_type" == "-" || "$member_type" == "d" ]] || + fail "test-only rootfs source archive contains a link or unsupported filesystem object" + done < <(LC_ALL=C tar --numeric-owner --quoting-style=escape -tvzf "$archive_path") +} + +rootfs_contract_sha256() { + local rootfs_dir="$1" + local contract_manifest="$2" + local require_root_owner="${3:-true}" + local absolute_path relative_path expected_mode metadata owner file_sha256 unsupported_path + + [[ -d "$rootfs_dir" && ! -L "$rootfs_dir" ]] || + fail "rootfs contract source must be a real directory" + unsupported_path="$(find "$rootfs_dir" -mindepth 1 ! -type d ! -type f -print -quit)" + if [[ -n "$unsupported_path" ]]; then + fail "rootfs contract contains a symlink or unsupported filesystem object" + fi + + : >"$contract_manifest" + while IFS= read -r -d '' absolute_path; do + relative_path="${absolute_path#"$rootfs_dir"/}" + require_safe_contract_path "$relative_path" + metadata="$(stat -c '%a:%F' -- "$absolute_path")" + [[ "$metadata" == "755:directory" ]] || + fail "rootfs contract directory is not mode 0755: $relative_path" + owner="$(stat -c '%u:%g' -- "$absolute_path")" + [[ "$require_root_owner" != "true" || "$owner" == "0:0" ]] || + fail "rootfs contract directory is not root-owned: $relative_path" + printf 'd\t0:0:755\t%s\n' "$relative_path" >>"$contract_manifest" + done < <(find "$rootfs_dir" -mindepth 1 -type d -print0 | LC_ALL=C sort -z) + + while IFS= read -r -d '' absolute_path; do + relative_path="${absolute_path#"$rootfs_dir"/}" + require_safe_contract_path "$relative_path" + expected_mode="$(canonical_file_mode "$relative_path")" + metadata="$(stat -c '%a:%h:%F' -- "$absolute_path")" + [[ "$metadata" == "${expected_mode}:1:regular file" ]] || + fail "rootfs contract file metadata is not canonical: $relative_path" + owner="$(stat -c '%u:%g' -- "$absolute_path")" + [[ "$require_root_owner" != "true" || "$owner" == "0:0" ]] || + fail "rootfs contract file is not root-owned: $relative_path" + read -r file_sha256 _ < <(sha256sum -- "$absolute_path") + printf 'f\t%s\t0:0:%s\t%s\n' \ + "$file_sha256" "$expected_mode" "$relative_path" >>"$contract_manifest" + done < <(find "$rootfs_dir" -type f -print0 | LC_ALL=C sort -z) + + read -r contract_sha256 _ < <(sha256sum -- "$contract_manifest") + printf '%s\n' "$contract_sha256" +} + +require_safe_overlay_dir() { + local overlay_dir="$1" + + if [[ -n "$overlay_dir" && "$overlay_dir" != "/var/lib/cloud-compose/rootfs-overlay" ]]; then + fail "overlay directory must be /var/lib/cloud-compose/rootfs-overlay" + fi + if [[ -L "$overlay_dir" ]]; then + fail "overlay directory must not be a symlink" + fi +} + +install_archive_tools() { + if command -v curl >/dev/null 2>&1 && command -v tar >/dev/null 2>&1; then + return + fi + + if command -v apt-get >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y ca-certificates curl tar + elif command -v dnf >/dev/null 2>&1; then + dnf install -y ca-certificates curl tar + elif command -v rpm-ostree >/dev/null 2>&1; then + rpm-ostree install --apply-live ca-certificates curl tar + else + fail "no supported package manager found to install curl and tar" + fi +} + +prepare_archive() { + local archive_url_b64="$1" archive_sha256="$2" expected_contract_sha256="$3" + local test_source_prefix="${4:-}" + local archive_url extract_dir rootfs_dir required_command + local contract_manifest contract_sha256 source_commit unsupported_path + + archive_url="$(printf '%s' "$archive_url_b64" | base64 -d)" || \ + fail "rootfs archive URL is not valid base64 data" + [[ "$archive_url" == https://* ]] || fail "rootfs archive URL must use HTTPS" + [[ "$archive_url" != *[[:space:]]* ]] || fail "rootfs archive URL must not contain whitespace" + [[ "$archive_sha256" =~ ^[0-9a-f]{64}$ ]] || \ + fail "rootfs archive checksum must be a lowercase SHA-256 digest" + [[ "$expected_contract_sha256" =~ ^[0-9a-f]{64}$ ]] || \ + fail "rootfs content contract must be a lowercase SHA-256 digest" + if [[ -n "$test_source_prefix" ]]; then + [[ "$test_source_prefix" =~ ^cloud-compose-([0-9a-f]{40})$ ]] || + fail "test-only rootfs source archive prefix must identify one exact cloud-compose commit" + source_commit="${BASH_REMATCH[1]}" + [[ "$archive_url" == "https://github.com/libops/cloud-compose/archive/${source_commit}.tar.gz" ]] || + fail "test-only rootfs source archive URL must select the exact commit named by its prefix" + fi + + install_archive_tools + for required_command in awk chmod curl find sha256sum sort stat tar; do + command -v "$required_command" >/dev/null 2>&1 || \ + fail "$required_command is required to install the verified rootfs archive" + done + + if [[ -L "$stage_root" ]]; then + fail "rootfs archive stage must not be a symlink" + fi + rm -rf -- "$stage_root" + install -d -m 0700 -o root -g root "$stage_root" + extract_dir="$stage_root/extract" + install -d -m 0700 -o root -g root "$extract_dir" + + curl -fsSL --proto '=https' --proto-redir '=https' --tlsv1.2 \ + --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 900 \ + --connect-timeout 10 --max-time 300 \ + -o "$stage_root/rootfs.tar.gz" -- "$archive_url" + printf '%s %s\n' "$archive_sha256" "$stage_root/rootfs.tar.gz" | sha256sum -c - + if [[ -n "$test_source_prefix" ]]; then + validate_rootfs_test_source_archive "$stage_root/rootfs.tar.gz" "$test_source_prefix" + tar --no-same-owner --same-permissions -xzf "$stage_root/rootfs.tar.gz" \ + -C "$extract_dir" "$test_source_prefix/rootfs" + rootfs_dir="$extract_dir/$test_source_prefix/rootfs" + # This test-only GitHub source path preserves Git modes rather than + # canonical rootfs package modes. Normalize only the isolated rootfs + # subtree; the exact byte/content contract remains authoritative. + find "$rootfs_dir" -depth -type d -empty -delete + find "$rootfs_dir" -type d -exec chmod 0755 -- {} + + find "$rootfs_dir" -type f -exec chmod 0644 -- {} + + find "$rootfs_dir" -type f -name '*.sh' -exec chmod 0755 -- {} + + else + validate_rootfs_archive "$stage_root/rootfs.tar.gz" + tar --no-same-owner --same-permissions -xzf "$stage_root/rootfs.tar.gz" -C "$extract_dir" + rootfs_dir="$extract_dir/rootfs" + fi + [[ -n "$rootfs_dir" && -d "$rootfs_dir" ]] || \ + fail "rootfs directory not found in $archive_url" + unsupported_path="$(find "$rootfs_dir" -mindepth 1 ! -type d ! -type f -print -quit)" + if [[ -n "$unsupported_path" ]]; then + fail "verified rootfs archive contains a symlink or unsupported filesystem object" + fi + contract_manifest="$stage_root/rootfs-contract.tsv" + contract_sha256="$(rootfs_contract_sha256 "$rootfs_dir" "$contract_manifest" true)" + [[ "$contract_sha256" == "$expected_contract_sha256" ]] || \ + fail "rootfs archive paths, bytes, or canonical metadata do not match this cloud-compose module source" + mv -- "$rootfs_dir" "$staged_rootfs" + rm -f -- "$stage_root/rootfs.tar.gz" + rm -rf -- "$extract_dir" + + [[ -f "$staged_rootfs/home/cloud-compose/prepare-filesystem.sh" && + -f "$staged_rootfs/home/cloud-compose/persist-filesystems.sh" ]] || \ + fail "verified rootfs archive is missing filesystem preparation scripts" + install -m 0600 -- \ + "$staged_rootfs/home/cloud-compose/prepare-filesystem.sh" \ + "$filesystem_prep" + install -m 0600 -- \ + "$staged_rootfs/home/cloud-compose/persist-filesystems.sh" \ + "$filesystem_persist" +} + +install_staged_archive() { + local overlay_dir="${1:-}" + + [[ -d "$staged_rootfs" && ! -L "$staged_rootfs" ]] || \ + fail "verified rootfs directory is unavailable during installation" + require_safe_overlay_dir "$overlay_dir" + cp -a "$staged_rootfs"/. / + if [[ -n "$overlay_dir" && -d "$overlay_dir" ]]; then + cp -a "$overlay_dir"/. / + rm -rf -- "$overlay_dir" + fi + rm -rf -- "$stage_root" +} + +verify_linux_bootstrap() { + local expected_sha256="$1" + local source="$bootstrap_dir/linux-vm-cloud-init.sh" + local metadata + + [[ "$expected_sha256" =~ ^[0-9a-f]{64}$ ]] || \ + fail "Linux VM cloud-init checksum must be a lowercase SHA-256 digest" + [[ -f "$source" && ! -L "$source" ]] || \ + fail "Linux VM cloud-init program is missing or redirected" + metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$source")" + [[ "$metadata" == "0:0:700:1:regular file" ]] || \ + fail "Linux VM cloud-init program is not an unlinked root-owned mode-0700 file" + printf '%s %s\n' "$expected_sha256" "$source" | sha256sum -c - +} + +install_bootstrap_diagnostics() { + local expected_sha256="$1" + local source="$bootstrap_dir/cloud-compose-diagnostics.sh" + local destination=/etc/cloud-compose/bin/cloud-compose-diagnostics.sh + local source_metadata destination_metadata + + [[ "$expected_sha256" =~ ^[0-9a-f]{64}$ ]] || \ + fail "diagnostics checksum must be a lowercase SHA-256 digest" + [[ -f "$source" && ! -L "$source" ]] || \ + fail "staged diagnostics program is missing or redirected" + source_metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$source")" + [[ "$source_metadata" == "0:0:600:1:regular file" ]] || \ + fail "staged diagnostics program is not an unlinked root-owned mode-0600 file" + printf '%s %s\n' "$expected_sha256" "$source" | sha256sum -c - + + for directory in /etc/cloud-compose /etc/cloud-compose/bin; do + [[ ! -L "$directory" ]] || fail "diagnostics destination is redirected: $directory" + install -d -m 0755 -o root -g root -- "$directory" + done + install -m 0755 -o root -g root -- "$source" "$destination" + destination_metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$destination")" + [[ "$destination_metadata" == "0:0:755:1:regular file" ]] || \ + fail "installed diagnostics program is not an unlinked root-owned mode-0755 file" + printf '%s %s\n' "$expected_sha256" "$destination" | sha256sum -c - +} + +action="${1:-}" +case "$action" in + contract) + [[ "$#" -eq 2 ]] || fail "usage: rootfs-archive.sh contract ROOTFS_DIR" + contract_tmp="$(mktemp)" + trap 'rm -f -- "$contract_tmp"' EXIT + rootfs_contract_sha256 "$2" "$contract_tmp" false + ;; + validate-archive) + [[ "$#" -eq 2 ]] || fail "usage: rootfs-archive.sh validate-archive ARCHIVE" + validate_rootfs_archive "$2" + ;; + prepare) + [[ "$#" -eq 4 ]] || \ + fail "usage: rootfs-archive.sh prepare URL_B64 SHA256 ROOTFS_CONTRACT_SHA256" + prepare_archive "$2" "$3" "$4" + ;; + prepare-linux) + [[ "$#" -eq 5 ]] || \ + fail "usage: rootfs-archive.sh prepare-linux URL_B64 SHA256 ROOTFS_CONTRACT_SHA256 LINUX_BOOTSTRAP_SHA256" + prepare_archive "$2" "$3" "$4" + verify_linux_bootstrap "$5" + ;; + prepare-linux-test-source) + [[ "$#" -eq 6 ]] || \ + fail "usage: rootfs-archive.sh prepare-linux-test-source URL_B64 SHA256 ROOTFS_CONTRACT_SHA256 LINUX_BOOTSTRAP_SHA256 TEST_SOURCE_PREFIX" + prepare_archive "$2" "$3" "$4" "$6" + verify_linux_bootstrap "$5" + ;; + install-staged) + [[ "$#" -le 2 ]] || fail "usage: rootfs-archive.sh install-staged [OVERLAY_DIR]" + install_staged_archive "${2:-}" + ;; + install) + [[ "$#" -le 5 && "$#" -ge 4 ]] || \ + fail "usage: rootfs-archive.sh install URL_B64 SHA256 ROOTFS_CONTRACT_SHA256 [OVERLAY_DIR]" + [[ -f /run/cloud-compose-filesystems-ready ]] || \ + fail "Cloud Compose filesystems were not prepared; refusing rootfs installation" + prepare_archive "$2" "$3" "$4" + install_staged_archive "${5:-}" + ;; + install-diagnostics) + [[ "$#" -eq 2 ]] || \ + fail "usage: rootfs-archive.sh install-diagnostics DIAGNOSTICS_SHA256" + install_bootstrap_diagnostics "$2" + ;; + *) + fail "usage: rootfs-archive.sh contract|validate-archive|prepare|prepare-linux|prepare-linux-test-source|install-staged|install|install-diagnostics ..." + ;; +esac diff --git a/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh b/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh new file mode 100755 index 0000000..156001b --- /dev/null +++ b/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euo pipefail + +validate_only=false +if [[ "${1:-}" == "--validate" ]]; then + validate_only=true + shift +fi + +if [[ "$#" -ne 2 ]]; then + echo "usage: run-lifecycle-program.sh [--validate] LIFECYCLE PROGRAM" >&2 + exit 2 +fi + +lifecycle="$1" +entry="$2" +case "$lifecycle" in + init | up | down | rollout) ;; + *) + echo "Unsupported Cloud Compose lifecycle: $lifecycle" >&2 + exit 2 + ;; +esac + +case "$entry" in + true) + exit 0 + ;; + false) + [[ "$validate_only" == "true" ]] && exit 0 + exit 1 + ;; + "/home/cloud-compose/default-lifecycle.sh $lifecycle") + program=/home/cloud-compose/default-lifecycle.sh + program_args=("$lifecycle") + ;; + *) + program_dir="${CLOUD_COMPOSE_LIFECYCLE_PROGRAM_DIR:-/etc/cloud-compose/lifecycle.d}" + if [[ "$program_dir" != /* || "$program_dir" == "/" || + "$program_dir" == *$'\n'* || "$program_dir" == *$'\r'* || + "$program_dir" =~ (^|/)\.\.?(/|$) ]]; then + echo "Unsafe Cloud Compose lifecycle program directory: $program_dir" >&2 + exit 2 + fi + program_name="${entry#"$program_dir"/}" + if [[ "$entry" != "$program_dir/"* || "$program_name" == */* || + ! "$program_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "Lifecycle entries must name one checked program in ${program_dir}: $entry" >&2 + exit 2 + fi + program="$entry" + program_args=() + ;; +esac + +executor_uid="$(stat -c '%u' -- "${BASH_SOURCE[0]}")" +program_parent="$(dirname -- "$program")" +if [[ -L "$program_parent" || ! -d "$program_parent" || -L "$program" || ! -f "$program" ]]; then + echo "Lifecycle program is missing or redirected: $program" >&2 + exit 1 +fi +parent_metadata="$(stat -c '%u:%a:%F' -- "$program_parent")" +program_metadata="$(stat -c '%u:%a:%h:%F' -- "$program")" +IFS=: read -r parent_uid parent_mode parent_kind <<<"$parent_metadata" +IFS=: read -r program_uid program_mode program_links program_kind <<<"$program_metadata" +if [[ "$parent_uid" != "$executor_uid" || "$parent_kind" != "directory" || + ! "$parent_mode" =~ ^[0-7]{3,4}$ || $((8#$parent_mode & 0022)) -ne 0 ]]; then + echo "Lifecycle program directory is not controlled by the executor owner: $program_parent" >&2 + exit 1 +fi +if [[ "$program_uid" != "$executor_uid" || "$program_links" != "1" || + "$program_kind" != "regular file" || ! "$program_mode" =~ ^[0-7]{3,4}$ || + $((8#$program_mode & 0122)) -ne 0100 ]]; then + echo "Lifecycle program is not an unlinked owner-executable file controlled by the executor owner: $program" >&2 + exit 1 +fi + +if [[ "$validate_only" == "true" ]]; then + exit 0 +fi + +exec "$program" "${program_args[@]}" diff --git a/rootfs/etc/systemd/system/cloud-compose-bootstrap.service b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service index 1149e95..6277dda 100644 --- a/rootfs/etc/systemd/system/cloud-compose-bootstrap.service +++ b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service @@ -3,7 +3,11 @@ Description=Converge Cloud Compose application bootstrap Wants=network-online.target After=network-online.target RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes -StartLimitIntervalSec=0 +# Three full two-hour activation attempts must all remain inside the rate-limit +# window; otherwise a repeatedly hung bootstrap could retry forever as the +# oldest start aged out before the fourth attempt. +StartLimitIntervalSec=8h +StartLimitBurst=3 [Service] Type=oneshot diff --git a/rootfs/etc/systemd/system/cloud-compose.service b/rootfs/etc/systemd/system/cloud-compose.service index 83ead84..25e7e8f 100644 --- a/rootfs/etc/systemd/system/cloud-compose.service +++ b/rootfs/etc/systemd/system/cloud-compose.service @@ -5,7 +5,10 @@ Requires=docker.service cloud-compose-metadata-firewall.service RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes Wants=network-online.target After=network-online.target docker.service cloud-compose-metadata-firewall.service cloud-compose-vault-agent.service -StartLimitIntervalSec=0 +# Keep all three possible 90-minute activation attempts inside one rate-limit +# window so repeated hangs cannot age out and restart forever. +StartLimitIntervalSec=6h +StartLimitBurst=3 [Service] Type=oneshot diff --git a/rootfs/home/cloud-compose/bootstrap-helpers.sh b/rootfs/home/cloud-compose/bootstrap-helpers.sh index 0e4cfa9..a83769a 100644 --- a/rootfs/home/cloud-compose/bootstrap-helpers.sh +++ b/rootfs/home/cloud-compose/bootstrap-helpers.sh @@ -142,6 +142,7 @@ cloud_compose_wait_for_oneshot() { local unit="$1" local timeout_seconds="$2" local poll_seconds="${CLOUD_COMPOSE_SYSTEMD_POLL_SECONDS:-2}" + local heartbeat_seconds="${CLOUD_COMPOSE_SYSTEMD_HEARTBEAT_SECONDS:-300}" local elapsed=0 active_state load_state cloud_compose_validate_systemd_unit "$unit" || return @@ -155,6 +156,11 @@ cloud_compose_wait_for_oneshot() { echo "CLOUD_COMPOSE_SYSTEMD_POLL_SECONDS must be from 1 through 300 seconds" >&2 return 2 fi + if [[ ! "$heartbeat_seconds" =~ ^[1-9][0-9]{0,3}$ ]] || + ((10#$heartbeat_seconds > 3600)); then + echo "CLOUD_COMPOSE_SYSTEMD_HEARTBEAT_SECONDS must be from 1 through 3600 seconds" >&2 + return 2 + fi while ((elapsed < 10#$timeout_seconds)); do load_state="$(systemctl show --property=LoadState --value -- "$unit")" || return 1 @@ -167,8 +173,19 @@ cloud_compose_wait_for_oneshot() { if [[ "$active_state" == "active" ]]; then return 0 fi + if [[ "$active_state" == "failed" ]]; then + echo "Cloud Compose systemd unit reached a terminal failed state: $unit" >&2 + systemctl status --no-pager --full -- "$unit" >&2 || true + return 1 + fi sleep "$poll_seconds" elapsed=$((elapsed + 10#$poll_seconds)) + if ((elapsed % 10#$heartbeat_seconds < 10#$poll_seconds)); then + echo "Still waiting for $unit after ${elapsed}s (active state: $active_state)" >&2 + systemctl show --no-pager \ + --property=ActiveState,SubState,Result,NRestarts,ExecMainCode,ExecMainStatus \ + -- "$unit" >&2 || true + fi done echo "Timed out waiting ${timeout_seconds}s for $unit to become active" >&2 diff --git a/rootfs/home/cloud-compose/compose-apps.sh b/rootfs/home/cloud-compose/compose-apps.sh index 81438ef..5d30dd0 100644 --- a/rootfs/home/cloud-compose/compose-apps.sh +++ b/rootfs/home/cloud-compose/compose-apps.sh @@ -8,6 +8,7 @@ COMPOSE_APPS_STATE_DIR="${COMPOSE_APPS_STATE_DIR:-/home/cloud-compose/state}" CLOUD_COMPOSE_DATA_ROOT="${CLOUD_COMPOSE_DATA_ROOT:-/mnt/disks/data}" compose_apps_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" COMPOSE_SECRET_FILES_PROGRAM="${CLOUD_COMPOSE_COMPOSE_SECRET_FILES_PROGRAM:-$compose_apps_dir/compose-secret-files.awk}" +readonly COMPOSE_LIFECYCLE_EXECUTOR="/etc/cloud-compose/libexec/run-lifecycle-program.sh" shell_env_line() { local name="$1" @@ -1051,6 +1052,22 @@ run_compose_app_lifecycle() { local command command_status local -a commands=() + case "$lifecycle" in + init | up | down | rollout) ;; + *) + echo "Unsupported cloud-compose lifecycle: $lifecycle" >&2 + return 2 + ;; + esac + + # Reject the entire program set before cloning, updating, or running + # anything so a bad later selector cannot leave partial lifecycle state. + compose_app_array_values "$app" "$field" commands || return 1 + for command in "${commands[@]}"; do + [[ -n "$command" ]] || continue + "$COMPOSE_LIFECYCLE_EXECUTOR" --validate "$lifecycle" "$command" || return 1 + done + case "$lifecycle" in init) # Initialization is the explicit baseline-source convergence phase. @@ -1074,14 +1091,9 @@ run_compose_app_lifecycle() { source_compose_app_env "$app" || return 1 validate_compose_git_source || return 1 ;; - *) - echo "Unsupported cloud-compose lifecycle: $lifecycle" >&2 - return 2 - ;; esac echo "Running cloud-compose ${lifecycle} for ${app}" - compose_app_array_values "$app" "$field" commands || return 1 configure_sitectl_verify_argv || return 1 pushd "$DOCKER_COMPOSE_DIR" >/dev/null || return 1 if [[ "$lifecycle" != "down" ]]; then @@ -1091,7 +1103,7 @@ run_compose_app_lifecycle() { if [ -z "$command" ]; then continue fi - bash -c "$command" || { + "$COMPOSE_LIFECYCLE_EXECUTOR" "$lifecycle" "$command" || { command_status=$? popd >/dev/null return "$command_status" diff --git a/rootfs/home/cloud-compose/default-lifecycle.sh b/rootfs/home/cloud-compose/default-lifecycle.sh new file mode 100644 index 0000000..845a0a9 --- /dev/null +++ b/rootfs/home/cloud-compose/default-lifecycle.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +set -euo pipefail + +action="${1:-}" +if [[ "$#" -ne 1 ]]; then + echo "usage: default-lifecycle.sh init|up|down|rollout" >&2 + exit 2 +fi + +case "$action" in + init | up | down | rollout) ;; + *) + echo "unknown default lifecycle action: $action" >&2 + exit 2 + ;; +esac + +context="${SITECTL_CONTEXT_NAME:?SITECTL_CONTEXT_NAME is required}" + +run_sitectl() { + if [[ -n "${SITECTL_EXECUTABLE:-}" ]]; then + "${SITECTL_EXECUTABLE}" "$@" + return + fi + command sitectl "$@" +} + +verify_nonproduction() { + local encoded decoded encoded_args + local verify_args_program="${CLOUD_COMPOSE_SITECTL_VERIFY_ARGS_PROGRAM:-/etc/cloud-compose/jq/sitectl-verify-args.jq}" + local -a verify_args=() + + if [[ "${SITECTL_ENVIRONMENT:?SITECTL_ENVIRONMENT is required}" != "production" ]]; then + if ! encoded_args="$(jq -r -f "$verify_args_program" \ + <<<"${SITECTL_VERIFY_ARGS_JSON:-[]}")"; then + echo "SITECTL_VERIFY_ARGS_JSON must be an array of strings" >&2 + return 1 + fi + while IFS= read -r encoded; do + [[ -n "$encoded" ]] || continue + decoded="$(printf '%s' "${encoded#x}" | base64 -d)" + verify_args+=("$decoded") + done <<<"$encoded_args" + run_sitectl verify --context "$context" "${verify_args[@]}" + fi +} + +case "$action" in + init) + run_sitectl config set-context "$context" \ + --type local \ + --project-dir "${DOCKER_COMPOSE_DIR:?DOCKER_COMPOSE_DIR is required}" \ + --site "${CLOUD_COMPOSE_INSTANCE_NAME:?CLOUD_COMPOSE_INSTANCE_NAME is required}" \ + --plugin "${SITECTL_PLUGIN:?SITECTL_PLUGIN is required}" \ + --environment "${SITECTL_ENVIRONMENT:?SITECTL_ENVIRONMENT is required}" \ + --compose-project-name "${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME is required}" \ + --docker-socket /var/run/docker.sock \ + --env-file .env \ + --yolo \ + --default + ;; + up) + run_sitectl compose --context "$context" up -d --remove-orphans + run_sitectl healthcheck --context "$context" --persist + verify_nonproduction + ;; + down) + run_sitectl compose --context "$context" down + ;; + rollout) + commit_sha="${GIT_COMMIT_SHA:-}" + if [[ -n "$commit_sha" && ! "$commit_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "GIT_COMMIT_SHA must be an exact lowercase 40-character commit SHA" >&2 + exit 2 + fi + target_ref="${commit_sha:-${GIT_REF:-${GIT_BRANCH:-}}}" + if [[ -n "$target_ref" ]]; then + run_sitectl deploy --context "$context" --ref "$target_ref" + else + run_sitectl deploy --context "$context" --skip-git + fi + run_sitectl healthcheck --context "$context" --persist + verify_nonproduction + ;; +esac diff --git a/rootfs/home/cloud-compose/lifecycle-entrypoint.sh b/rootfs/home/cloud-compose/lifecycle-entrypoint.sh new file mode 100644 index 0000000..7516a14 --- /dev/null +++ b/rootfs/home/cloud-compose/lifecycle-entrypoint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +lifecycle="${0##*/}" +case "$lifecycle" in + init | up | down | rollout) ;; + *) + echo "lifecycle entrypoint must be installed as init, up, down, or rollout" >&2 + exit 2 + ;; +esac + +# shellcheck disable=SC1091 +source /home/cloud-compose/profile.sh +exec bash /home/cloud-compose/compose-dispatch.sh "$lifecycle" diff --git a/rootfs/home/cloud-compose/run.sh b/rootfs/home/cloud-compose/run.sh index d4c6358..dcd884a 100644 --- a/rootfs/home/cloud-compose/run.sh +++ b/rootfs/home/cloud-compose/run.sh @@ -17,10 +17,8 @@ run_as_cloud_compose() ( runuser -u cloud-compose -- env HOME=/home/cloud-compose PATH="$PATH" "$@" elif command -v sudo >/dev/null 2>&1; then sudo -u cloud-compose env HOME=/home/cloud-compose PATH="$PATH" "$@" - elif command -v su >/dev/null 2>&1; then - su -s /bin/bash -c "HOME=/home/cloud-compose PATH=$(printf '%q' "$PATH") $(printf '%q ' "$@")" cloud-compose else - echo "No supported user-switching command found for cloud-compose app init" >&2 + echo "Neither runuser nor sudo is available for cloud-compose app init" >&2 return 1 fi ) diff --git a/salt/cloud-compose/README.md b/salt/cloud-compose/README.md index 4829b17..2008296 100644 --- a/salt/cloud-compose/README.md +++ b/salt/cloud-compose/README.md @@ -65,6 +65,12 @@ empty list disables that phase and is preserved instead of restoring a default. Runtime feature switches must be YAML booleans, not quoted strings; ambiguous values are rejected before host mutation. +Lifecycle list entries are program selectors, not shell source. Use the +built-in `/home/cloud-compose/default-lifecycle.sh ACTION`, `true`, `false`, or +one argument-free root-owned executable immediately below +`/etc/cloud-compose/lifecycle.d`. Put multi-step logic and quoting inside that +checked program file. + The provider-neutral disaster-recovery interface is shared with Terraform: ```yaml diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls index 122b563..de3f914 100644 --- a/salt/cloud-compose/init.sls +++ b/salt/cloud-compose/init.sls @@ -117,6 +117,11 @@ {% set reload_systemd = cc.get('reload_systemd', True) %} {% set run_bootstrap = cc.get('run_bootstrap', True) %} {% set force_bootstrap = cc.get('force_bootstrap', False) %} +{% set bootstrap_wait_seconds = cc.get('bootstrap_wait_seconds', 10800) %} +{% if bootstrap_wait_seconds is boolean or bootstrap_wait_seconds is not number or bootstrap_wait_seconds != bootstrap_wait_seconds | int or bootstrap_wait_seconds < 1 or bootstrap_wait_seconds > 43200 %} +{% set ignored = invalid_runtime_inputs.append('bootstrap_wait_seconds must be a whole number from 1 through 43200') %} +{% set bootstrap_wait_seconds = 10800 %} +{% endif %} {% set raw_template_name = cc.get('template', '') %} {% if raw_template_name is string %} {% set template_name = raw_template_name | lower | trim %} @@ -209,21 +214,16 @@ 'upload_timeout': '' } %} {% set default_init = [ - 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --yolo --default' + '/home/cloud-compose/default-lifecycle.sh init' ] %} {% set default_up = [ - 'sitectl compose --context "${SITECTL_CONTEXT_NAME}" up -d --remove-orphans', - 'sitectl healthcheck --context "${SITECTL_CONTEXT_NAME}" --persist', - 'if [ "${SITECTL_ENVIRONMENT}" != "production" ]; then sitectl verify --context "${SITECTL_CONTEXT_NAME}" ${SITECTL_VERIFY_ARGS:-}; fi' + '/home/cloud-compose/default-lifecycle.sh up' ] %} {% set default_down = [ - 'sitectl compose --context "${SITECTL_CONTEXT_NAME}" down' + '/home/cloud-compose/default-lifecycle.sh down' ] %} {% set default_rollout = [ - 'TARGET_REF="${GIT_REF:-${GIT_BRANCH:-}}"', - 'if [ -n "$TARGET_REF" ]; then sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --ref "$TARGET_REF"; else sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --skip-git; fi', - 'sitectl healthcheck --context "${SITECTL_CONTEXT_NAME}" --persist', - 'if [ "${SITECTL_ENVIRONMENT}" != "production" ]; then sitectl verify --context "${SITECTL_CONTEXT_NAME}" ${SITECTL_VERIFY_ARGS:-}; fi' + '/home/cloud-compose/default-lifecycle.sh rollout' ] %} {% set lifecycle_defaults = { 'init': default_init, @@ -704,16 +704,10 @@ cloud-compose-rootfs-jq-modes: cloud-compose-lifecycle-{{ lifecycle }}: file.managed: - name: {{ (home ~ '/' ~ lifecycle) | json }} + - source: salt://rootfs/home/cloud-compose/lifecycle-entrypoint.sh - user: root - group: {{ group | json }} - mode: '0750' - - contents: | - #!/usr/bin/env bash - - set -eou pipefail - - source /home/cloud-compose/profile.sh - exec bash /home/cloud-compose/compose-dispatch.sh "{{ lifecycle }}" - require: - file: cloud-compose-rootfs {% endfor %} @@ -832,6 +826,8 @@ cloud-compose-clear-bootstrap-marker: cloud-compose-bootstrap: cmd.run: - name: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh + - env: + CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS: {{ (bootstrap_wait_seconds | string) | json }} - unless: bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh - require: {% if install_packages %} diff --git a/template_versions.tftest.hcl b/template_versions.tftest.hcl index 4b992a8..35a899c 100644 --- a/template_versions.tftest.hcl +++ b/template_versions.tftest.hcl @@ -1,4 +1,12 @@ mock_provider "cloudinit" {} +mock_provider "http" { + mock_data "http" { + defaults = { + response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + status_code = 200 + } + } +} mock_provider "google" { mock_data "google_project" { defaults = { diff --git a/templates/cloud-init.yml b/templates/cloud-init.yml index 94d2d13..202361f 100644 --- a/templates/cloud-init.yml +++ b/templates/cloud-init.yml @@ -28,56 +28,37 @@ users: %{ endfor ~} %{ endfor ~} -bootcmd: -# mount the main Data disk -- | - set -eu - rm -f /run/cloud-compose-filesystems-ready - filesystem_prep=/run/cloud-compose-prepare-filesystem - filesystem_persist=/run/cloud-compose-persist-filesystems - printf '%s' '${FILESYSTEM_PREP_SCRIPT_B64}' | base64 -d >"$filesystem_prep" - printf '%s' '${FILESYSTEM_PERSIST_SCRIPT_B64}' | base64 -d >"$filesystem_persist" - chmod 0600 "$filesystem_prep" "$filesystem_persist" - bash "$filesystem_prep" /dev/disk/by-id/google-data /mnt/disks/data \ - --publish-fresh-marker ${jsonencode(FRESH_FILESYSTEM_IDENTITY)} - -# mount the specific Volumes disk -- bash /run/cloud-compose-prepare-filesystem /dev/disk/by-id/google-docker-volumes /mnt/disks/volumes - -- mkdir -p /mnt/disks/data/docker/volumes -- | - set -eu - if ! mountpoint -q /mnt/disks/data/docker/volumes; then - mount --bind /mnt/disks/volumes /mnt/disks/data/docker/volumes - fi - for required_mount in /mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes; do - if ! mountpoint -q -- "$required_mount"; then - echo "Required cloud-compose mount is unavailable: $required_mount" >&2 - exit 1 - fi - done -%{ if USE_OVERLAY ~} - mkdir -p /mnt/disks/prod-readonly - if ! mountpoint -q /mnt/disks/prod-readonly; then - mount -o ro "$(readlink -f /dev/disk/by-id/google-prod-volumes)" /mnt/disks/prod-readonly - fi - bash /run/cloud-compose-persist-filesystems \ - /dev/disk/by-id/google-data \ - /dev/disk/by-id/google-docker-volumes \ - /dev/disk/by-id/google-prod-volumes -%{ else ~} - bash /run/cloud-compose-persist-filesystems \ - /dev/disk/by-id/google-data \ - /dev/disk/by-id/google-docker-volumes -%{ endif ~} - install -m 0600 /dev/null /run/cloud-compose-filesystems-ready - write_files: +- path: /var/lib/cloud-compose/bootstrap/rootfs-archive.sh + owner: root:root + permissions: "0700" + encoding: b64 + content: ${ROOTFS_ARCHIVE_SCRIPT_B64} +- path: /var/lib/cloud-compose/bootstrap/gcp-cloud-init-finalize.sh + owner: root:root + permissions: "0700" + encoding: b64 + content: ${GCP_CLOUD_INIT_FINALIZE_SCRIPT_B64} +- path: /var/lib/cloud-compose/bootstrap/gcp-cloud-init-post-bootstrap.sh + owner: root:root + permissions: "0700" + encoding: b64 + content: ${GCP_CLOUD_INIT_POST_SCRIPT_B64} - path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh owner: root:root permissions: "0600" encoding: b64 content: ${DIAGNOSTICS_SCRIPT_B64} +- path: /var/lib/cloud-compose/bootstrap/init-commands.sh + owner: root:root + permissions: "0600" + encoding: b64 + content: ${jsonencode(INIT_COMMANDS_B64)} +- path: /var/lib/cloud-compose/bootstrap/runcmd.sh + owner: root:root + permissions: "0600" + encoding: b64 + content: ${jsonencode(RUNCMD_B64)} ${WRITE_FILES_CONTENT} ${DOCKER_COMPOSE_SCRIPTS} ${COMPOSE_PROJECTS_FILE} @@ -87,33 +68,10 @@ ${VAULT_AGENT_FILES} ${MANAGED_RUNTIME_ARTIFACTS_FILE} runcmd: -${ROOTFS_ARCHIVE_COMMAND} -- | - set -eu - install -d -m 0755 -o root -g root /etc/cloud-compose/bin - install -m 0755 -o root -g root \ - /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh \ - /etc/cloud-compose/bin/cloud-compose-diagnostics.sh - test -f /run/cloud-compose-filesystems-ready || { - echo "Cloud Compose filesystems were not prepared; refusing application initialization" >&2 - exit 1 - } -%{ for CMD in ADDITIONAL_INITCMD ~} - ${indent(2, CMD)} -%{ endfor ~} - chown root:cloud-compose /mnt/disks/data - chmod 1775 /mnt/disks/data - chown cloud-compose:cloud-compose /mnt/disks/volumes - chmod 0775 /mnt/disks/volumes - install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops - rm -f /var/lib/cloud-compose/bootstrap-complete - /etc/cloud-compose/libexec/harden-bootstrap-paths.sh - bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh -%{ if length(ADDITIONAL_RUNCMD) > 0 ~} -- | - set -eu - bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh -%{ for CMD in ADDITIONAL_RUNCMD ~} - ${indent(2, CMD)} -%{ endfor ~} +%{ if ROOTFS_ARCHIVE_ENABLED ~} +- [bash, /var/lib/cloud-compose/bootstrap/rootfs-archive.sh, install, ${jsonencode(ROOTFS_ARCHIVE_URL_B64)}, ${jsonencode(ROOTFS_ARCHIVE_SHA256)}, ${jsonencode(ROOTFS_CONTRACT_SHA256)}, ${jsonencode(ROOTFS_OVERLAY_DIR)}] +%{ endif ~} +- [bash, /var/lib/cloud-compose/bootstrap/gcp-cloud-init-finalize.sh, /var/lib/cloud-compose/bootstrap/init-commands.sh, ${jsonencode(DIAGNOSTICS_SCRIPT_SHA256)}] +%{ if POST_BOOTSTRAP_REQUIRED ~} +- [bash, /var/lib/cloud-compose/bootstrap/gcp-cloud-init-post-bootstrap.sh, ${jsonencode(ROLLOUT_ENABLED)}, /var/lib/cloud-compose/bootstrap/runcmd.sh] %{ endif ~} diff --git a/templates/gcp-cloud-init.mime.tftpl b/templates/gcp-cloud-init.mime.tftpl new file mode 100644 index 0000000..8d3afb5 --- /dev/null +++ b/templates/gcp-cloud-init.mime.tftpl @@ -0,0 +1,18 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="===============cloud-compose==" + +--===============cloud-compose== +Content-Type: text/cloud-boothook; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="gcp-filesystem-boothook.sh" + +${FILESYSTEM_BOOTHOOK} +--===============cloud-compose== +Content-Type: text/cloud-config; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="cloud-config.yml" + +${CLOUD_CONFIG} +--===============cloud-compose==-- diff --git a/templates/gcp-filesystem-boothook.sh.tftpl b/templates/gcp-filesystem-boothook.sh.tftpl new file mode 100644 index 0000000..08d5235 --- /dev/null +++ b/templates/gcp-filesystem-boothook.sh.tftpl @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -euo pipefail + +umask 077 +readonly filesystem_boot=/run/cloud-compose-gcp-filesystem-boot +readonly filesystem_prep=/run/cloud-compose-prepare-filesystem +readonly filesystem_persist=/run/cloud-compose-persist-filesystems + +printf '%s' '${GCP_FILESYSTEM_BOOT_SCRIPT_B64}' | base64 -d >"$filesystem_boot" +printf '%s' '${FILESYSTEM_PREP_SCRIPT_B64}' | base64 -d >"$filesystem_prep" +printf '%s' '${FILESYSTEM_PERSIST_SCRIPT_B64}' | base64 -d >"$filesystem_persist" +chmod 0600 "$filesystem_boot" "$filesystem_prep" "$filesystem_persist" + +bash "$filesystem_boot" \ + ${jsonencode(FRESH_FILESYSTEM_IDENTITY)} \ + ${jsonencode(USE_OVERLAY ? "true" : "false")} \ + "$filesystem_prep" \ + "$filesystem_persist" diff --git a/tests/smoke/do/main.tf b/tests/smoke/do/main.tf index 852e030..00a659f 100644 --- a/tests/smoke/do/main.tf +++ b/tests/smoke/do/main.tf @@ -14,23 +14,25 @@ provider "digitalocean" {} module "context" { source = "../modules/context" - cloud_provider = "digitalocean" - template = var.template - ssh_public_key = var.ssh_public_key - operator_ssh_public_keys = var.operator_ssh_public_keys - smoke_run_id = var.smoke_run_id - docker_compose_branch = var.docker_compose_branch - ingress_port = var.ingress_port - rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/${var.cloud_compose_source_ref}.tar.gz" - rootfs_archive_sha256 = var.cloud_compose_source_sha256 - tags = var.tags + cloud_provider = "digitalocean" + template = var.template + ssh_public_key = var.ssh_public_key + operator_ssh_public_keys = var.operator_ssh_public_keys + smoke_run_id = var.smoke_run_id + docker_compose_branch = var.docker_compose_branch + ingress_port = var.ingress_port + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/${var.cloud_compose_source_ref}.tar.gz" + rootfs_archive_sha256 = var.cloud_compose_source_sha256 + rootfs_test_source_archive_prefix = "cloud-compose-${var.cloud_compose_source_ref}" + tags = var.tags } module "app" { - source = "../../../providers/do" + # Hosted smoke alone needs the exact-commit source-archive fixture. Keep that + # test-only escape hatch below the public provider entrypoint. + source = "../../../modules/digitalocean" - name = module.context.name - template = module.context.template + name = module.context.name digitalocean = { region = var.digitalocean_region tags = module.context.tags @@ -58,7 +60,7 @@ module "smoke" { cloud_provider = "digitalocean" template = module.context.template name = module.context.name - host = module.app.external_ip + host = module.app.instance.ipv4 primary_compose_project = module.app.primary_compose_project } diff --git a/tests/smoke/do/variables.tf b/tests/smoke/do/variables.tf index 80a89d4..600811a 100644 --- a/tests/smoke/do/variables.tf +++ b/tests/smoke/do/variables.tf @@ -83,8 +83,12 @@ variable "tags" { variable "cloud_compose_source_ref" { type = string - default = "main" - description = "cloud-compose Git ref whose rootfs is fetched to keep DigitalOcean user_data below 64 KiB." + description = "Exact lowercase cloud-compose commit whose source archive is fetched to keep DigitalOcean user_data below 64 KiB." + + validation { + condition = can(regex("^[0-9a-f]{40}$", var.cloud_compose_source_ref)) + error_message = "cloud_compose_source_ref must be an exact lowercase 40-character commit SHA." + } } variable "cloud_compose_source_sha256" { diff --git a/tests/smoke/gcp-upgrade/main.tf b/tests/smoke/gcp-upgrade/main.tf index 980bb8c..6ab7c42 100644 --- a/tests/smoke/gcp-upgrade/main.tf +++ b/tests/smoke/gcp-upgrade/main.tf @@ -79,9 +79,7 @@ module "app" { # timer can suspend the disposable host during a long bootstrap, so # disabling it in the later runcmd phase is too late. initcmd = [ - "systemctl disable --now internal-services.timer internal-services.service 2>/dev/null || true", - "systemctl disable --now cloud-compose-internal-services.timer cloud-compose-internal-services.service 2>/dev/null || true", - "bash -ceu 'project=${local.wordpress_project_dir}; git_project() { git -c safe.directory=\"$project\" -C \"$project\" \"$@\"; }; install -d -m 0755 \"$project\"; if [[ ! -d \"$project/.git\" ]]; then git_project init; git_project remote add origin https://github.com/libops/wp.git; fi; git_project remote set-url origin https://github.com/libops/wp.git; git_project fetch --force --no-tags --depth=1 origin ${var.wordpress_compose_ref}; git_project checkout --detach ${var.wordpress_compose_ref}; test \"$(git_project rev-parse HEAD)\" = ${var.wordpress_compose_ref}; chown -R cloud-compose:cloud-compose \"$project\"'", + "bash /home/cloud-compose/gcp-upgrade-prepare-repository.sh", ] } power_management = { @@ -91,6 +89,7 @@ module "app" { } } runtime = { + rootfs = "${path.module}/rootfs" users = { cloud-compose = [var.ssh_public_key] } @@ -112,8 +111,7 @@ module "app" { } } up = [ - "sitectl compose --context \"$${SITECTL_CONTEXT_NAME}\" up -d --remove-orphans", - "sitectl healthcheck --context \"$${SITECTL_CONTEXT_NAME}\" --persist", + "/etc/cloud-compose/lifecycle.d/gcp-upgrade-up.sh", ] } sitectl = { diff --git a/tests/smoke/gcp-upgrade/rootfs/etc/cloud-compose/lifecycle.d/gcp-upgrade-up.sh b/tests/smoke/gcp-upgrade/rootfs/etc/cloud-compose/lifecycle.d/gcp-upgrade-up.sh new file mode 100755 index 0000000..8cc1988 --- /dev/null +++ b/tests/smoke/gcp-upgrade/rootfs/etc/cloud-compose/lifecycle.d/gcp-upgrade-up.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +context="${SITECTL_CONTEXT_NAME:?SITECTL_CONTEXT_NAME is required}" +sitectl compose --context "$context" up -d --remove-orphans +sitectl healthcheck --context "$context" --persist diff --git a/tests/smoke/gcp-upgrade/rootfs/home/cloud-compose/gcp-upgrade-prepare-repository.sh b/tests/smoke/gcp-upgrade/rootfs/home/cloud-compose/gcp-upgrade-prepare-repository.sh new file mode 100755 index 0000000..11cc2d3 --- /dev/null +++ b/tests/smoke/gcp-upgrade/rootfs/home/cloud-compose/gcp-upgrade-prepare-repository.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source /home/cloud-compose/profile.sh + +project="${DOCKER_COMPOSE_DIR:?DOCKER_COMPOSE_DIR is required}" +repository="${DOCKER_COMPOSE_REPO:?DOCKER_COMPOSE_REPO is required}" +revision="${DOCKER_COMPOSE_BRANCH:?DOCKER_COMPOSE_BRANCH is required}" + +[[ "$repository" == "https://github.com/libops/wp.git" ]] || { + echo "Unexpected GCP upgrade fixture repository: $repository" >&2 + exit 1 +} +[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || { + echo "GCP upgrade fixture revision must be an exact commit" >&2 + exit 1 +} +[[ "$project" == "/mnt/disks/data/libops/wp.git/${revision}" ]] || { + echo "Unexpected GCP upgrade fixture project path: $project" >&2 + exit 1 +} + +systemctl disable --now internal-services.timer internal-services.service 2>/dev/null || true +systemctl disable --now cloud-compose-internal-services.timer cloud-compose-internal-services.service 2>/dev/null || true + +git_project() { + git -c safe.directory="$project" -C "$project" "$@" +} + +install -d -m 0755 "$project" +if [[ ! -d "$project/.git" ]]; then + git_project init + git_project remote add origin "$repository" +fi +git_project remote set-url origin "$repository" +git_project fetch --force --no-tags --depth=1 origin "$revision" +git_project checkout --detach "$revision" +[[ "$(git_project rev-parse HEAD)" == "$revision" ]] || { + echo "GCP upgrade fixture checkout did not reach $revision" >&2 + exit 1 +} +chown -R cloud-compose:cloud-compose "$project" diff --git a/tests/smoke/linode/main.tf b/tests/smoke/linode/main.tf index 7fa37ca..d985c33 100644 --- a/tests/smoke/linode/main.tf +++ b/tests/smoke/linode/main.tf @@ -14,23 +14,25 @@ provider "linode" {} module "context" { source = "../modules/context" - cloud_provider = "linode" - template = var.template - ssh_public_key = var.ssh_public_key - operator_ssh_public_keys = var.operator_ssh_public_keys - smoke_run_id = var.smoke_run_id - docker_compose_branch = var.docker_compose_branch - ingress_port = var.ingress_port - rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/${var.cloud_compose_source_ref}.tar.gz" - rootfs_archive_sha256 = var.cloud_compose_source_sha256 - tags = var.tags + cloud_provider = "linode" + template = var.template + ssh_public_key = var.ssh_public_key + operator_ssh_public_keys = var.operator_ssh_public_keys + smoke_run_id = var.smoke_run_id + docker_compose_branch = var.docker_compose_branch + ingress_port = var.ingress_port + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/${var.cloud_compose_source_ref}.tar.gz" + rootfs_archive_sha256 = var.cloud_compose_source_sha256 + rootfs_test_source_archive_prefix = "cloud-compose-${var.cloud_compose_source_ref}" + tags = var.tags } module "app" { - source = "../../../providers/linode" + # Hosted smoke alone needs the exact-commit source-archive fixture. Keep that + # test-only escape hatch below the public provider entrypoint. + source = "../../../modules/linode" - name = module.context.name - template = module.context.template + name = module.context.name linode = { region = var.linode_region tags = module.context.tags @@ -56,7 +58,7 @@ module "smoke" { cloud_provider = "linode" template = module.context.template name = module.context.name - host = module.app.external_ip + host = module.app.instance.public_ipv4 primary_compose_project = module.app.primary_compose_project } diff --git a/tests/smoke/linode/variables.tf b/tests/smoke/linode/variables.tf index aac5b0b..17bb339 100644 --- a/tests/smoke/linode/variables.tf +++ b/tests/smoke/linode/variables.tf @@ -83,8 +83,12 @@ variable "tags" { variable "cloud_compose_source_ref" { type = string - default = "main" - description = "cloud-compose Git ref whose rootfs is fetched by providers with metadata size limits." + description = "Exact lowercase cloud-compose commit whose source archive is fetched by providers with metadata size limits." + + validation { + condition = can(regex("^[0-9a-f]{40}$", var.cloud_compose_source_ref)) + error_message = "cloud_compose_source_ref must be an exact lowercase 40-character commit SHA." + } } variable "cloud_compose_source_sha256" { diff --git a/tests/smoke/modules/context/main.tf b/tests/smoke/modules/context/main.tf index b0646ee..359ba3e 100644 --- a/tests/smoke/modules/context/main.tf +++ b/tests/smoke/modules/context/main.tf @@ -29,6 +29,11 @@ locals { cloud_provider = lower(trimspace(var.cloud_provider)) template = lower(trimspace(var.template)) + app_registry = jsondecode(file("${path.module}/../../../../templates/apps.json")) + app_templates = local.app_registry.templates + empty_template = local.app_registry.default + app_template = try(local.app_templates[local.template], local.empty_template) + provider_prefixes = { digitalocean = "do" gcp = "g" @@ -55,18 +60,22 @@ locals { tags = distinct(concat(var.tags, ["cloud-compose-smoke", local.target], local.run_tag != "" ? [local.run_tag] : [])) ssh_keys = distinct(concat([var.ssh_public_key], var.operator_ssh_public_keys)) runtime_base = { - rootfs_archive_url = var.rootfs_archive_url - rootfs_archive_sha256 = var.rootfs_archive_sha256 + rootfs_archive_url = var.rootfs_archive_url + rootfs_archive_sha256 = var.rootfs_archive_sha256 + rootfs_test_source_archive_prefix = var.rootfs_test_source_archive_prefix compose = { + repo = local.app_template.repo branch = var.docker_compose_branch ingress_port = var.ingress_port up = [ - "sitectl compose --context \"$${SITECTL_CONTEXT_NAME}\" up -d --remove-orphans", - "sitectl healthcheck --context \"$${SITECTL_CONTEXT_NAME}\" --persist" + "/home/cloud-compose/default-lifecycle.sh up" ] } sitectl = { - environment = "smoke" + packages = local.app_template.packages + package_versions = local.app_template.package_versions + plugin = local.app_template.plugin + environment = "smoke" } managed_runtime = { enabled = true @@ -76,6 +85,7 @@ locals { vault = { auth_method = "consumer-managed" } + extra_env = local.app_template.extra_env } gcp_runtime = merge(local.runtime_base, { diff --git a/tests/smoke/modules/context/outputs.tf b/tests/smoke/modules/context/outputs.tf index 6539a63..dbec719 100644 --- a/tests/smoke/modules/context/outputs.tf +++ b/tests/smoke/modules/context/outputs.tf @@ -5,7 +5,7 @@ output "name" { output "runtime" { value = local.runtime_base - description = "Provider-neutral runtime overrides for smoke tests." + description = "Provider-neutral runtime with template defaults and private fixture inputs resolved for hosted smoke tests." } output "gcp_runtime" { diff --git a/tests/smoke/modules/context/variables.tf b/tests/smoke/modules/context/variables.tf index 09d1937..bd01af5 100644 --- a/tests/smoke/modules/context/variables.tf +++ b/tests/smoke/modules/context/variables.tf @@ -75,6 +75,20 @@ variable "rootfs_archive_sha256" { description = "SHA-256 for rootfs_archive_url." } +variable "rootfs_test_source_archive_prefix" { + type = string + default = "" + description = "Explicit source-archive prefix used only by hosted smoke tests for an unreleased exact commit." + + validation { + condition = ( + trimspace(var.rootfs_test_source_archive_prefix) == "" || + can(regex("^cloud-compose-[0-9a-f]{40}$", trimspace(var.rootfs_test_source_archive_prefix))) + ) + error_message = "rootfs_test_source_archive_prefix must be empty or identify one exact lowercase 40-character commit SHA." + } +} + variable "tags" { type = list(string) default = ["cloud-compose"] From 91daa245aab10c6e4de1afc9434a9629e0f504e7 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 05:44:50 +0000 Subject: [PATCH 26/67] [patch] Surface configuration smoke failures --- ci/config-management-smoke-inner.sh | 21 ++++++++++++------- .../gcp-upgrade-smoke-contract-harness.sh | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ci/config-management-smoke-inner.sh b/ci/config-management-smoke-inner.sh index 88ead96..65c6900 100755 --- a/ci/config-management-smoke-inner.sh +++ b/ci/config-management-smoke-inner.sh @@ -165,13 +165,20 @@ run_invalid_ansible_case() { echo "Ansible accepted invalid settings from ${invalid_case}.yml" >&2 exit 1 fi - grep -Fq -- "$expected" "$log" - [[ "$(stat -c '%u:%g:%a' /)" == "$root_before" ]] - [[ "$(stat -c '%u:%g:%a' /etc)" == "$etc_before" ]] - [[ "$(getent passwd cloud-compose || true)" == "$passwd_before" ]] - [[ "$(getent group cloud-compose || true)" == "$group_before" ]] - [[ ! -e /home/cloud-compose ]] - [[ ! -e /mnt/disks/volumes ]] + if ! grep -Fq -- "$expected" "$log"; then + echo "Ansible invalid case ${invalid_case} did not report: ${expected}" >&2 + cat "$log" >&2 + exit 1 + fi + if [[ "$(stat -c '%u:%g:%a' /)" != "$root_before" || + "$(stat -c '%u:%g:%a' /etc)" != "$etc_before" || + "$(getent passwd cloud-compose || true)" != "$passwd_before" || + "$(getent group cloud-compose || true)" != "$group_before" || + -e /home/cloud-compose || -e /mnt/disks/volumes ]]; then + echo "Ansible invalid case ${invalid_case} mutated host state before rejection" >&2 + cat "$log" >&2 + exit 1 + fi if [[ "$invalid_case" == "invalid-project-dir-symlink" ]]; then grep -Fxq 'must-not-change' /tmp/cloud-compose-symlink-target/sentinel [[ -L /mnt/disks/data/escape ]] diff --git a/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh b/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh index efd3c59..9664cba 100755 --- a/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh +++ b/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh @@ -36,7 +36,7 @@ case "$action" in ;; cleanup-failure) [[ "$#" -eq 3 ]] - repo_root="$3" + [[ "$repo_root" == "$3" ]] : "${CLEANUP_LOG:?CLEANUP_LOG is required}" require_cmd() { :; } require_env() { :; } From 3dee4a6ed26345638529ae3f88818dea12c5a6f0 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 05:48:08 +0000 Subject: [PATCH 27/67] [patch] Repair hosted contract expectations --- ansible/roles/cloud_compose/tasks/main.yml | 2 +- ci/fixtures/gcp-upgrade-smoke-contract-harness.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/cloud_compose/tasks/main.yml b/ansible/roles/cloud_compose/tasks/main.yml index d315f18..fd5e56d 100644 --- a/ansible/roles/cloud_compose/tasks/main.yml +++ b/ansible/roles/cloud_compose/tasks/main.yml @@ -39,7 +39,7 @@ - (cloud_compose_bootstrap_wait_seconds | int) >= 1 - (cloud_compose_bootstrap_wait_seconds | int) <= 43200 - (cloud_compose_bootstrap_timeout | int) > (cloud_compose_bootstrap_wait_seconds | int) - fail_msg: "The adapter requires an onprem provider, a runtime/extra_env map, a name matching ^[a-z][a-z0-9-]*$, dedicated-host acknowledgement, and a bootstrap async timeout longer than the 1-43200 second service wait." + fail_msg: "The adapter requires an onprem provider, a runtime/extra_env map, a name matching ^[a-z][a-z0-9-]*$, cloud_compose_dedicated_host_acknowledged=true, and a bootstrap async timeout longer than the 1-43200 second service wait." - name: Require safe cloud-compose environment entries ansible.builtin.assert: diff --git a/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh b/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh index 9664cba..07212ac 100755 --- a/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh +++ b/ci/fixtures/gcp-upgrade-smoke-contract-harness.sh @@ -36,7 +36,7 @@ case "$action" in ;; cleanup-failure) [[ "$#" -eq 3 ]] - [[ "$repo_root" == "$3" ]] + [[ -d "$3/.git" ]] : "${CLEANUP_LOG:?CLEANUP_LOG is required}" require_cmd() { :; } require_env() { :; } From 5badc2ffe328f0a296a1b352ded8f9f610525650 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 05:54:00 +0000 Subject: [PATCH 28/67] [patch] Cover both rootfs archive trust paths --- ci/host-runtime-security.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ci/host-runtime-security.sh b/ci/host-runtime-security.sh index 998c9b9..d42701a 100644 --- a/ci/host-runtime-security.sh +++ b/ci/host-runtime-security.sh @@ -245,13 +245,18 @@ fi archive_source="$repo_root/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh" verify_line="$(grep -n 'sha256sum -c -' "$archive_source" | head -n 1 | cut -d: -f1)" +source_members_line="$(grep -n 'validate_rootfs_test_source_archive "\$stage_root/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" +source_extract_line="$(grep -n 'tar --no-same-owner --same-permissions -xzf "\$stage_root/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" members_line="$(grep -n 'validate_rootfs_archive "\$stage_root/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" -extract_line="$(grep -n 'tar --no-same-owner --same-permissions -xzf "\$stage_root/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)" +extract_line="$(grep -n 'tar --no-same-owner --same-permissions -xzf "\$stage_root/rootfs.tar.gz"' "$archive_source" | tail -n 1 | cut -d: -f1)" contract_line="$(grep -n 'rootfs archive paths, bytes, or canonical metadata do not match this cloud-compose module source' "$archive_source" | head -n 1 | cut -d: -f1)" copy_line="$(grep -n 'cp -a "\$staged_rootfs"/. /' "$archive_source" | head -n 1 | cut -d: -f1)" -[[ -n "$verify_line" && -n "$members_line" && -n "$extract_line" && -n "$contract_line" && -n "$copy_line" && - "$verify_line" -lt "$members_line" && "$members_line" -lt "$extract_line" && - "$extract_line" -lt "$contract_line" && "$contract_line" -lt "$copy_line" ]] || \ +[[ -n "$verify_line" && -n "$source_members_line" && -n "$source_extract_line" && + -n "$members_line" && -n "$extract_line" && -n "$contract_line" && -n "$copy_line" && + "$verify_line" -lt "$source_members_line" && "$source_members_line" -lt "$source_extract_line" && + "$source_extract_line" -lt "$contract_line" && "$verify_line" -lt "$members_line" && + "$members_line" -lt "$extract_line" && "$extract_line" -lt "$contract_line" && + "$contract_line" -lt "$copy_line" ]] || \ fail "$archive_source does not verify archive bytes and canonical rootfs metadata before installation" assert_contains "$archive_source" "stat -c '%a:%h:%F'" assert_contains "$archive_source" '[[ "$metadata" == "${expected_mode}:1:regular file" ]]' From f956455b4f5f17c0a9a3200812cdca90dcb03ad9 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 05:59:44 +0000 Subject: [PATCH 29/67] [patch] Exercise checked lifecycle executor in CI --- ci/compose-runtime-contract.sh | 14 ++++++++++++-- rootfs/home/cloud-compose/compose-apps.sh | 8 ++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index 49e9580..958acd5 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -44,13 +44,16 @@ grep -Fq 'cd /home/cloud-compose' "$repo_root/rootfs/home/cloud-compose/run.sh" if grep -Fq 'su -s /bin/bash -c' "$repo_root/rootfs/home/cloud-compose/run.sh"; then fail "privilege-drop helper still synthesizes a shell program through su" fi -grep -Fq '"$COMPOSE_LIFECYCLE_EXECUTOR" "$lifecycle" "$command"' \ +grep -Fq 'run_compose_lifecycle_executor "$lifecycle" "$command"' \ "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" || \ fail "Compose lifecycle entries do not pass through the checked executor" grep -Fq 'readonly COMPOSE_LIFECYCLE_EXECUTOR="/etc/cloud-compose/libexec/run-lifecycle-program.sh"' \ "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" || \ fail "Compose lifecycle entries do not use the canonical checked executor path" -grep -Fq '"$COMPOSE_LIFECYCLE_EXECUTOR" --validate "$lifecycle" "$command" || return 1' \ +grep -Fq '"$COMPOSE_LIFECYCLE_EXECUTOR" "$@"' \ + "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" || \ + fail "Compose lifecycle executor wrapper does not invoke the canonical checked program" +grep -Fq 'run_compose_lifecycle_executor --validate "$lifecycle" "$command" || return 1' \ "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" || \ fail "Compose lifecycle program sets are not validated before execution" if grep -Fq 'bash -c "$command"' "$repo_root/rootfs/home/cloud-compose/compose-apps.sh"; then @@ -66,6 +69,13 @@ mkdir -p "$CLOUD_COMPOSE_DATA_ROOT/project" # shellcheck disable=SC1091 source "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" +# Exercise the checked-in executor while retaining the immutable production +# path contract above. The production image installs the same file at the +# canonical /etc location. +run_compose_lifecycle_executor() { + "$repo_root/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh" "$@" +} + cat >"$CLOUD_COMPOSE_DATA_ROOT/project/compose.yaml" <<'EOF' secrets: DB_ROOT_PASSWORD: diff --git a/rootfs/home/cloud-compose/compose-apps.sh b/rootfs/home/cloud-compose/compose-apps.sh index 5d30dd0..448c07d 100644 --- a/rootfs/home/cloud-compose/compose-apps.sh +++ b/rootfs/home/cloud-compose/compose-apps.sh @@ -10,6 +10,10 @@ compose_apps_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" COMPOSE_SECRET_FILES_PROGRAM="${CLOUD_COMPOSE_COMPOSE_SECRET_FILES_PROGRAM:-$compose_apps_dir/compose-secret-files.awk}" readonly COMPOSE_LIFECYCLE_EXECUTOR="/etc/cloud-compose/libexec/run-lifecycle-program.sh" +run_compose_lifecycle_executor() { + "$COMPOSE_LIFECYCLE_EXECUTOR" "$@" +} + shell_env_line() { local name="$1" local value="$2" @@ -1065,7 +1069,7 @@ run_compose_app_lifecycle() { compose_app_array_values "$app" "$field" commands || return 1 for command in "${commands[@]}"; do [[ -n "$command" ]] || continue - "$COMPOSE_LIFECYCLE_EXECUTOR" --validate "$lifecycle" "$command" || return 1 + run_compose_lifecycle_executor --validate "$lifecycle" "$command" || return 1 done case "$lifecycle" in @@ -1103,7 +1107,7 @@ run_compose_app_lifecycle() { if [ -z "$command" ]; then continue fi - "$COMPOSE_LIFECYCLE_EXECUTOR" "$lifecycle" "$command" || { + run_compose_lifecycle_executor "$lifecycle" "$command" || { command_status=$? popd >/dev/null return "$command_status" From 7bce652c9c3c4e2516bc496b2847fa16059ca52f Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:05:30 +0000 Subject: [PATCH 30/67] [patch] Use checked lifecycle fixture in CI --- ci/compose-runtime-contract.sh | 25 ++++++++++++++++++------- ci/fixtures/lifecycle.d/default-up | 6 ++++++ ci/fixtures/sitectl-argv-log.sh | 5 +++++ 3 files changed, 29 insertions(+), 7 deletions(-) create mode 100755 ci/fixtures/lifecycle.d/default-up create mode 100755 ci/fixtures/sitectl-argv-log.sh diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index 958acd5..96d74f9 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -213,27 +213,38 @@ reject_host_network_compose_services # Preserve list(string) verify arguments as argv. A value containing spaces is # one argument, not an unquoted scalar split by the lifecycle shell. -jq -n --arg project_dir "$CLOUD_COMPOSE_DATA_ROOT/project" '{app: { +jq -n \ + --arg project_dir "$CLOUD_COMPOSE_DATA_ROOT/project" \ + --arg up_program "$repo_root/ci/fixtures/lifecycle.d/default-up" '{app: { docker_compose_repo: "https://github.com/libops/wp.git", docker_compose_branch: "main", project_dir: $project_dir, compose_project_name: "app", sitectl_context_name: "app", + sitectl_environment: "preview", sitectl_verify_args: ["--label", "value with spaces"], - up_commands: ["sitectl verify --context \"$SITECTL_CONTEXT_NAME\" ${SITECTL_VERIFY_ARGS:-}"], + up_commands: [$up_program], init_commands: [], down_commands: [], rollout_commands: [] }}' >"$COMPOSE_PROJECTS_FILE" -cat >"$tmp/bin/sitectl" <<'EOF' -#!/usr/bin/env bash -printf '<%s>\n' "$@" >"${SITECTL_ARGV_LOG:?}" -EOF -chmod +x "$tmp/bin/sitectl" +ln -s "$repo_root/ci/fixtures/sitectl-argv-log.sh" "$tmp/bin/sitectl" export SITECTL_ARGV_LOG="$tmp/sitectl.argv" +export CLOUD_COMPOSE_LIFECYCLE_PROGRAM_DIR="$repo_root/ci/fixtures/lifecycle.d" +export CLOUD_COMPOSE_SITECTL_VERIFY_ARGS_PROGRAM="$repo_root/rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq" clone_or_update_compose_app() { source_compose_app_env "$1"; } record_compose_app_head() { return 0; } CLOUD_COMPOSE_PROVIDER=linode run_compose_app_lifecycle app up cat >"$tmp/expected.argv" <<'EOF' + +<--context> + + +<-d> +<--remove-orphans> + +<--context> + +<--persist> <--context> diff --git a/ci/fixtures/lifecycle.d/default-up b/ci/fixtures/lifecycle.d/default-up new file mode 100755 index 0000000..6888e8d --- /dev/null +++ b/ci/fixtures/lifecycle.d/default-up @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd)" +exec "$repo_root/rootfs/home/cloud-compose/default-lifecycle.sh" up diff --git a/ci/fixtures/sitectl-argv-log.sh b/ci/fixtures/sitectl-argv-log.sh new file mode 100755 index 0000000..0d284d2 --- /dev/null +++ b/ci/fixtures/sitectl-argv-log.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +printf '<%s>\n' "$@" >>"${SITECTL_ARGV_LOG:?SITECTL_ARGV_LOG is required}" From 7adeeaf6dcd7b9c9db7614b0e540d3f840d3c645 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:11:55 +0000 Subject: [PATCH 31/67] [patch] Invoke lifecycle fixture with Bash --- ci/fixtures/lifecycle.d/default-up | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/fixtures/lifecycle.d/default-up b/ci/fixtures/lifecycle.d/default-up index 6888e8d..db50257 100755 --- a/ci/fixtures/lifecycle.d/default-up +++ b/ci/fixtures/lifecycle.d/default-up @@ -3,4 +3,4 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd)" -exec "$repo_root/rootfs/home/cloud-compose/default-lifecycle.sh" up +exec bash "$repo_root/rootfs/home/cloud-compose/default-lifecycle.sh" up From d0944553a0504a64b49768433401c8de4fe061b2 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:18:05 +0000 Subject: [PATCH 32/67] [patch] Reuse checked executor fixture in contracts --- ci/compose-runtime-contract.sh | 6 +++--- ci/fixtures/checked-lifecycle-executor.sh | 7 +++++++ ci/source-trust-contract.sh | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 ci/fixtures/checked-lifecycle-executor.sh diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index 96d74f9..96a0a9c 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -72,9 +72,9 @@ source "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" # Exercise the checked-in executor while retaining the immutable production # path contract above. The production image installs the same file at the # canonical /etc location. -run_compose_lifecycle_executor() { - "$repo_root/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh" "$@" -} +export CLOUD_COMPOSE_TEST_LIFECYCLE_EXECUTOR="$repo_root/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh" +# shellcheck disable=SC1091 +source "$repo_root/ci/fixtures/checked-lifecycle-executor.sh" cat >"$CLOUD_COMPOSE_DATA_ROOT/project/compose.yaml" <<'EOF' secrets: diff --git a/ci/fixtures/checked-lifecycle-executor.sh b/ci/fixtures/checked-lifecycle-executor.sh new file mode 100644 index 0000000..5823272 --- /dev/null +++ b/ci/fixtures/checked-lifecycle-executor.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +: "${CLOUD_COMPOSE_TEST_LIFECYCLE_EXECUTOR:?CLOUD_COMPOSE_TEST_LIFECYCLE_EXECUTOR is required}" + +run_compose_lifecycle_executor() { + "$CLOUD_COMPOSE_TEST_LIFECYCLE_EXECUTOR" "$@" +} diff --git a/ci/source-trust-contract.sh b/ci/source-trust-contract.sh index 60a6b0d..7ddb613 100644 --- a/ci/source-trust-contract.sh +++ b/ci/source-trust-contract.sh @@ -68,6 +68,9 @@ retry_until_success() { # shellcheck disable=SC1091 source "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" +export CLOUD_COMPOSE_TEST_LIFECYCLE_EXECUTOR="$repo_root/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh" +# shellcheck disable=SC1091 +source "$repo_root/ci/fixtures/checked-lifecycle-executor.sh" pinned_checkout="$tmp/pinned" write_project pinned "$commit_one" "$pinned_checkout" From 166df9b08c03223ab9404aedc249b8d2ee4271ec Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:24:00 +0000 Subject: [PATCH 33/67] [patch] Refresh hosted smoke contracts --- .../contracttest/cloud_smoke_cleanup_test.go | 17 +++++++++++++++++ .../testdata/cloud-smoke-lifecycle/ssh.sh | 6 ++++++ 2 files changed, 23 insertions(+) diff --git a/internal/contracttest/cloud_smoke_cleanup_test.go b/internal/contracttest/cloud_smoke_cleanup_test.go index 451e643..6577b74 100644 --- a/internal/contracttest/cloud_smoke_cleanup_test.go +++ b/internal/contracttest/cloud_smoke_cleanup_test.go @@ -217,10 +217,12 @@ func TestHostedProviderTokensAreScopedToLifecycleSteps(t *testing.T) { } for label, marker := range map[string]string{ "config-management smoke token": ` - name: Run Linode config-management smoke test + timeout-minutes: 100 env: LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}`, "config-management destroy token": ` - name: Destroy Linode config-management smoke resources if: always() + timeout-minutes: 20 env: LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}`, } { @@ -347,6 +349,7 @@ printf '%s\n' "$@" >"$FAKE_CLEANUP_LOG" func TestNonGCPSmokeRunExitCleanupLifecycle(t *testing.T) { t.Parallel() root := repositoryRoot(t) + checkoutSHA := repositoryCommitSHA(t, root) tests := []struct { name string driver string @@ -419,6 +422,7 @@ func TestNonGCPSmokeRunExitCleanupLifecycle(t *testing.T) { "CLOUD_COMPOSE_SMOKE_RUN_ID": "123456789", "CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS": "false", "CLOUD_COMPOSE_SMOKE_WORKDIR": filepath.Join(stateDirectory, "smoke"), + "CLOUD_COMPOSE_SOURCE_REF": checkoutSHA, "CLOUD_COMPOSE_SOURCE_SHA256": strings.Repeat("0", 64), "DIGITALOCEAN_TOKEN": "do-lifecycle-secret", "FAKE_APPLY_SIGNAL": test.applySignal, @@ -580,6 +584,19 @@ func processExitCode(t testing.TB, err error) int { return exitError.ExitCode() } +func repositoryCommitSHA(t testing.TB, root string) string { + t.Helper() + output, err := exec.Command("git", "-C", root, "rev-parse", "HEAD").Output() + if err != nil { + t.Fatalf("resolve repository commit: %v", err) + } + commit := strings.TrimSpace(string(output)) + if !regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString(commit) { + t.Fatalf("repository commit is not an exact lowercase SHA: %q", commit) + } + return commit +} + func readTestLog(t testing.TB, path string) string { t.Helper() contents, err := os.ReadFile(path) diff --git a/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh b/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh index be68e10..4460617 100644 --- a/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh +++ b/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh @@ -3,6 +3,12 @@ set -euo pipefail case "$*" in + *mktemp\ -d\ /tmp/cloud-compose-hosted-contract.XXXXXX*) + printf '/tmp/cloud-compose-hosted-contract.fixture123\n' + ;; + *install\ -m\ 0700\ /dev/stdin*) + cat >/dev/null + ;; *cloud-compose-diagnostics.sh\ state*) printf 'complete\n' ;; *cloud-compose-bootstrap-complete*) printf 'complete\n' ;; *cloud-init\ status*) printf 'cloud-init not installed\n' ;; From df3a69cdb12d4d9f9031f337395c80a95747defb Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:29:19 +0000 Subject: [PATCH 34/67] [patch] Keep template tests offline --- template_versions.tftest.hcl | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/template_versions.tftest.hcl b/template_versions.tftest.hcl index 35a899c..7e0ec96 100644 --- a/template_versions.tftest.hcl +++ b/template_versions.tftest.hcl @@ -27,8 +27,6 @@ run "default_template_uses_v1_core" { project_number = "123456789" } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" compose = { repo = "https://github.com/libops/wp.git" } @@ -60,8 +58,6 @@ run "non_isle_template_uses_v1_release_set" { project_number = "123456789" } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } } @@ -91,8 +87,6 @@ run "isle_template_uses_v1_release_set" { project_number = "123456789" } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } } @@ -131,8 +125,6 @@ run "explicit_application_environment_overrides_template_defaults" { project_number = "123456789" } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" extra_env = { ISLANDORA_TAG = "6.3.20" SITE_LABEL = "repository" @@ -162,8 +154,6 @@ run "explicit_package_versions_override_template_defaults" { project_number = "123456789" } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { package_versions = { sitectl = "v0.40.1" @@ -195,8 +185,6 @@ run "custom_package_set_filters_template_versions" { project_number = "123456789" } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = ["sitectl", "sitectl-wp"] package_versions = { @@ -227,8 +215,6 @@ run "explicit_core_only_package_set_disables_template_plugins" { project_number = "123456789" } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = ["sitectl"] } From e8bc5cc74f4ff34c0fd3042ab9d33f3c7489b615 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:35:46 +0000 Subject: [PATCH 35/67] [patch] Isolate provider tests from release network --- modules/digitalocean/runtime_inputs.tftest.hcl | 6 ------ modules/linode/runtime_inputs.tftest.hcl | 17 +++-------------- providers/do/template_versions.tftest.hcl | 4 ---- providers/linode/template_versions.tftest.hcl | 4 ---- 4 files changed, 3 insertions(+), 28 deletions(-) diff --git a/modules/digitalocean/runtime_inputs.tftest.hcl b/modules/digitalocean/runtime_inputs.tftest.hcl index 6b7eebc..021c3e5 100644 --- a/modules/digitalocean/runtime_inputs.tftest.hcl +++ b/modules/digitalocean/runtime_inputs.tftest.hcl @@ -22,8 +22,6 @@ run "merges_provider_neutral_and_provider_specific_ssh_users" { } } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" users = { shared = ["ssh-ed25519 AAAARUNTIME"] runtime-only = ["ssh-ed25519 AAAANEUTRAL"] @@ -51,8 +49,6 @@ run "exposes_independent_sitectl_package_versions" { variables { name = "do-contract" runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" compose = { repo = "https://github.com/libops/isle.git" } @@ -83,8 +79,6 @@ run "rejects_reserved_extra_environment" { variables { name = "do-contract" runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" compose = { repo = "https://github.com/libops/wp.git" } diff --git a/modules/linode/runtime_inputs.tftest.hcl b/modules/linode/runtime_inputs.tftest.hcl index 672df96..2ec5778 100644 --- a/modules/linode/runtime_inputs.tftest.hcl +++ b/modules/linode/runtime_inputs.tftest.hcl @@ -25,8 +25,6 @@ run "merges_provider_neutral_and_provider_specific_ssh_users" { } } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" users = { shared = ["ssh-ed25519 AAAARUNTIME"] runtime-only = ["ssh-ed25519 AAAANEUTRAL"] @@ -61,8 +59,6 @@ run "rejects_multiline_authorized_key" { } } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" compose = { repo = "https://github.com/libops/wp.git" } @@ -83,8 +79,6 @@ run "rejects_unsafe_authorized_username" { } } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" compose = { repo = "https://github.com/libops/wp.git" } @@ -114,8 +108,6 @@ run "rejects_public_rollout_listener" { } } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" compose = { repo = "https://github.com/libops/wp.git" } @@ -152,8 +144,6 @@ run "exposes_independent_sitectl_package_versions" { } } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" compose = { repo = "https://github.com/libops/isle.git" } @@ -189,8 +179,6 @@ run "rejects_reserved_extra_environment" { } } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" compose = { repo = "https://github.com/libops/wp.git" } @@ -222,8 +210,9 @@ run "archive_bootstrap_fits_linode_metadata_limit" { } } runtime = { - rootfs_archive_url = "https://github.com/libops/cloud-compose/releases/download/1.0.0/cloud-compose-rootfs.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" compose = { repo = "https://github.com/libops/wp.git" branch = "main" diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index b61494a..a215441 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -15,8 +15,6 @@ run "custom_package_set_merges_only_applicable_template_versions" { name = "template-versions" template = "isle" runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = ["sitectl", "sitectl-wp"] package_versions = { @@ -53,8 +51,6 @@ run "explicit_core_only_package_set_disables_template_plugins" { name = "template-versions" template = "isle" runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = [] } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index 84936d0..9135629 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -20,8 +20,6 @@ run "custom_package_set_merges_only_applicable_template_versions" { } } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = ["sitectl", "sitectl-wp"] package_versions = { @@ -63,8 +61,6 @@ run "explicit_core_only_package_set_disables_template_plugins" { } } runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" - rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = [] } From 8bdc89a67df074c7ae1b1edd8125d5dcb14cd2e6 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:49:52 +0000 Subject: [PATCH 36/67] [patch] Keep DigitalOcean tests in archive mode --- modules/digitalocean/runtime_inputs.tftest.hcl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/digitalocean/runtime_inputs.tftest.hcl b/modules/digitalocean/runtime_inputs.tftest.hcl index 021c3e5..e78e2a1 100644 --- a/modules/digitalocean/runtime_inputs.tftest.hcl +++ b/modules/digitalocean/runtime_inputs.tftest.hcl @@ -22,6 +22,9 @@ run "merges_provider_neutral_and_provider_specific_ssh_users" { } } runtime = { + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" users = { shared = ["ssh-ed25519 AAAARUNTIME"] runtime-only = ["ssh-ed25519 AAAANEUTRAL"] @@ -49,6 +52,9 @@ run "exposes_independent_sitectl_package_versions" { variables { name = "do-contract" runtime = { + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" compose = { repo = "https://github.com/libops/isle.git" } @@ -79,6 +85,9 @@ run "rejects_reserved_extra_environment" { variables { name = "do-contract" runtime = { + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" compose = { repo = "https://github.com/libops/wp.git" } From cbf66568b07699bcce22790fad452c2d8cc767b1 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:02:38 +0000 Subject: [PATCH 37/67] [patch] Make the GCP archive contract test hermetic --- modules/gcp/runtime_contracts.tftest.hcl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index a91dacb..1013baf 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -564,6 +564,14 @@ run "renders_verified_archive_before_downstream_overlay" { } } + override_data { + target = data.http.rootfs_contract[0] + values = { + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + status_code = 200 + } + } + assert { condition = ( strcontains( From 01daec8284f1952eaeb88fc54157557ee94c17a5 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:07:38 +0000 Subject: [PATCH 38/67] [patch] Correct the rootfs release sidecar contract --- modules/gcp/main.tf | 2 +- modules/gcp/runtime_contracts.tftest.hcl | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index 99a459b..371b5cd 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -27,7 +27,7 @@ locals { rootfs_archive_url = trimspace(var.rootfs_archive_url) rootfs_archive_sha256 = lower(trimspace(var.rootfs_archive_sha256)) rootfs_archive_asset_url = split("#", split("?", local.rootfs_archive_url)[0])[0] - rootfs_contract_sidecar_url = local.rootfs_archive_url == "" ? "" : replace(local.rootfs_archive_asset_url, "/[^/]+$/", "/cloud-compose-rootfs.contract.sha256") + rootfs_contract_sidecar_url = local.rootfs_archive_url == "" ? "" : replace(local.rootfs_archive_asset_url, "/[^/]+$/", "cloud-compose-rootfs.contract.sha256") rootfs_overlay_staging_path = "/var/lib/cloud-compose/rootfs-overlay" project_number = tostring(data.google_project.service.number) diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index 1013baf..c2c83d1 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -747,9 +747,18 @@ run "rejects_archive_without_checksum" { project_id = "test-project" project_number = "123456789" docker_compose_repo = "https://github.com/libops/wp.git" + rootfs = "testdata/rootfs" rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" } + override_data { + target = data.http.rootfs_contract[0] + values = { + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + status_code = 200 + } + } + expect_failures = [google_compute_instance.cloud-compose] } From 770edcc232c8e2871e962ea6ee5ac8f06070bbec Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:13:29 +0000 Subject: [PATCH 39/67] [patch] Scope the GCP bootstrap order contract --- modules/gcp/runtime_contracts.tftest.hcl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index c2c83d1..822edc0 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -588,10 +588,8 @@ run "renders_verified_archive_before_downstream_overlay" { assert { condition = ( - strcontains(local.cloud_init_yaml, "[bash, /var/lib/cloud-compose/bootstrap/rootfs-archive.sh, install") && - strcontains(local.cloud_init_yaml, "[bash, /var/lib/cloud-compose/bootstrap/gcp-cloud-init-finalize.sh") && - length(split("/var/lib/cloud-compose/bootstrap/rootfs-archive.sh, install", local.cloud_init_yaml)[0]) < - length(split("/var/lib/cloud-compose/bootstrap/gcp-cloud-init-finalize.sh", local.cloud_init_yaml)[0]) + strcontains(local.cloud_config_yaml, "runcmd:\n- [bash, /var/lib/cloud-compose/bootstrap/rootfs-archive.sh, install") && + strcontains(local.cloud_config_yaml, "\n- [bash, /var/lib/cloud-compose/bootstrap/gcp-cloud-init-finalize.sh") ) error_message = "The verified base archive and downstream overlay must be installed before GCP application initialization." } From aea360041fab66d4a8b7a0e8cdcad4c3833e8e41 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:21:25 +0000 Subject: [PATCH 40/67] [patch] Exercise Linode contracts in archive mode --- modules/linode/runtime_inputs.tftest.hcl | 21 ++++++++++++++++++++- modules/linux-vm-runtime/main.tf | 2 +- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/modules/linode/runtime_inputs.tftest.hcl b/modules/linode/runtime_inputs.tftest.hcl index 2ec5778..2072907 100644 --- a/modules/linode/runtime_inputs.tftest.hcl +++ b/modules/linode/runtime_inputs.tftest.hcl @@ -25,6 +25,9 @@ run "merges_provider_neutral_and_provider_specific_ssh_users" { } } runtime = { + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" users = { shared = ["ssh-ed25519 AAAARUNTIME"] runtime-only = ["ssh-ed25519 AAAANEUTRAL"] @@ -59,6 +62,9 @@ run "rejects_multiline_authorized_key" { } } runtime = { + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" compose = { repo = "https://github.com/libops/wp.git" } @@ -79,6 +85,9 @@ run "rejects_unsafe_authorized_username" { } } runtime = { + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" compose = { repo = "https://github.com/libops/wp.git" } @@ -108,6 +117,9 @@ run "rejects_public_rollout_listener" { } } runtime = { + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" compose = { repo = "https://github.com/libops/wp.git" } @@ -123,7 +135,8 @@ run "rejects_archive_without_checksum" { variables { name = "contract-test" runtime = { - rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz" + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" compose = { repo = "https://github.com/libops/wp.git" } @@ -144,6 +157,9 @@ run "exposes_independent_sitectl_package_versions" { } } runtime = { + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" compose = { repo = "https://github.com/libops/isle.git" } @@ -179,6 +195,9 @@ run "rejects_reserved_extra_environment" { } } runtime = { + rootfs_archive_url = "https://github.com/libops/cloud-compose/archive/1111111111111111111111111111111111111111.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rootfs_test_source_archive_prefix = "cloud-compose-1111111111111111111111111111111111111111" compose = { repo = "https://github.com/libops/wp.git" } diff --git a/modules/linux-vm-runtime/main.tf b/modules/linux-vm-runtime/main.tf index 9130e30..375ddd7 100644 --- a/modules/linux-vm-runtime/main.tf +++ b/modules/linux-vm-runtime/main.tf @@ -5,7 +5,7 @@ locals { rootfs_archive_sha256 = lower(trimspace(var.rootfs_archive_sha256)) rootfs_test_source_archive_prefix = trimspace(var.rootfs_test_source_archive_prefix) rootfs_archive_asset_url = split("#", split("?", local.rootfs_archive_url)[0])[0] - rootfs_contract_sidecar_url = local.rootfs_archive_url == "" || local.rootfs_test_source_archive_prefix != "" ? "" : replace(local.rootfs_archive_asset_url, "/[^/]+$/", "/cloud-compose-rootfs.contract.sha256") + rootfs_contract_sidecar_url = local.rootfs_archive_url == "" || local.rootfs_test_source_archive_prefix != "" ? "" : replace(local.rootfs_archive_asset_url, "/[^/]+$/", "cloud-compose-rootfs.contract.sha256") rootfs_overlay_staging_path = "/var/lib/cloud-compose/rootfs-overlay" single_compose_project = { From 216a15dcf6deae42c90fcefe1de1af2ad4d7eb7c Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:29:01 +0000 Subject: [PATCH 41/67] [patch] Refresh rootfs contract fixture --- modules/linux-vm-runtime/runtime_inputs.tftest.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index f5afb43..9e66041 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -1,7 +1,7 @@ mock_provider "http" { mock_data "http" { defaults = { - response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" status_code = 200 } } From d9d6f3160f86ef432d99347d53ae6c80b287f18c Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:36:04 +0000 Subject: [PATCH 42/67] [patch] Keep provider contracts in archive mode --- modules/digitalocean/runtime_inputs.tftest.hcl | 2 +- modules/gcp/runtime_contracts.tftest.hcl | 2 +- modules/linode/runtime_inputs.tftest.hcl | 2 +- providers/do/template_versions.tftest.hcl | 6 +++++- providers/linode/template_versions.tftest.hcl | 2 +- template_versions.tftest.hcl | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/digitalocean/runtime_inputs.tftest.hcl b/modules/digitalocean/runtime_inputs.tftest.hcl index e78e2a1..de805fe 100644 --- a/modules/digitalocean/runtime_inputs.tftest.hcl +++ b/modules/digitalocean/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "digitalocean" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" status_code = 200 } } diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index 822edc0..14064a6 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "cloudinit" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" status_code = 200 } } diff --git a/modules/linode/runtime_inputs.tftest.hcl b/modules/linode/runtime_inputs.tftest.hcl index 2072907..8d32e50 100644 --- a/modules/linode/runtime_inputs.tftest.hcl +++ b/modules/linode/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "linode" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" status_code = 200 } } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index a215441..e3b2105 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "digitalocean" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" status_code = 200 } } @@ -15,6 +15,8 @@ run "custom_package_set_merges_only_applicable_template_versions" { name = "template-versions" template = "isle" runtime = { + rootfs_archive_url = "https://example.invalid/cloud-compose-rootfs.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = ["sitectl", "sitectl-wp"] package_versions = { @@ -51,6 +53,8 @@ run "explicit_core_only_package_set_disables_template_plugins" { name = "template-versions" template = "isle" runtime = { + rootfs_archive_url = "https://example.invalid/cloud-compose-rootfs.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = [] } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index 9135629..b99ebdc 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "linode" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" status_code = 200 } } diff --git a/template_versions.tftest.hcl b/template_versions.tftest.hcl index 7e0ec96..12caae8 100644 --- a/template_versions.tftest.hcl +++ b/template_versions.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "cloudinit" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "8cc800954d4780c933ebd680b25ec7dacfb61a733b9295f272ab56ac8fbf6b74\n" + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" status_code = 200 } } From 54bb577978b90b3a64c729ec4d6f55ad83d6180e Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:48:19 +0000 Subject: [PATCH 43/67] [patch] Override nested provider contract data --- providers/do/template_versions.tftest.hcl | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index e3b2105..78b46d5 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -1,15 +1,15 @@ mock_provider "digitalocean" {} -mock_provider "http" { - mock_data "http" { - defaults = { + +run "custom_package_set_merges_only_applicable_template_versions" { + command = plan + + override_data { + target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] + values = { response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" status_code = 200 } } -} - -run "custom_package_set_merges_only_applicable_template_versions" { - command = plan variables { name = "template-versions" @@ -49,6 +49,14 @@ run "custom_package_set_merges_only_applicable_template_versions" { run "explicit_core_only_package_set_disables_template_plugins" { command = plan + override_data { + target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] + values = { + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + status_code = 200 + } + } + variables { name = "template-versions" template = "isle" From 73a12f65011bc9638b466ca8cedb796279385523 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 07:56:25 +0000 Subject: [PATCH 44/67] [patch] Keep Linode provider contracts in archive mode --- providers/linode/template_versions.tftest.hcl | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index b99ebdc..f93c7ee 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -1,15 +1,15 @@ mock_provider "linode" {} -mock_provider "http" { - mock_data "http" { - defaults = { + +run "custom_package_set_merges_only_applicable_template_versions" { + command = plan + + override_data { + target = module.linode.module.runtime.data.http.rootfs_contract[0] + values = { response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" status_code = 200 } } -} - -run "custom_package_set_merges_only_applicable_template_versions" { - command = plan variables { name = "template-versions" @@ -20,6 +20,8 @@ run "custom_package_set_merges_only_applicable_template_versions" { } } runtime = { + rootfs_archive_url = "https://example.invalid/cloud-compose-rootfs.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = ["sitectl", "sitectl-wp"] package_versions = { @@ -52,6 +54,14 @@ run "custom_package_set_merges_only_applicable_template_versions" { run "explicit_core_only_package_set_disables_template_plugins" { command = plan + override_data { + target = module.linode.module.runtime.data.http.rootfs_contract[0] + values = { + response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + status_code = 200 + } + } + variables { name = "template-versions" template = "isle" @@ -61,6 +71,8 @@ run "explicit_core_only_package_set_disables_template_plugins" { } } runtime = { + rootfs_archive_url = "https://example.invalid/cloud-compose-rootfs.tar.gz" + rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sitectl = { packages = [] } From 2eca1e5abfcbb003277c69030fbc9b660fc65c54 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 08:11:48 +0000 Subject: [PATCH 45/67] [patch] Promote hardened compose templates --- README.md | 6 +++--- docs/examples.md | 2 +- docs/managed-runtime.md | 8 ++++---- providers/do/template_versions.tftest.hcl | 4 ++-- providers/gcp/template_versions.tftest.hcl | 4 ++-- providers/linode/template_versions.tftest.hcl | 4 ++-- templates/apps.json | 14 +++++++------- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 429bcd4..1398cd6 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ Its pull requests are promotion candidates: keep the complete compatible set together and require the hosted application smoke before merging them. All presets pin sitectl v1.8.2 plus the latest reviewed application plugin. -ISLE selects the `libops/isle` v1.3.0 template, Drupal and WordPress select -their v1.1.0 templates, and the remaining application templates stay on -v1.0.0. Keep each preset's complete template and package set together when +ISLE selects the `libops/isle` v1.3.1 template, ArchivesSpace selects v1.0.1, +WordPress selects v1.1.1, and Drupal, OJS, Omeka Classic, and Omeka S select +v1.2.1. Keep each preset's complete template and package set together when promoting an override. GCP deployments have two Terraform ownership layers. Apply diff --git a/docs/examples.md b/docs/examples.md index 075a732..6cd9e1c 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -141,7 +141,7 @@ runtime = { } } compose = { - branch = "v1.1.0" + branch = "v1.1.1" ingress = { letsencrypt = true bot_mitigation = true diff --git a/docs/managed-runtime.md b/docs/managed-runtime.md index bc10f56..1bd971c 100644 --- a/docs/managed-runtime.md +++ b/docs/managed-runtime.md @@ -27,10 +27,10 @@ backward-compatible fallback only for an installed package with neither a template selector nor an explicit per-package selector. All presets pin sitectl v1.8.2 with a reviewed application plugin. ISLE uses -the `libops/isle` v1.3.0 template with sitectl-drupal v1.3.0 and sitectl-isle -v1.5.0. Drupal and WordPress use their v1.1.0 templates; the other application -templates remain on v1.0.0. Override the template or package selectors only as -one reviewed, compatible release set. +the `libops/isle` v1.3.1 template with sitectl-drupal v1.3.0 and sitectl-isle +v1.5.0. ArchivesSpace uses v1.0.1, WordPress uses v1.1.1, and Drupal, OJS, +Omeka Classic, and Omeka S use v1.2.1. Override template or package selectors +only as one reviewed, compatible release set. Omitting `runtime.sitectl.packages` selects the template's package set. An explicit list replaces that set; `packages = []` or `packages = ["sitectl"]` diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index 78b46d5..43974ae 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -36,8 +36,8 @@ run "custom_package_set_merges_only_applicable_template_versions" { } assert { - condition = local.runtime.compose.branch == "v1.3.0" - error_message = "The DigitalOcean entrypoint must inherit the ISLE v1.3.0 template branch when no override is supplied." + condition = local.runtime.compose.branch == "v1.3.1" + error_message = "The DigitalOcean entrypoint must inherit the ISLE v1.3.1 template branch when no override is supplied." } assert { diff --git a/providers/gcp/template_versions.tftest.hcl b/providers/gcp/template_versions.tftest.hcl index 4b7ff77..6b9d232 100644 --- a/providers/gcp/template_versions.tftest.hcl +++ b/providers/gcp/template_versions.tftest.hcl @@ -38,8 +38,8 @@ run "custom_package_set_merges_only_applicable_template_versions" { } assert { - condition = local.runtime.compose.branch == "v1.3.0" - error_message = "The GCP entrypoint must inherit the ISLE v1.3.0 template branch when no override is supplied." + condition = local.runtime.compose.branch == "v1.3.1" + error_message = "The GCP entrypoint must inherit the ISLE v1.3.1 template branch when no override is supplied." } assert { diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index f93c7ee..bfaa8c2 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -41,8 +41,8 @@ run "custom_package_set_merges_only_applicable_template_versions" { } assert { - condition = local.runtime.compose.branch == "v1.3.0" - error_message = "The Linode entrypoint must inherit the ISLE v1.3.0 template branch when no override is supplied." + condition = local.runtime.compose.branch == "v1.3.1" + error_message = "The Linode entrypoint must inherit the ISLE v1.3.1 template branch when no override is supplied." } assert { diff --git a/templates/apps.json b/templates/apps.json index ac347ac..38a6a32 100644 --- a/templates/apps.json +++ b/templates/apps.json @@ -12,7 +12,7 @@ "templates": { "archivesspace": { "repo": "https://github.com/libops/archivesspace.git", - "branch": "v1.0.0", + "branch": "v1.0.1", "plugin": "archivesspace", "packages": ["sitectl", "sitectl-archivesspace"], "extra_env": {}, @@ -23,7 +23,7 @@ }, "ojs": { "repo": "https://github.com/libops/ojs.git", - "branch": "v1.0.0", + "branch": "v1.2.1", "plugin": "ojs", "packages": ["sitectl", "sitectl-ojs"], "extra_env": {}, @@ -34,7 +34,7 @@ }, "isle": { "repo": "https://github.com/libops/isle", - "branch": "v1.3.0", + "branch": "v1.3.1", "plugin": "isle", "packages": ["sitectl", "sitectl-drupal", "sitectl-isle"], "extra_env": { @@ -48,7 +48,7 @@ }, "drupal": { "repo": "https://github.com/libops/drupal.git", - "branch": "v1.1.0", + "branch": "v1.2.1", "plugin": "drupal", "packages": ["sitectl", "sitectl-drupal"], "extra_env": {}, @@ -59,7 +59,7 @@ }, "wp": { "repo": "https://github.com/libops/wp.git", - "branch": "v1.1.0", + "branch": "v1.1.1", "plugin": "wp", "packages": ["sitectl", "sitectl-wp"], "extra_env": {}, @@ -70,7 +70,7 @@ }, "omeka-s": { "repo": "https://github.com/libops/omeka-s.git", - "branch": "v1.0.0", + "branch": "v1.2.1", "plugin": "omeka-s", "packages": ["sitectl", "sitectl-omeka-s"], "extra_env": {}, @@ -81,7 +81,7 @@ }, "omeka-classic": { "repo": "https://github.com/libops/omeka-classic.git", - "branch": "v1.0.0", + "branch": "v1.2.1", "plugin": "omeka-classic", "packages": ["sitectl", "sitectl-omeka-classic"], "extra_env": {}, From 54cb265301de6d89c41c61312ee7b5893078c0c5 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 08:21:35 +0000 Subject: [PATCH 46/67] [patch] Align promoted template contracts --- ci/config-management-smoke-assert.py | 2 +- internal/contracttest/template_version_test.go | 14 +++++++------- template_versions.tftest.hcl | 8 ++++---- tests/config-management/ansible/smoke.yml | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ci/config-management-smoke-assert.py b/ci/config-management-smoke-assert.py index f132cfe..f229946 100644 --- a/ci/config-management-smoke-assert.py +++ b/ci/config-management-smoke-assert.py @@ -158,7 +158,7 @@ def assert_ansible_runtime() -> None: } assert project["docker_compose_repo"] == "https://github.com/libops/isle" assert project["project_dir"] == "/mnt/disks/data/libops/isle/isle-prod" - assert project["compose_project_name"] == "libops-isle-v1-3-0" + assert project["compose_project_name"] == "libops-isle-v1-3-1" assert project["ingress"]["domain"] == "isle.example.edu" assert project["sitectl_plugin"] == "isle" assert project["init_commands"] == [] diff --git a/internal/contracttest/template_version_test.go b/internal/contracttest/template_version_test.go index 1b531bb..2e79cce 100644 --- a/internal/contracttest/template_version_test.go +++ b/internal/contracttest/template_version_test.go @@ -63,13 +63,13 @@ func TestTemplateVersionContract(t *testing.T) { }, } expectedBranches := map[string]string{ - "archivesspace": "v1.0.0", - "drupal": "v1.1.0", - "isle": "v1.3.0", - "ojs": "v1.0.0", - "omeka-classic": "v1.0.0", - "omeka-s": "v1.0.0", - "wp": "v1.1.0", + "archivesspace": "v1.0.1", + "drupal": "v1.2.1", + "isle": "v1.3.1", + "ojs": "v1.2.1", + "omeka-classic": "v1.2.1", + "omeka-s": "v1.2.1", + "wp": "v1.1.1", } if !maps.Equal(registry.Default.PackageVersions, expectedVersions["default"]) { diff --git a/template_versions.tftest.hcl b/template_versions.tftest.hcl index 12caae8..b2b4bf2 100644 --- a/template_versions.tftest.hcl +++ b/template_versions.tftest.hcl @@ -70,8 +70,8 @@ run "non_isle_template_uses_v1_release_set" { } assert { - condition = local.compose.branch == "v1.1.0" - error_message = "The WordPress preset must select its stable v1.1.0 template contract." + condition = local.compose.branch == "v1.1.1" + error_message = "The WordPress preset must select its stable v1.1.1 template contract." } } @@ -100,8 +100,8 @@ run "isle_template_uses_v1_release_set" { } assert { - condition = local.compose.branch == "v1.3.0" - error_message = "The ISLE preset must select the stable v1.3.0 template contract." + condition = local.compose.branch == "v1.3.1" + error_message = "The ISLE preset must select the stable v1.3.1 template contract." } assert { diff --git a/tests/config-management/ansible/smoke.yml b/tests/config-management/ansible/smoke.yml index f03fb35..c2f33ca 100644 --- a/tests/config-management/ansible/smoke.yml +++ b/tests/config-management/ansible/smoke.yml @@ -124,7 +124,7 @@ - '(cloud_compose_application_env_file.content | b64decode | from_json).PORT == "9999"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].docker_compose_repo == "https://github.com/libops/isle"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].project_dir == "/mnt/disks/data/libops/isle/isle-prod"' - - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].compose_project_name == "libops-isle-v1-3-0"' + - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].compose_project_name == "libops-isle-v1-3-1"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].ingress.domain == "isle.example.edu"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].sitectl_plugin == "isle"' - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].init_commands == []' From 6300d66e6f7ce1a61d17ae72178019007b063f7c Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 08:29:15 +0000 Subject: [PATCH 47/67] [patch] Align Salt template contract --- ci/config-management-smoke-inner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/config-management-smoke-inner.sh b/ci/config-management-smoke-inner.sh index 65c6900..01168d1 100755 --- a/ci/config-management-smoke-inner.sh +++ b/ci/config-management-smoke-inner.sh @@ -303,7 +303,7 @@ run_salt_case \ sitectl-wp \ wp.example.edu \ /mnt/disks/data/libops/wp.git/wp-prod \ - libops-wp-v1-1-0 + libops-wp-v1-1-1 run_salt_case \ drupal-prod \ From 9f354ddb588a539423f62075a7648d3057dda9e4 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 08:36:19 +0000 Subject: [PATCH 48/67] [patch] Align Drupal Salt contract --- ci/config-management-smoke-inner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/config-management-smoke-inner.sh b/ci/config-management-smoke-inner.sh index 01168d1..6b6cde6 100755 --- a/ci/config-management-smoke-inner.sh +++ b/ci/config-management-smoke-inner.sh @@ -313,7 +313,7 @@ run_salt_case \ sitectl-drupal \ drupal.example.edu \ /mnt/disks/data/libops/drupal.git/drupal-prod \ - libops-drupal-v1-1-0 + libops-drupal-v1-2-1 run_invalid_salt_case() { local invalid_case="$1" expected="$2" From 180318679efda21f30109d04f633b87a9341fba2 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 09:07:08 +0000 Subject: [PATCH 49/67] [patch] Initialize default component state --- ci/compose-runtime-contract.sh | 31 ++++++++++++------- .../default-ingress-compose-project.jq | 16 ++++++++++ ci/fixtures/default-ingress-sitectl.argv | 10 ++++++ ci/fixtures/verify-argv-compose-project.jq | 15 +++++++++ rootfs/home/cloud-compose/compose-apps.sh | 22 ++++--------- 5 files changed, 67 insertions(+), 27 deletions(-) create mode 100644 ci/fixtures/default-ingress-compose-project.jq create mode 100644 ci/fixtures/default-ingress-sitectl.argv create mode 100644 ci/fixtures/verify-argv-compose-project.jq diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index 96a0a9c..6ecd144 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -215,23 +215,32 @@ reject_host_network_compose_services # one argument, not an unquoted scalar split by the lifecycle shell. jq -n \ --arg project_dir "$CLOUD_COMPOSE_DATA_ROOT/project" \ - --arg up_program "$repo_root/ci/fixtures/lifecycle.d/default-up" '{app: { - docker_compose_repo: "https://github.com/libops/wp.git", - docker_compose_branch: "main", - project_dir: $project_dir, - compose_project_name: "app", - sitectl_context_name: "app", - sitectl_environment: "preview", - sitectl_verify_args: ["--label", "value with spaces"], - up_commands: [$up_program], - init_commands: [], down_commands: [], rollout_commands: [] -}}' >"$COMPOSE_PROJECTS_FILE" + --arg up_program "$repo_root/ci/fixtures/lifecycle.d/default-up" \ + -f "$repo_root/ci/fixtures/verify-argv-compose-project.jq" >"$COMPOSE_PROJECTS_FILE" ln -s "$repo_root/ci/fixtures/sitectl-argv-log.sh" "$tmp/bin/sitectl" export SITECTL_ARGV_LOG="$tmp/sitectl.argv" export CLOUD_COMPOSE_LIFECYCLE_PROGRAM_DIR="$repo_root/ci/fixtures/lifecycle.d" export CLOUD_COMPOSE_SITECTL_VERIFY_ARGS_PROGRAM="$repo_root/rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq" clone_or_update_compose_app() { source_compose_app_env "$1"; } record_compose_app_head() { return 0; } + +# Even a default ingress configuration must initialize durable desired state; +# otherwise sitectl verify correctly rejects the missing .libops/site.yaml. +jq -n \ + --arg project_dir "$CLOUD_COMPOSE_DATA_ROOT/project" \ + -f "$repo_root/ci/fixtures/default-ingress-compose-project.jq" >"$COMPOSE_PROJECTS_FILE" +write_compose_app_env app +: >"$SITECTL_ARGV_LOG" +configure_sitectl_app_features app +cmp -s "$repo_root/ci/fixtures/default-ingress-sitectl.argv" "$SITECTL_ARGV_LOG" || \ + fail "default ingress did not initialize and converge sitectl component state" + +jq -n \ + --arg project_dir "$CLOUD_COMPOSE_DATA_ROOT/project" \ + --arg up_program "$repo_root/ci/fixtures/lifecycle.d/default-up" \ + -f "$repo_root/ci/fixtures/verify-argv-compose-project.jq" >"$COMPOSE_PROJECTS_FILE" +write_compose_app_env app +: >"$SITECTL_ARGV_LOG" CLOUD_COMPOSE_PROVIDER=linode run_compose_app_lifecycle app up cat >"$tmp/expected.argv" <<'EOF' diff --git a/ci/fixtures/default-ingress-compose-project.jq b/ci/fixtures/default-ingress-compose-project.jq new file mode 100644 index 0000000..154789f --- /dev/null +++ b/ci/fixtures/default-ingress-compose-project.jq @@ -0,0 +1,16 @@ +{ + app: { + docker_compose_repo: "https://github.com/libops/wp.git", + docker_compose_branch: "v1.1.1", + project_dir: $project_dir, + compose_project_name: "app", + sitectl_context_name: "app", + sitectl_plugin: "wp", + sitectl_environment: "preview", + ingress: {}, + init_commands: [], + up_commands: [], + down_commands: [], + rollout_commands: [] + } +} diff --git a/ci/fixtures/default-ingress-sitectl.argv b/ci/fixtures/default-ingress-sitectl.argv new file mode 100644 index 0000000..fb5dc4f --- /dev/null +++ b/ci/fixtures/default-ingress-sitectl.argv @@ -0,0 +1,10 @@ + + + +<--context> + +<--yolo> + +<--context> + +<--yolo> diff --git a/ci/fixtures/verify-argv-compose-project.jq b/ci/fixtures/verify-argv-compose-project.jq new file mode 100644 index 0000000..3f22402 --- /dev/null +++ b/ci/fixtures/verify-argv-compose-project.jq @@ -0,0 +1,15 @@ +{ + app: { + docker_compose_repo: "https://github.com/libops/wp.git", + docker_compose_branch: "main", + project_dir: $project_dir, + compose_project_name: "app", + sitectl_context_name: "app", + sitectl_environment: "preview", + sitectl_verify_args: ["--label", "value with spaces"], + up_commands: [$up_program], + init_commands: [], + down_commands: [], + rollout_commands: [] + } +} diff --git a/rootfs/home/cloud-compose/compose-apps.sh b/rootfs/home/cloud-compose/compose-apps.sh index 448c07d..28f4655 100644 --- a/rootfs/home/cloud-compose/compose-apps.sh +++ b/rootfs/home/cloud-compose/compose-apps.sh @@ -988,8 +988,6 @@ verify_existing_compose_app_checkout() { configure_sitectl_app_features() { local app="$1" local letsencrypt bot_mitigation mode domain acme_email max_upload_size upload_timeout - local configure_ingress=false - local changed=false local trusted_ip local -a trusted_ips=() @@ -1010,43 +1008,35 @@ configure_sitectl_app_features() { local ingress_args=(set ingress enabled --context "$SITECTL_CONTEXT_NAME" --yolo) if [ -n "$mode" ]; then ingress_args+=(--mode "$mode") - configure_ingress=true fi if [ -n "$domain" ]; then ingress_args+=(--domain "$domain") - configure_ingress=true fi if [ -n "$acme_email" ]; then ingress_args+=(--acme-email "$acme_email") - configure_ingress=true fi compose_app_ingress_array_values "$app" trusted_ips trusted_ips || return 1 for trusted_ip in "${trusted_ips[@]}"; do if [ -n "$trusted_ip" ]; then ingress_args+=(--trusted-ip "$trusted_ip") - configure_ingress=true fi done if [ -n "$max_upload_size" ]; then ingress_args+=(--max-upload-size "$max_upload_size") - configure_ingress=true fi if [ -n "$upload_timeout" ]; then ingress_args+=(--upload-timeout "$upload_timeout") - configure_ingress=true fi - if [ "$configure_ingress" = true ]; then - sitectl "${ingress_args[@]}" - changed=true - fi + # Always initialize component desired state, including when every ingress + # option uses its default. Verification deliberately fails when + # .libops/site.yaml is absent, and component set initializes every + # registered component from its declared default before applying ingress. + sitectl "${ingress_args[@]}" if sitectl_truthy "$bot_mitigation"; then sitectl set bot-mitigation on --context "$SITECTL_CONTEXT_NAME" --yolo - changed=true - fi - if [ "$changed" = true ]; then - sitectl converge --context "$SITECTL_CONTEXT_NAME" --yolo fi + sitectl converge --context "$SITECTL_CONTEXT_NAME" --yolo } run_compose_app_lifecycle() { From 3294224b1fcdb553bd2f93fb6510a0cf4417dd39 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 09:14:51 +0000 Subject: [PATCH 50/67] [patch] Refresh rootfs contract fixtures --- modules/digitalocean/runtime_inputs.tftest.hcl | 2 +- modules/gcp/runtime_contracts.tftest.hcl | 6 +++--- modules/linode/runtime_inputs.tftest.hcl | 2 +- modules/linux-vm-runtime/runtime_inputs.tftest.hcl | 2 +- providers/do/template_versions.tftest.hcl | 4 ++-- providers/linode/template_versions.tftest.hcl | 4 ++-- template_versions.tftest.hcl | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/digitalocean/runtime_inputs.tftest.hcl b/modules/digitalocean/runtime_inputs.tftest.hcl index de805fe..e781ac0 100644 --- a/modules/digitalocean/runtime_inputs.tftest.hcl +++ b/modules/digitalocean/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "digitalocean" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index 14064a6..c377739 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "cloudinit" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } @@ -567,7 +567,7 @@ run "renders_verified_archive_before_downstream_overlay" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } @@ -752,7 +752,7 @@ run "rejects_archive_without_checksum" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } diff --git a/modules/linode/runtime_inputs.tftest.hcl b/modules/linode/runtime_inputs.tftest.hcl index 8d32e50..2979534 100644 --- a/modules/linode/runtime_inputs.tftest.hcl +++ b/modules/linode/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "linode" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index 9e66041..1a7bd3d 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -1,7 +1,7 @@ mock_provider "http" { mock_data "http" { defaults = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index 43974ae..a43780d 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } @@ -52,7 +52,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index bfaa8c2..f096071 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } @@ -57,7 +57,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } diff --git a/template_versions.tftest.hcl b/template_versions.tftest.hcl index b2b4bf2..57fac94 100644 --- a/template_versions.tftest.hcl +++ b/template_versions.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "cloudinit" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "c71bcb8a431176c641eaded5a8a9d8f36d76ea3ad3d709f8b6d2b3eaa12c7cb0\n" + response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" status_code = 200 } } From 274f255157952e8ec58f1a9daa77854324fa8300 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 09:37:35 +0000 Subject: [PATCH 51/67] [patch] Fence managed Compose reconciliation --- ci/source-trust-contract.sh | 23 +++++ docs/runtime-contracts.md | 10 +++ rootfs/home/cloud-compose/app-init.sh | 1 + rootfs/home/cloud-compose/compose-apps.sh | 101 ++++++++++++++++++++-- 4 files changed, 129 insertions(+), 6 deletions(-) diff --git a/ci/source-trust-contract.sh b/ci/source-trust-contract.sh index 7ddb613..035d48c 100644 --- a/ci/source-trust-contract.sh +++ b/ci/source-trust-contract.sh @@ -209,6 +209,29 @@ fi rm -f -- "$tag_checkout/compose.override.yaml" clone_or_update_compose_app tag assert_head "$tag_checkout" "$commit_one" + +# A successful sitectl reconciliation may intentionally derive tracked runtime +# configuration from the untracked desired-state document. Accept only the +# exact recorded diff, reject any later mutation, and restore the committed +# source before a source move. +printf 'managed\n' >"$tag_checkout/version.txt" +pushd "$tag_checkout" >/dev/null +record_compose_managed_diff tag +verify_clean_compose_checkout tag +printf 'unexpected\n' >>version.txt +if verify_clean_compose_checkout tag >/dev/null 2>&1; then + fail "recorded managed Compose state accepted a different tracked change" +fi +printf 'managed\n' >version.txt +restore_recorded_compose_managed_diff tag +popd >/dev/null +[[ "$(<"$tag_checkout/version.txt")" == "one" ]] || \ + fail "recorded managed Compose state was not restored to committed source" +if ! git -C "$tag_checkout" diff --quiet --ignore-submodules -- || + ! git -C "$tag_checkout" diff --cached --quiet --ignore-submodules --; then + fail "recorded managed Compose restore left tracked changes" +fi + run_compose_app_lifecycle tag init [[ "$(<"$COMPOSE_APPS_STATE_DIR/tag.deployed-head")" == "$commit_one" ]] || \ fail "tag deployed HEAD was not recorded" diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index e160352..f93e60e 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -138,6 +138,16 @@ remote branch moves. The deployed commit for every app is recorded at `/home/cloud-compose/state/.deployed-head`, including branch/tag deployments, so operators can compare desired and observed source state. +`sitectl` component reconciliation can derive tracked runtime configuration +from `.libops/site.yaml`. After a successful initialization or rollout, +cloud-compose records the exact binary Git-diff fingerprint at +`/home/cloud-compose/state/.managed-diff`. Service starts accept that +derived state only while its fingerprint is unchanged. A source update or +rollout restores the committed tree only after the current diff matches the +recorded value, then derives a new fingerprint after success. Partial +reconciliation and operator edits still fail closed; institution-specific +source changes must be committed in the downstream repository. + Use a full commit for reproducible production rollouts. A commit pin fixes the repository contents but does not prove who authored them; protect the selected repository and review/sign commits according to your downstream governance. diff --git a/rootfs/home/cloud-compose/app-init.sh b/rootfs/home/cloud-compose/app-init.sh index 36bf261..a23aa3a 100644 --- a/rootfs/home/cloud-compose/app-init.sh +++ b/rootfs/home/cloud-compose/app-init.sh @@ -31,5 +31,6 @@ for app in "${apps[@]}"; do update_compose_env COMPOSE_BIND_PORT "$COMPOSE_BIND_PORT" run_compose_app_lifecycle "$app" init configure_sitectl_app_features "$app" + record_compose_managed_diff "$app" popd >/dev/null done diff --git a/rootfs/home/cloud-compose/compose-apps.sh b/rootfs/home/cloud-compose/compose-apps.sh index 28f4655..d0efb89 100644 --- a/rootfs/home/cloud-compose/compose-apps.sh +++ b/rootfs/home/cloud-compose/compose-apps.sh @@ -789,6 +789,82 @@ record_compose_app_head() { mv -f "$state_tmp" "$state_file" } +compose_checkout_diff_digest() { + local digest digest_output diff_file + + diff_file="$(mktemp)" || return 1 + if ! git diff --binary --full-index --no-color --no-ext-diff --no-textconv HEAD -- >"$diff_file"; then + rm -f -- "$diff_file" + echo "Could not fingerprint managed Compose changes" >&2 + return 1 + fi + if ! digest_output="$(sha256sum "$diff_file")"; then + rm -f -- "$diff_file" + echo "Could not hash managed Compose changes" >&2 + return 1 + fi + rm -f -- "$diff_file" + digest="${digest_output%% *}" + if [[ ! "$digest" =~ ^[0-9a-f]{64}$ ]]; then + echo "Managed Compose change fingerprint is invalid" >&2 + return 1 + fi + printf '%s\n' "$digest" +} + +record_compose_managed_diff() { + local app="$1" + local digest state_file state_tmp + + validate_compose_app_name "$app" || return 1 + digest="$(compose_checkout_diff_digest)" || return 1 + install -d -m 0750 "$COMPOSE_APPS_STATE_DIR" + state_file="${COMPOSE_APPS_STATE_DIR}/${app}.managed-diff" + state_tmp="$(mktemp "${state_file}.tmp.XXXXXX")" + printf '%s\n' "$digest" >"$state_tmp" + chmod 0640 "$state_tmp" + chown cloud-compose:cloud-compose "$state_tmp" 2>/dev/null || true + mv -f "$state_tmp" "$state_file" +} + +verify_recorded_compose_managed_diff() { + local app="$1" + local current_digest recorded_digest state_file + + validate_compose_app_name "$app" || return 1 + state_file="${COMPOSE_APPS_STATE_DIR}/${app}.managed-diff" + if [[ -L "$state_file" || ! -f "$state_file" ]]; then + echo "Compose checkout contains tracked changes without recorded managed state: $state_file" >&2 + return 1 + fi + recorded_digest="$(<"$state_file")" + if [[ ! "$recorded_digest" =~ ^[0-9a-f]{64}$ ]]; then + echo "Recorded managed Compose state is invalid for ${app}: $state_file" >&2 + return 1 + fi + current_digest="$(compose_checkout_diff_digest)" || return 1 + if [[ "$current_digest" != "$recorded_digest" ]]; then + echo "Compose checkout differs from its recorded sitectl-managed state; commit operator changes in the downstream fork before deployment" >&2 + return 1 + fi +} + +restore_recorded_compose_managed_diff() { + local app="$1" + + if git diff --quiet --ignore-submodules -- && + git diff --cached --quiet --ignore-submodules --; then + return 0 + fi + verify_recorded_compose_managed_diff "$app" || return 1 + git restore --source=HEAD --staged --worktree -- . || return 1 + if ! git diff --quiet --ignore-submodules -- || + ! git diff --cached --quiet --ignore-submodules --; then + echo "Could not restore the committed Compose source before deployment" >&2 + return 1 + fi +} + checkout_exact_compose_commit() { local app="$1" local requested_commit="$2" @@ -824,12 +900,16 @@ verify_compose_origin() { } verify_clean_compose_checkout() { + local app="${1:-}" local untracked_compose_control if ! git diff --quiet --ignore-submodules -- || ! git diff --cached --quiet --ignore-submodules --; then - echo "Compose checkout contains tracked or staged changes; commit them in the downstream fork before deployment" >&2 - return 1 + if [[ -z "$app" ]]; then + echo "Compose checkout contains tracked or staged changes; commit them in the downstream fork before deployment" >&2 + return 1 + fi + verify_recorded_compose_managed_diff "$app" || return 1 fi untracked_compose_control="$(git ls-files --others --exclude-standard -- \ ':(glob)**/compose*.yml' ':(glob)**/compose*.yaml' \ @@ -899,7 +979,7 @@ clone_or_update_compose_app() { return 1 } fi - verify_clean_compose_checkout || { popd >/dev/null; return 1; } + verify_clean_compose_checkout "$app" || { popd >/dev/null; return 1; } if [ "$(id -u)" -eq 0 ]; then chown -R cloud-compose:cloud-compose . || { popd >/dev/null; return 1; } fi @@ -907,7 +987,8 @@ clone_or_update_compose_app() { else pushd "$DOCKER_COMPOSE_DIR" >/dev/null || return 1 verify_compose_origin || { popd >/dev/null; return 1; } - verify_clean_compose_checkout || { popd >/dev/null; return 1; } + verify_clean_compose_checkout "$app" || { popd >/dev/null; return 1; } + restore_recorded_compose_managed_diff "$app" || { popd >/dev/null; return 1; } if compose_ref_is_full_commit "$DOCKER_COMPOSE_BRANCH"; then checkout_exact_compose_commit "$app" "$DOCKER_COMPOSE_BRANCH" || { popd >/dev/null; return 1; } else @@ -953,7 +1034,7 @@ clone_or_update_compose_app() { return 1 fi fi - verify_clean_compose_checkout || { popd >/dev/null; return 1; } + verify_clean_compose_checkout "$app" || { popd >/dev/null; return 1; } popd >/dev/null fi } @@ -971,7 +1052,7 @@ verify_existing_compose_app_checkout() { pushd "$DOCKER_COMPOSE_DIR" >/dev/null || return 1 verify_compose_origin || { popd >/dev/null; return 1; } - verify_clean_compose_checkout || { popd >/dev/null; return 1; } + verify_clean_compose_checkout "$app" || { popd >/dev/null; return 1; } current_head="$(git rev-parse --verify 'HEAD^{commit}')" || { echo "Compose checkout has no deployed commit for ${app}: $DOCKER_COMPOSE_DIR" >&2 popd >/dev/null @@ -1080,6 +1161,11 @@ run_compose_app_lifecycle() { else clone_or_update_compose_app "$app" || return 1 fi + if [[ "$lifecycle" == "rollout" ]]; then + pushd "$DOCKER_COMPOSE_DIR" >/dev/null || return 1 + restore_recorded_compose_managed_diff "$app" || { popd >/dev/null; return 1; } + popd >/dev/null + fi ;; down) source_compose_app_env "$app" || return 1 @@ -1106,5 +1192,8 @@ run_compose_app_lifecycle() { if [[ "$lifecycle" != "down" ]]; then record_compose_app_head "$app" || { popd >/dev/null; return 1; } fi + if [[ "$lifecycle" == "rollout" ]]; then + record_compose_managed_diff "$app" || { popd >/dev/null; return 1; } + fi popd >/dev/null } From 41aaa8cf09cdea0b179bf71e0743207f716b08a0 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 09:54:01 +0000 Subject: [PATCH 52/67] [patch] Repin rootfs contract fixtures --- modules/digitalocean/runtime_inputs.tftest.hcl | 2 +- modules/gcp/runtime_contracts.tftest.hcl | 6 +++--- modules/linode/runtime_inputs.tftest.hcl | 2 +- modules/linux-vm-runtime/runtime_inputs.tftest.hcl | 2 +- providers/do/template_versions.tftest.hcl | 4 ++-- providers/linode/template_versions.tftest.hcl | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/digitalocean/runtime_inputs.tftest.hcl b/modules/digitalocean/runtime_inputs.tftest.hcl index e781ac0..d3caac9 100644 --- a/modules/digitalocean/runtime_inputs.tftest.hcl +++ b/modules/digitalocean/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "digitalocean" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index c377739..7313f36 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "cloudinit" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } @@ -567,7 +567,7 @@ run "renders_verified_archive_before_downstream_overlay" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } @@ -752,7 +752,7 @@ run "rejects_archive_without_checksum" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } diff --git a/modules/linode/runtime_inputs.tftest.hcl b/modules/linode/runtime_inputs.tftest.hcl index 2979534..0b7b594 100644 --- a/modules/linode/runtime_inputs.tftest.hcl +++ b/modules/linode/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "linode" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index 1a7bd3d..4a0bc89 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -1,7 +1,7 @@ mock_provider "http" { mock_data "http" { defaults = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index a43780d..c4ff5b5 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } @@ -52,7 +52,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index f096071..a76d3cb 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } @@ -57,7 +57,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "20732b7e37bab43eb1b7478305e242f6b3ccced09669d646b05ef656284cfab1\n" + response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" status_code = 200 } } From ed1e716ac377511e310d7d4c75321619ff40fa05 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 10:35:52 +0000 Subject: [PATCH 53/67] [patch] Make GCP smoke zone selectable --- .github/workflows/cloud-smoke-cleanup.yml | 1 + .github/workflows/cloud-smoke.yml | 1 + docs/runtime-contracts.md | 8 ++++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cloud-smoke-cleanup.yml b/.github/workflows/cloud-smoke-cleanup.yml index 6cdb48c..f371544 100644 --- a/.github/workflows/cloud-smoke-cleanup.yml +++ b/.github/workflows/cloud-smoke-cleanup.yml @@ -100,6 +100,7 @@ jobs: GCLOUD_OIDC_POOL: ${{ vars.GCLOUD_OIDC_POOL || secrets.GCLOUD_OIDC_POOL }} GCLOUD_PROJECT: ${{ vars.GCLOUD_PROJECT || secrets.GCLOUD_PROJECT }} GCLOUD_REGION: ${{ vars.GCLOUD_REGION || secrets.GCLOUD_REGION }} + GCLOUD_ZONE: ${{ vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }} GSA: ${{ vars.GSA || secrets.GSA }} steps: - name: Checkout repository diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml index cd10e02..37946e3 100644 --- a/.github/workflows/cloud-smoke.yml +++ b/.github/workflows/cloud-smoke.yml @@ -218,6 +218,7 @@ jobs: GCLOUD_OIDC_POOL: ${{ vars.GCLOUD_OIDC_POOL || secrets.GCLOUD_OIDC_POOL }} GCLOUD_PROJECT: ${{ vars.GCLOUD_PROJECT || secrets.GCLOUD_PROJECT }} GCLOUD_REGION: ${{ vars.GCLOUD_REGION || secrets.GCLOUD_REGION }} + GCLOUD_ZONE: ${{ vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }} GCLOUD_NETWORK_PROJECT_ID: ${{ vars.GCLOUD_NETWORK_PROJECT_ID || secrets.GCLOUD_NETWORK_PROJECT_ID }} GCLOUD_NETWORK_NAME: ${{ vars.GCLOUD_NETWORK_NAME || secrets.GCLOUD_NETWORK_NAME }} GCLOUD_SUBNETWORK_NAME: ${{ vars.GCLOUD_SUBNETWORK_NAME || secrets.GCLOUD_SUBNETWORK_NAME }} diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index f93e60e..b5e7329 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -994,11 +994,15 @@ convention: | --- | --- | --- | | DigitalOcean create/test/destroy | `cloud-smoke-digitalocean` | `DIGITALOCEAN_TOKEN` | | Linode create/test/destroy | `cloud-smoke-linode` | `LINODE_TOKEN` | -| GCP create/test/destroy | `cloud-smoke-gcp` | `GCLOUD_OIDC_POOL`, `GSA`, `GCLOUD_PROJECT`, optional `GCLOUD_REGION` | +| GCP create/test/destroy | `cloud-smoke-gcp` | `GCLOUD_OIDC_POOL`, `GSA`, `GCLOUD_PROJECT`, optional `GCLOUD_REGION` and `GCLOUD_ZONE` | | GCP major-version upgrade | `cloud-smoke-gcp` | the GCP values above plus `GCLOUD_NETWORK_PROJECT_ID`, `GCLOUD_NETWORK_NAME`, `GCLOUD_SUBNETWORK_NAME`, `GCLOUD_POWER_START_ROLE`, and `GCLOUD_POWER_SUSPEND_ROLE` | | DigitalOcean fallback deletion | `cloud-smoke-cleanup-digitalocean` | a distinct cleanup-only `DIGITALOCEAN_TOKEN` | | Linode fallback deletion | `cloud-smoke-cleanup-linode` | a distinct cleanup-only `LINODE_TOKEN` | -| GCP fallback deletion | `cloud-smoke-cleanup-gcp` | cleanup-specific `GCLOUD_OIDC_POOL`, `GSA`, `GCLOUD_PROJECT`, optional `GCLOUD_REGION` | +| GCP fallback deletion | `cloud-smoke-cleanup-gcp` | cleanup-specific `GCLOUD_OIDC_POOL`, `GSA`, `GCLOUD_PROJECT`, optional `GCLOUD_REGION` and `GCLOUD_ZONE` | + +When `GCLOUD_ZONE` is set, it must belong to `GCLOUD_REGION`. Mirror the same +zone in `cloud-smoke-gcp` and `cloud-smoke-cleanup-gcp` so fallback cleanup +targets the smoke-test location. Configure required reviewers and prevent self-review on the three `cloud-smoke-*` environments. Permit only the same-repository feature branches From fadd0ab80494ca2861e2a471d610410efcccb556 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 11:16:03 +0000 Subject: [PATCH 54/67] [patch] Isolate fresh GCP smoke location --- .github/workflows/cloud-smoke-cleanup.yml | 11 ++++++++++- .github/workflows/cloud-smoke.yml | 8 ++++++++ docs/runtime-contracts.md | 16 ++++++++++------ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cloud-smoke-cleanup.yml b/.github/workflows/cloud-smoke-cleanup.yml index f371544..3e2f515 100644 --- a/.github/workflows/cloud-smoke-cleanup.yml +++ b/.github/workflows/cloud-smoke-cleanup.yml @@ -99,6 +99,8 @@ jobs: CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.event.workflow_run.id }} GCLOUD_OIDC_POOL: ${{ vars.GCLOUD_OIDC_POOL || secrets.GCLOUD_OIDC_POOL }} GCLOUD_PROJECT: ${{ vars.GCLOUD_PROJECT || secrets.GCLOUD_PROJECT }} + GCLOUD_FRESH_REGION: ${{ vars.GCLOUD_FRESH_REGION || secrets.GCLOUD_FRESH_REGION }} + GCLOUD_FRESH_ZONE: ${{ vars.GCLOUD_FRESH_ZONE || secrets.GCLOUD_FRESH_ZONE }} GCLOUD_REGION: ${{ vars.GCLOUD_REGION || secrets.GCLOUD_REGION }} GCLOUD_ZONE: ${{ vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }} GSA: ${{ vars.GSA || secrets.GSA }} @@ -134,5 +136,12 @@ jobs: - name: Install gcloud uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3 - - name: Sweep GCP smoke resources + - name: Sweep GCP upgrade-region resources + run: ci/cloud-smoke.sh sweep-gcp-wp + + - name: Sweep GCP fresh-smoke region + if: ${{ env.GCLOUD_FRESH_REGION != '' && env.GCLOUD_FRESH_REGION != env.GCLOUD_REGION }} + env: + GCLOUD_REGION: ${{ env.GCLOUD_FRESH_REGION }} + GCLOUD_ZONE: ${{ env.GCLOUD_FRESH_ZONE }} run: ci/cloud-smoke.sh sweep-gcp-wp diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml index 37946e3..f8e1b8f 100644 --- a/.github/workflows/cloud-smoke.yml +++ b/.github/workflows/cloud-smoke.yml @@ -217,6 +217,8 @@ jobs: CLOUD_COMPOSE_UPGRADE_CURRENT_REF: ${{ github.sha }} GCLOUD_OIDC_POOL: ${{ vars.GCLOUD_OIDC_POOL || secrets.GCLOUD_OIDC_POOL }} GCLOUD_PROJECT: ${{ vars.GCLOUD_PROJECT || secrets.GCLOUD_PROJECT }} + GCLOUD_FRESH_REGION: ${{ vars.GCLOUD_FRESH_REGION || secrets.GCLOUD_FRESH_REGION }} + GCLOUD_FRESH_ZONE: ${{ vars.GCLOUD_FRESH_ZONE || secrets.GCLOUD_FRESH_ZONE }} GCLOUD_REGION: ${{ vars.GCLOUD_REGION || secrets.GCLOUD_REGION }} GCLOUD_ZONE: ${{ vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }} GCLOUD_NETWORK_PROJECT_ID: ${{ vars.GCLOUD_NETWORK_PROJECT_ID || secrets.GCLOUD_NETWORK_PROJECT_ID }} @@ -278,6 +280,9 @@ jobs: - name: Run fresh smoke test if: ${{ !startsWith(github.event.pull_request.title, '[major]') }} + env: + GCLOUD_REGION: ${{ vars.GCLOUD_FRESH_REGION || secrets.GCLOUD_FRESH_REGION || vars.GCLOUD_REGION || secrets.GCLOUD_REGION }} + GCLOUD_ZONE: ${{ vars.GCLOUD_FRESH_ZONE || secrets.GCLOUD_FRESH_ZONE || vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }} run: make smoke-test PROVIDER=gcp TEMPLATE=wp - name: Run 0.10.2 upgrade smoke test @@ -286,6 +291,9 @@ jobs: - name: Destroy fresh smoke resources if: ${{ always() && !startsWith(github.event.pull_request.title, '[major]') && env.GCLOUD_OIDC_POOL != '' && env.GSA != '' && env.GCLOUD_PROJECT != '' }} + env: + GCLOUD_REGION: ${{ vars.GCLOUD_FRESH_REGION || secrets.GCLOUD_FRESH_REGION || vars.GCLOUD_REGION || secrets.GCLOUD_REGION }} + GCLOUD_ZONE: ${{ vars.GCLOUD_FRESH_ZONE || secrets.GCLOUD_FRESH_ZONE || vars.GCLOUD_ZONE || secrets.GCLOUD_ZONE }} run: ci/cloud-smoke.sh destroy-gcp-wp - name: Destroy upgrade smoke resources diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index b5e7329..91826fd 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -994,15 +994,19 @@ convention: | --- | --- | --- | | DigitalOcean create/test/destroy | `cloud-smoke-digitalocean` | `DIGITALOCEAN_TOKEN` | | Linode create/test/destroy | `cloud-smoke-linode` | `LINODE_TOKEN` | -| GCP create/test/destroy | `cloud-smoke-gcp` | `GCLOUD_OIDC_POOL`, `GSA`, `GCLOUD_PROJECT`, optional `GCLOUD_REGION` and `GCLOUD_ZONE` | +| GCP create/test/destroy | `cloud-smoke-gcp` | `GCLOUD_OIDC_POOL`, `GSA`, `GCLOUD_PROJECT`, optional `GCLOUD_REGION` and `GCLOUD_ZONE`, and optional fresh-smoke overrides `GCLOUD_FRESH_REGION` and `GCLOUD_FRESH_ZONE` | | GCP major-version upgrade | `cloud-smoke-gcp` | the GCP values above plus `GCLOUD_NETWORK_PROJECT_ID`, `GCLOUD_NETWORK_NAME`, `GCLOUD_SUBNETWORK_NAME`, `GCLOUD_POWER_START_ROLE`, and `GCLOUD_POWER_SUSPEND_ROLE` | | DigitalOcean fallback deletion | `cloud-smoke-cleanup-digitalocean` | a distinct cleanup-only `DIGITALOCEAN_TOKEN` | | Linode fallback deletion | `cloud-smoke-cleanup-linode` | a distinct cleanup-only `LINODE_TOKEN` | -| GCP fallback deletion | `cloud-smoke-cleanup-gcp` | cleanup-specific `GCLOUD_OIDC_POOL`, `GSA`, `GCLOUD_PROJECT`, optional `GCLOUD_REGION` and `GCLOUD_ZONE` | - -When `GCLOUD_ZONE` is set, it must belong to `GCLOUD_REGION`. Mirror the same -zone in `cloud-smoke-gcp` and `cloud-smoke-cleanup-gcp` so fallback cleanup -targets the smoke-test location. +| GCP fallback deletion | `cloud-smoke-cleanup-gcp` | cleanup-specific `GCLOUD_OIDC_POOL`, `GSA`, `GCLOUD_PROJECT`, optional `GCLOUD_REGION` and `GCLOUD_ZONE`, plus any `GCLOUD_FRESH_REGION` and `GCLOUD_FRESH_ZONE` overrides used by the smoke environment | + +When `GCLOUD_ZONE` is set, it must belong to `GCLOUD_REGION`. Those values keep +the historical major-upgrade smoke aligned with its persistent subnet. A fresh +smoke can use a different location by setting `GCLOUD_FRESH_REGION` and +`GCLOUD_FRESH_ZONE`; the latter must belong to the former. Mirror both location +pairs in `cloud-smoke-gcp` and `cloud-smoke-cleanup-gcp`. The fallback first +sweeps the upgrade region, then also sweeps the fresh-smoke region when it is +different, so region-scoped Cloud Run resources cannot be stranded. Configure required reviewers and prevent self-review on the three `cloud-smoke-*` environments. Permit only the same-repository feature branches From 908b8befe8f242cd5d4802e96a2765f09dc72c56 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 11:54:34 +0000 Subject: [PATCH 55/67] [patch] Run built-in lifecycle on noexec hosts --- ci/compose-runtime-contract.sh | 12 ++++++++++++ docs/runtime-contracts.md | 8 +++++++- .../cloud-compose/libexec/run-lifecycle-program.sh | 10 ++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index 6ecd144..cbaab62 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -59,6 +59,18 @@ grep -Fq 'run_compose_lifecycle_executor --validate "$lifecycle" "$command" || r if grep -Fq 'bash -c "$command"' "$repo_root/rootfs/home/cloud-compose/compose-apps.sh"; then fail "Compose lifecycle entries still execute as shell strings" fi +lifecycle_executor="$repo_root/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh" +grep -Fq 'if [[ "$program" == "/home/cloud-compose/default-lifecycle.sh" ]]; then' \ + "$lifecycle_executor" || \ + fail "checked executor does not isolate the built-in lifecycle program" +grep -Fq 'exec /bin/bash -- "$program" "${program_args[@]}"' \ + "$lifecycle_executor" || \ + fail "checked executor cannot open the built-in lifecycle program from a noexec home mount" +grep -Fq 'exec "$program" "${program_args[@]}"' "$lifecycle_executor" || \ + fail "checked executor does not preserve direct execution for custom lifecycle programs" +if grep -Eq '/bin/bash[[:space:]]+-[^[:space:]]*c([[:space:]]|$)' "$lifecycle_executor"; then + fail "checked executor evaluates a lifecycle entry as shell source" +fi export COMPOSE_PROJECTS_FILE="$tmp/compose-projects.json" export COMPOSE_APPS_ENV_DIR="$tmp/apps" diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 91826fd..e79c635 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -237,7 +237,13 @@ argument-free, root-controlled program immediately below environment. `true` and `false` remain explicit no-op and failure sentinels. Multi-step work, state such as local variables or traps, and any quoting belong inside that reviewed program file. The constrained executor invokes its argv -directly and never evaluates a manifest value with a shell parser. +without evaluating a manifest value as shell source. Custom programs retain +direct execution. After the same root-owner, link-count, parent-directory, and +mode checks, the built-in script is opened as +`/bin/bash -- /home/cloud-compose/default-lifecycle.sh ACTION`; this fixed +interpreter path keeps the built-in lifecycle usable on Container-Optimized OS, +where `/home` is deliberately mounted `noexec`, without turning the manifest +entry into a shell command string. ## Vault diff --git a/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh b/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh index 156001b..bc7b557 100755 --- a/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh +++ b/rootfs/etc/cloud-compose/libexec/run-lifecycle-program.sh @@ -80,4 +80,14 @@ if [[ "$validate_only" == "true" ]]; then exit 0 fi +# Container-Optimized OS mounts /home with noexec. The built-in lifecycle +# program is still an unlinked, root-owned regular file below the checked +# root-owned home boundary, so open it through the fixed system interpreter +# after validation instead of asking the kernel to execute it from that mount. +# Keep custom programs on the direct-exec path so their reviewed interpreter or +# binary contract is preserved. No manifest value is evaluated as shell source. +if [[ "$program" == "/home/cloud-compose/default-lifecycle.sh" ]]; then + exec /bin/bash -- "$program" "${program_args[@]}" +fi + exec "$program" "${program_args[@]}" From 1b1872be88043cf25ce4872da7451a3c9bce405c Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:01:15 +0000 Subject: [PATCH 56/67] [patch] Repin lifecycle rootfs contract fixtures --- modules/digitalocean/runtime_inputs.tftest.hcl | 2 +- modules/gcp/runtime_contracts.tftest.hcl | 6 +++--- modules/linode/runtime_inputs.tftest.hcl | 2 +- modules/linux-vm-runtime/runtime_inputs.tftest.hcl | 2 +- providers/do/template_versions.tftest.hcl | 4 ++-- providers/linode/template_versions.tftest.hcl | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/digitalocean/runtime_inputs.tftest.hcl b/modules/digitalocean/runtime_inputs.tftest.hcl index d3caac9..7781b12 100644 --- a/modules/digitalocean/runtime_inputs.tftest.hcl +++ b/modules/digitalocean/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "digitalocean" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index 7313f36..dbacaf8 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "cloudinit" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } @@ -567,7 +567,7 @@ run "renders_verified_archive_before_downstream_overlay" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } @@ -752,7 +752,7 @@ run "rejects_archive_without_checksum" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } diff --git a/modules/linode/runtime_inputs.tftest.hcl b/modules/linode/runtime_inputs.tftest.hcl index 0b7b594..2b43b40 100644 --- a/modules/linode/runtime_inputs.tftest.hcl +++ b/modules/linode/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "linode" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index 4a0bc89..1e9c388 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -1,7 +1,7 @@ mock_provider "http" { mock_data "http" { defaults = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index c4ff5b5..e610cde 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } @@ -52,7 +52,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index a76d3cb..ac54aa5 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } @@ -57,7 +57,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "fb6105bfdc7ecf37c7eb84cf5de7c4c513dd7a3087b40742cd7fe0dab18fe255\n" + response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" status_code = 200 } } From 2698a1231f70ccf6517cd28c3195f68ec2c3ebd4 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:22:36 +0000 Subject: [PATCH 57/67] [patch] Use executable Compose plugin for diagnostics --- ci/systemd-contract.sh | 1 + docs/runtime-contracts.md | 5 ++++- rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh | 7 ++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ci/systemd-contract.sh b/ci/systemd-contract.sh index f95bb49..0236e66 100755 --- a/ci/systemd-contract.sh +++ b/ci/systemd-contract.sh @@ -81,6 +81,7 @@ assert_contains "$root_program_runner" 'Unsupported Cloud Compose root program:' fail "validated bootstrap readiness gate is missing or not executable" assert_contains "$diagnostics_program" '--- Cloud Compose provisioning heartbeat ---' assert_contains "$diagnostics_program" 'ps -p "$main_pid" -o pid=,ppid=,stat=,etime=,comm=' +assert_contains "$diagnostics_program" 'DOCKER_CONFIG=/mnt/disks/data/docker-config' assert_contains "$smoke_healthcheck_program" 'source /home/cloud-compose/profile.sh' assert_contains "$smoke_healthcheck_program" 'exec sitectl healthcheck --context "$context" --persist --format table' assert_contains "$unit_dir/cloud-compose-mariadb-backup.service" 'TimeoutStartSec=12h' diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index e79c635..2b66939 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -175,7 +175,10 @@ allowlisted `/home/cloud-compose` program. The privileged entrypoints, diagnostics command, and checked-in `jq` programs live below `/etc/cloud-compose/{libexec,bin,jq}` because COS permits cloud-init to rebuild that stateless tree while its `/usr` filesystem is immutable. Application -services retain their unprivileged execution model. +services retain their unprivileged execution model. Compose diagnostics run as +the application account and select the verified plugin copy below the +executable data disk explicitly; they do not fall back to the compatibility +copy below COS's `noexec` `/home` mount. Terraform and cloud-init are transport and orchestration boundaries, not the home of bootstrap implementations. The GCP and provider-neutral templates diff --git a/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh b/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh index 185b24b..ddbb352 100755 --- a/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh +++ b/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh @@ -163,7 +163,12 @@ dump_compose_state() { continue fi echo "--- docker compose ps: ${app} ---" - runuser -u cloud-compose -- env HOME=/home/cloud-compose \ + # COS mounts /home with noexec. Select the verified plugin copy on the + # executable data disk instead of allowing Docker to discover the + # compatibility copy below /home/cloud-compose/.docker. + runuser -u cloud-compose -- env \ + HOME=/home/cloud-compose \ + DOCKER_CONFIG=/mnt/disks/data/docker-config \ "$docker_path" compose --project-directory "$project_dir" ps 2>&1 || true done < <(jq -r -f "$jq_program_dir/diagnostics-project-entries.jq" "$manifest") } From 4627b00128d8fb1238f0c766c7e104e779ea8785 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:32:21 +0000 Subject: [PATCH 58/67] [patch] Repin diagnostic rootfs contract fixtures --- modules/digitalocean/runtime_inputs.tftest.hcl | 2 +- modules/gcp/runtime_contracts.tftest.hcl | 6 +++--- modules/linode/runtime_inputs.tftest.hcl | 2 +- modules/linux-vm-runtime/runtime_inputs.tftest.hcl | 2 +- providers/do/template_versions.tftest.hcl | 4 ++-- providers/linode/template_versions.tftest.hcl | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/digitalocean/runtime_inputs.tftest.hcl b/modules/digitalocean/runtime_inputs.tftest.hcl index 7781b12..d77207b 100644 --- a/modules/digitalocean/runtime_inputs.tftest.hcl +++ b/modules/digitalocean/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "digitalocean" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index dbacaf8..9d60c72 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "cloudinit" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } @@ -567,7 +567,7 @@ run "renders_verified_archive_before_downstream_overlay" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } @@ -752,7 +752,7 @@ run "rejects_archive_without_checksum" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } diff --git a/modules/linode/runtime_inputs.tftest.hcl b/modules/linode/runtime_inputs.tftest.hcl index 2b43b40..69818ef 100644 --- a/modules/linode/runtime_inputs.tftest.hcl +++ b/modules/linode/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "linode" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index 1e9c388..3333cb8 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -1,7 +1,7 @@ mock_provider "http" { mock_data "http" { defaults = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index e610cde..a535fba 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } @@ -52,7 +52,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index ac54aa5..d06bee4 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } @@ -57,7 +57,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "91e0257c5867a0f5084ffb1820df585b23502e4f5b66c91e8bb704565abe6ad4\n" + response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" status_code = 200 } } From bf363ceb3c8452345cdcbbff61b62fc646a67f9e Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:49:55 +0000 Subject: [PATCH 59/67] [patch] Stage hosted contracts on executable disk --- ci/cloud-smoke.sh | 19 +++++++++++++------ ci/compose-runtime-contract.sh | 10 ++++++++++ docs/runtime-contracts.md | 4 ++++ .../testdata/cloud-smoke-lifecycle/ssh.sh | 4 ++-- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh index 04a84c0..aefc697 100755 --- a/ci/cloud-smoke.sh +++ b/ci/cloud-smoke.sh @@ -606,9 +606,12 @@ run_lifecycle_program_contract() { host="$(jq -r '.host' "$output_json")" port="$(jq -r '.ssh_port' "$output_json")" user="$(jq -r '.ssh_user' "$output_json")" - remote_contract_dir="$(ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ - 'mktemp -d /tmp/cloud-compose-hosted-contract.XXXXXX')" || return 1 - if [[ ! "$remote_contract_dir" =~ ^/tmp/cloud-compose-hosted-contract\.[A-Za-z0-9]+$ ]]; then + if ! remote_contract_dir="$(ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + 'mktemp -d /mnt/disks/data/cloud-compose-hosted-contract.XXXXXX')"; then + echo "Could not create the remote lifecycle contract directory on the executable data disk" >&2 + return 1 + fi + if [[ ! "$remote_contract_dir" =~ ^/mnt/disks/data/cloud-compose-hosted-contract\.[A-Za-z0-9]+$ ]]; then echo "Remote lifecycle contract directory is unsafe: $remote_contract_dir" >&2 return 1 fi @@ -617,8 +620,9 @@ run_lifecycle_program_contract() { if ! ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ "install -m 0700 /dev/stdin $remote_contract" \ <"$repo_root/ci/lifecycle-program-contract.sh"; then + echo "Could not install the remote lifecycle program contract" >&2 ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ - "rmdir -- $remote_contract_dir" || true + "rm -f -- $remote_contract && rmdir -- $remote_contract_dir" || true return 1 fi @@ -627,9 +631,12 @@ run_lifecycle_program_contract() { status=0 else status=$? + echo "Remote lifecycle program contract failed with status $status" >&2 + fi + if ! ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ + "rm -f -- $remote_contract && rmdir -- $remote_contract_dir"; then + echo "Could not remove the remote lifecycle program contract" >&2 fi - ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ - "rm -f -- $remote_contract && rmdir -- $remote_contract_dir" || true return "$status" } diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index cbaab62..0d5d341 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -31,6 +31,16 @@ bash "$repo_root/ci/lifecycle-program-contract.sh" \ "$repo_root/rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq" grep -Fq -- 'run_lifecycle_program_contract "$home_dir" "$key_path" "$output_json"' \ "$repo_root/ci/cloud-smoke.sh" || fail "provider smoke does not execute the lifecycle program contract" +smoke_lifecycle_contract="$(sed -n '/^run_lifecycle_program_contract()/,/^}/p' "$repo_root/ci/cloud-smoke.sh")" +grep -Fq -- 'mktemp -d /mnt/disks/data/cloud-compose-hosted-contract.XXXXXX' \ + <<<"$smoke_lifecycle_contract" || \ + fail "provider smoke does not stage its executable lifecycle contract on the data disk" +grep -Fq -- '^/mnt/disks/data/cloud-compose-hosted-contract\.[A-Za-z0-9]+$' \ + <<<"$smoke_lifecycle_contract" || \ + fail "provider smoke does not validate the exact remote lifecycle contract directory" +if grep -Fq -- '/tmp/cloud-compose-hosted-contract' <<<"$smoke_lifecycle_contract"; then + fail "provider smoke still stages an executable lifecycle contract below /tmp" +fi grep -Fq -- 'bash "$lifecycle_program_contract" /home/cloud-compose/default-lifecycle.sh' \ "$repo_root/ci/remote/config-management-verify.sh" || \ fail "config-management smoke does not execute the lifecycle program contract" diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 2b66939..fcc29eb 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -1042,6 +1042,10 @@ trusted and provider-job checkouts set `persist-credentials: false`. Pull-request smoke jobs also build one binary before apply and reuse that exact workspace binary from their `always()` cleanup step. Shell remains responsible for Terraform, SSH, cloud-init, diagnostics, and host-runtime black-box checks. +The application smoke stages its checked-in lifecycle contract in a private, +strictly validated directory on the executable data disk, so its fake `sitectl` +program remains executable on COS hosts whose temporary filesystems are +`noexec`; the contract is removed immediately after the check. No privileged fallback executes a pull-request binary or downloads one as an artifact. The fallback runs automatically after a failed, cancelled, or timed-out smoke workflow, including diff --git a/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh b/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh index 4460617..4462855 100644 --- a/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh +++ b/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh @@ -3,8 +3,8 @@ set -euo pipefail case "$*" in - *mktemp\ -d\ /tmp/cloud-compose-hosted-contract.XXXXXX*) - printf '/tmp/cloud-compose-hosted-contract.fixture123\n' + *mktemp\ -d\ /mnt/disks/data/cloud-compose-hosted-contract.XXXXXX*) + printf '/mnt/disks/data/cloud-compose-hosted-contract.fixture123\n' ;; *install\ -m\ 0700\ /dev/stdin*) cat >/dev/null From e627c1c2c8098a0a0e0712e574d6bc499b6b1b0b Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:12:48 +0000 Subject: [PATCH 60/67] [patch] Avoid noexec lifecycle contract preflight --- ci/cloud-smoke.sh | 2 +- ci/compose-runtime-contract.sh | 13 +++++++++++++ ci/lifecycle-program-contract.sh | 21 ++++++++++++++++++++- docs/runtime-contracts.md | 5 ++++- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh index aefc697..ef5c654 100755 --- a/ci/cloud-smoke.sh +++ b/ci/cloud-smoke.sh @@ -627,7 +627,7 @@ run_lifecycle_program_contract() { fi if ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \ - "test -x /home/cloud-compose/default-lifecycle.sh && bash $remote_contract /home/cloud-compose/default-lifecycle.sh"; then + "/bin/bash -- $remote_contract /home/cloud-compose/default-lifecycle.sh"; then status=0 else status=$? diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index 0d5d341..d72890f 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -41,6 +41,19 @@ grep -Fq -- '^/mnt/disks/data/cloud-compose-hosted-contract\.[A-Za-z0-9]+$' \ if grep -Fq -- '/tmp/cloud-compose-hosted-contract' <<<"$smoke_lifecycle_contract"; then fail "provider smoke still stages an executable lifecycle contract below /tmp" fi +grep -Fq -- '/bin/bash -- $remote_contract /home/cloud-compose/default-lifecycle.sh' \ + <<<"$smoke_lifecycle_contract" || \ + fail "provider smoke does not open the checked-in contract through the fixed interpreter" +if grep -Fq -- 'test -x /home/cloud-compose/default-lifecycle.sh' <<<"$smoke_lifecycle_contract"; then + fail "provider smoke still rejects a lifecycle program solely because /home is noexec" +fi +lifecycle_contract="$repo_root/ci/lifecycle-program-contract.sh" +grep -Fq -- 'trap report_unexpected_failure ERR' "$lifecycle_contract" || \ + fail "lifecycle program contract does not report unexpected command failures" +grep -Fq -- '[[ -L "$lifecycle_program" ]]' "$lifecycle_contract" || \ + fail "lifecycle program contract accepts a redirected target" +grep -Fq -- '[[ ! -r "$lifecycle_program" ]]' "$lifecycle_contract" || \ + fail "lifecycle program contract accepts an unreadable target" grep -Fq -- 'bash "$lifecycle_program_contract" /home/cloud-compose/default-lifecycle.sh' \ "$repo_root/ci/remote/config-management-verify.sh" || \ fail "config-management smoke does not execute the lifecycle program contract" diff --git a/ci/lifecycle-program-contract.sh b/ci/lifecycle-program-contract.sh index 6d62224..5ddbebe 100755 --- a/ci/lifecycle-program-contract.sh +++ b/ci/lifecycle-program-contract.sh @@ -1,6 +1,17 @@ #!/usr/bin/env bash -set -euo pipefail +set -Eeuo pipefail + +report_unexpected_failure() { + local status=$? + local line="${BASH_LINENO[0]:-unknown}" + + trap - ERR + echo "lifecycle program contract: unexpected failure at line ${line} with status ${status}" >&2 + exit "$status" +} + +trap report_unexpected_failure ERR if [[ "${0##*/}" == "sitectl" ]]; then : "${SITECTL_ARGV_LOG:?SITECTL_ARGV_LOG is required}" @@ -17,10 +28,18 @@ fi lifecycle_program="${1:-/home/cloud-compose/default-lifecycle.sh}" verify_args_program="${2:-/etc/cloud-compose/jq/sitectl-verify-args.jq}" +if [[ -L "$lifecycle_program" ]]; then + echo "lifecycle program contract: target is redirected: $lifecycle_program" >&2 + exit 1 +fi if [[ ! -f "$lifecycle_program" ]]; then echo "lifecycle program contract: missing $lifecycle_program" >&2 exit 1 fi +if [[ ! -r "$lifecycle_program" ]]; then + echo "lifecycle program contract: target is unreadable: $lifecycle_program" >&2 + exit 1 +fi contract_dir="$(mktemp -d "${TMPDIR:-/tmp}/cloud-compose-lifecycle-program.XXXXXX")" trap 'rm -rf -- "$contract_dir"' EXIT diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index fcc29eb..e014e07 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -1045,7 +1045,10 @@ for Terraform, SSH, cloud-init, diagnostics, and host-runtime black-box checks. The application smoke stages its checked-in lifecycle contract in a private, strictly validated directory on the executable data disk, so its fake `sitectl` program remains executable on COS hosts whose temporary filesystems are -`noexec`; the contract is removed immediately after the check. +`noexec`. The smoke driver opens that checked-in contract through fixed +`/bin/bash`; the contract rejects a missing, redirected, or unreadable lifecycle +target without using an execute-access probe that fails on COS's `noexec` +`/home`. The contract is removed immediately after the check. No privileged fallback executes a pull-request binary or downloads one as an artifact. The fallback runs automatically after a failed, cancelled, or timed-out smoke workflow, including From e4e7372282e50d549c8819ca9f8a87438dff8463 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:06:11 +0000 Subject: [PATCH 61/67] [patch] Pin final sitectl release catalog --- docs/examples.md | 10 ++--- docs/managed-runtime.md | 19 ++++++---- examples/binpack/main.tf | 6 +-- .../contracttest/template_version_test.go | 38 +++++++++---------- providers/do/template_versions.tftest.hcl | 2 +- providers/gcp/template_versions.tftest.hcl | 2 +- providers/linode/template_versions.tftest.hcl | 2 +- templates/apps.json | 32 ++++++++-------- 8 files changed, 57 insertions(+), 54 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 6cd9e1c..fb4f74c 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -136,8 +136,8 @@ healthcheck settings: runtime = { sitectl = { package_versions = { - sitectl = "v1.8.2" - sitectl-wp = "v2.0.0" + sitectl = "v1.9.0" + sitectl-wp = "v2.1.0" } } compose = { @@ -182,9 +182,9 @@ runtime = { } sitectl = { package_versions = { - sitectl = "v1.8.2" - sitectl-wp = "v2.0.0" - sitectl-drupal = "v1.3.0" + sitectl = "v1.9.0" + sitectl-wp = "v2.1.0" + sitectl-drupal = "v1.5.0" } } } diff --git a/docs/managed-runtime.md b/docs/managed-runtime.md index 1bd971c..fdce740 100644 --- a/docs/managed-runtime.md +++ b/docs/managed-runtime.md @@ -26,11 +26,14 @@ core and plugins independently. The provider-neutral backward-compatible fallback only for an installed package with neither a template selector nor an explicit per-package selector. -All presets pin sitectl v1.8.2 with a reviewed application plugin. ISLE uses -the `libops/isle` v1.3.1 template with sitectl-drupal v1.3.0 and sitectl-isle -v1.5.0. ArchivesSpace uses v1.0.1, WordPress uses v1.1.1, and Drupal, OJS, -Omeka Classic, and Omeka S use v1.2.1. Override template or package selectors -only as one reviewed, compatible release set. +All presets pin sitectl v1.9.0 with reviewed application plugins: +sitectl-archivesspace v2.1.1, sitectl-drupal v1.5.0, sitectl-isle v1.6.0, +sitectl-ojs v1.4.0, sitectl-omeka-classic v1.4.0, sitectl-omeka-s v1.4.0, +and sitectl-wp v2.1.0. ISLE includes both the Drupal and ISLE plugins. The +Compose template branches remain independently pinned: ISLE uses v1.3.1, +ArchivesSpace uses v1.0.1, WordPress uses v1.1.1, and Drupal, OJS, Omeka +Classic, and Omeka S use v1.2.1. Override template or package selectors only +as one reviewed, compatible release set. Omitting `runtime.sitectl.packages` selects the template's package set. An explicit list replaces that set; `packages = []` or `packages = ["sitectl"]` @@ -51,9 +54,9 @@ runtime = { "sitectl-isle", ] package_versions = { - sitectl = "v1.8.2" - sitectl-drupal = "v1.3.0" - sitectl-isle = "v1.5.0" + sitectl = "v1.9.0" + sitectl-drupal = "v1.5.0" + sitectl-isle = "v1.6.0" } plugin = "isle" } diff --git a/examples/binpack/main.tf b/examples/binpack/main.tf index eefa061..f22d15f 100644 --- a/examples/binpack/main.tf +++ b/examples/binpack/main.tf @@ -29,9 +29,9 @@ module "apps" { } sitectl = { package_versions = { - sitectl = "v1.8.2" - sitectl-wp = "v2.0.0" - sitectl-drupal = "v1.3.0" + sitectl = "v1.9.0" + sitectl-wp = "v2.1.0" + sitectl-drupal = "v1.5.0" } } } diff --git a/internal/contracttest/template_version_test.go b/internal/contracttest/template_version_test.go index 2e79cce..92d1215 100644 --- a/internal/contracttest/template_version_test.go +++ b/internal/contracttest/template_version_test.go @@ -30,36 +30,36 @@ func TestTemplateVersionContract(t *testing.T) { expectedVersions := map[string]map[string]string{ "default": { - "sitectl": "v1.8.2", + "sitectl": "v1.9.0", }, "archivesspace": { - "sitectl": "v1.8.2", - "sitectl-archivesspace": "v2.0.0", + "sitectl": "v1.9.0", + "sitectl-archivesspace": "v2.1.1", }, "drupal": { - "sitectl": "v1.8.2", - "sitectl-drupal": "v1.3.0", + "sitectl": "v1.9.0", + "sitectl-drupal": "v1.5.0", }, "isle": { - "sitectl": "v1.8.2", - "sitectl-drupal": "v1.3.0", - "sitectl-isle": "v1.5.0", + "sitectl": "v1.9.0", + "sitectl-drupal": "v1.5.0", + "sitectl-isle": "v1.6.0", }, "ojs": { - "sitectl": "v1.8.2", - "sitectl-ojs": "v1.2.0", + "sitectl": "v1.9.0", + "sitectl-ojs": "v1.4.0", }, "omeka-classic": { - "sitectl": "v1.8.2", - "sitectl-omeka-classic": "v1.2.0", + "sitectl": "v1.9.0", + "sitectl-omeka-classic": "v1.4.0", }, "omeka-s": { - "sitectl": "v1.8.2", - "sitectl-omeka-s": "v1.2.0", + "sitectl": "v1.9.0", + "sitectl-omeka-s": "v1.4.0", }, "wp": { - "sitectl": "v1.8.2", - "sitectl-wp": "v2.0.0", + "sitectl": "v1.9.0", + "sitectl-wp": "v2.1.0", }, } expectedBranches := map[string]string{ @@ -146,8 +146,8 @@ func TestTemplateVersionContract(t *testing.T) { for _, packageName := range []string{"sitectl", "sitectl-wp", "sitectl-drupal"} { requireContains(t, content, packageName, relativePath+" bin-pack package") } - requireContains(t, content, `sitectl = "v1.8.2"`, relativePath+" bin-pack core selector") - requireContains(t, content, `sitectl-wp = "v2.0.0"`, relativePath+" bin-pack WordPress selector") - requireContains(t, content, `sitectl-drupal = "v1.3.0"`, relativePath+" bin-pack Drupal selector") + requireContains(t, content, `sitectl = "v1.9.0"`, relativePath+" bin-pack core selector") + requireContains(t, content, `sitectl-wp = "v2.1.0"`, relativePath+" bin-pack WordPress selector") + requireContains(t, content, `sitectl-drupal = "v1.5.0"`, relativePath+" bin-pack Drupal selector") } } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index a535fba..f0bc663 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -71,7 +71,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == { - sitectl = "v1.8.2" + sitectl = "v1.9.0" } error_message = "The DigitalOcean entrypoint must preserve an explicit core-only package set." } diff --git a/providers/gcp/template_versions.tftest.hcl b/providers/gcp/template_versions.tftest.hcl index 6b9d232..f06698b 100644 --- a/providers/gcp/template_versions.tftest.hcl +++ b/providers/gcp/template_versions.tftest.hcl @@ -67,7 +67,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == { - sitectl = "v1.8.2" + sitectl = "v1.9.0" } error_message = "The GCP entrypoint must preserve an explicit core-only package set." } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index d06bee4..6c440dc 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -81,7 +81,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == { - sitectl = "v1.8.2" + sitectl = "v1.9.0" } error_message = "The Linode entrypoint must preserve an explicit core-only package set." } diff --git a/templates/apps.json b/templates/apps.json index 38a6a32..7fc4b13 100644 --- a/templates/apps.json +++ b/templates/apps.json @@ -6,7 +6,7 @@ "packages": ["sitectl"], "extra_env": {}, "package_versions": { - "sitectl": "v1.8.2" + "sitectl": "v1.9.0" } }, "templates": { @@ -17,8 +17,8 @@ "packages": ["sitectl", "sitectl-archivesspace"], "extra_env": {}, "package_versions": { - "sitectl": "v1.8.2", - "sitectl-archivesspace": "v2.0.0" + "sitectl": "v1.9.0", + "sitectl-archivesspace": "v2.1.1" } }, "ojs": { @@ -28,8 +28,8 @@ "packages": ["sitectl", "sitectl-ojs"], "extra_env": {}, "package_versions": { - "sitectl": "v1.8.2", - "sitectl-ojs": "v1.2.0" + "sitectl": "v1.9.0", + "sitectl-ojs": "v1.4.0" } }, "isle": { @@ -41,9 +41,9 @@ "ISLANDORA_TAG": "6.3.19" }, "package_versions": { - "sitectl": "v1.8.2", - "sitectl-drupal": "v1.3.0", - "sitectl-isle": "v1.5.0" + "sitectl": "v1.9.0", + "sitectl-drupal": "v1.5.0", + "sitectl-isle": "v1.6.0" } }, "drupal": { @@ -53,8 +53,8 @@ "packages": ["sitectl", "sitectl-drupal"], "extra_env": {}, "package_versions": { - "sitectl": "v1.8.2", - "sitectl-drupal": "v1.3.0" + "sitectl": "v1.9.0", + "sitectl-drupal": "v1.5.0" } }, "wp": { @@ -64,8 +64,8 @@ "packages": ["sitectl", "sitectl-wp"], "extra_env": {}, "package_versions": { - "sitectl": "v1.8.2", - "sitectl-wp": "v2.0.0" + "sitectl": "v1.9.0", + "sitectl-wp": "v2.1.0" } }, "omeka-s": { @@ -75,8 +75,8 @@ "packages": ["sitectl", "sitectl-omeka-s"], "extra_env": {}, "package_versions": { - "sitectl": "v1.8.2", - "sitectl-omeka-s": "v1.2.0" + "sitectl": "v1.9.0", + "sitectl-omeka-s": "v1.4.0" } }, "omeka-classic": { @@ -86,8 +86,8 @@ "packages": ["sitectl", "sitectl-omeka-classic"], "extra_env": {}, "package_versions": { - "sitectl": "v1.8.2", - "sitectl-omeka-classic": "v1.2.0" + "sitectl": "v1.9.0", + "sitectl-omeka-classic": "v1.4.0" } } } From 94aba41941ede2dec46a05e2cfaa5dda0e0d7bbc Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:10:54 +0000 Subject: [PATCH 62/67] [patch] Align release catalog contract fixtures --- template_versions.tftest.hcl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/template_versions.tftest.hcl b/template_versions.tftest.hcl index 57fac94..972b920 100644 --- a/template_versions.tftest.hcl +++ b/template_versions.tftest.hcl @@ -35,7 +35,7 @@ run "default_template_uses_v1_core" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.8.2" + sitectl = "v1.9.0" } error_message = "The default template must select the released sitectl v1 core." } @@ -63,8 +63,8 @@ run "non_isle_template_uses_v1_release_set" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.8.2" - sitectl-wp = "v2.0.0" + sitectl = "v1.9.0" + sitectl-wp = "v2.1.0" } error_message = "Non-ISLE templates must select their coordinated sitectl v1 release set." } @@ -92,9 +92,9 @@ run "isle_template_uses_v1_release_set" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.8.2" - sitectl-drupal = "v1.3.0" - sitectl-isle = "v1.5.0" + sitectl = "v1.9.0" + sitectl-drupal = "v1.5.0" + sitectl-isle = "v1.6.0" } error_message = "The ISLE template must select its coordinated sitectl v1 release set by default." } @@ -166,7 +166,7 @@ run "explicit_package_versions_override_template_defaults" { assert { condition = local.sitectl.package_versions == { sitectl = "v0.40.1" - sitectl-drupal = "v1.3.0" + sitectl-drupal = "v1.5.0" sitectl-isle = "v0.19.1" } error_message = "Explicit per-package selectors must override only their matching template defaults." @@ -196,7 +196,7 @@ run "custom_package_set_filters_template_versions" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.8.2" + sitectl = "v1.9.0" sitectl-wp = "v0.6.1" } error_message = "Template selectors for packages omitted by a custom package set must not reach the runtime." @@ -223,7 +223,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.sitectl.packages == tolist(["sitectl"]) && local.sitectl.package_versions == { - sitectl = "v1.8.2" + sitectl = "v1.9.0" } error_message = "An explicit core-only package set must not be mistaken for an omitted template package selection." } From 9a3c8f68286801e1941d3b39bbc7801eb3f9c1c8 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:17:46 +0000 Subject: [PATCH 63/67] [patch] Align inherited config catalog assertions --- README.md | 2 +- ci/config-management-smoke-assert.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1398cd6..adeebd3 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Renovate tracks both the Compose tags and sitectl release tags in the registry. Its pull requests are promotion candidates: keep the complete compatible set together and require the hosted application smoke before merging them. -All presets pin sitectl v1.8.2 plus the latest reviewed application plugin. +All presets pin sitectl v1.9.0 plus the latest reviewed application plugin. ISLE selects the `libops/isle` v1.3.1 template, ArchivesSpace selects v1.0.1, WordPress selects v1.1.1, and Drupal, OJS, Omeka Classic, and Omeka S select v1.2.1. Keep each preset's complete template and package set together when diff --git a/ci/config-management-smoke-assert.py b/ci/config-management-smoke-assert.py index f229946..82c999f 100644 --- a/ci/config-management-smoke-assert.py +++ b/ci/config-management-smoke-assert.py @@ -300,8 +300,8 @@ def assert_salt_runtime( assert not Path("/tmp/cloud-compose-salt-command-injection").exists() elif expected_name == "drupal-prod": assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { - "sitectl": "v1.8.2", - "sitectl-drupal": "v1.3.0", + "sitectl": "v1.9.0", + "sitectl-drupal": "v1.5.0", } assert_runtime_files() From 52f1e22f735bebc049f32cfb5c34b758e6551d0b Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:58:07 +0000 Subject: [PATCH 64/67] [patch] Pin sitectl ownership fix release --- README.md | 2 +- ci/config-management-smoke-assert.py | 2 +- docs/examples.md | 4 ++-- docs/managed-runtime.md | 4 ++-- examples/binpack/main.tf | 2 +- internal/contracttest/template_version_test.go | 18 +++++++++--------- providers/do/template_versions.tftest.hcl | 2 +- providers/gcp/template_versions.tftest.hcl | 2 +- providers/linode/template_versions.tftest.hcl | 2 +- template_versions.tftest.hcl | 10 +++++----- templates/apps.json | 16 ++++++++-------- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index adeebd3..4fc7397 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Renovate tracks both the Compose tags and sitectl release tags in the registry. Its pull requests are promotion candidates: keep the complete compatible set together and require the hosted application smoke before merging them. -All presets pin sitectl v1.9.0 plus the latest reviewed application plugin. +All presets pin sitectl v1.9.1 plus the latest reviewed application plugin. ISLE selects the `libops/isle` v1.3.1 template, ArchivesSpace selects v1.0.1, WordPress selects v1.1.1, and Drupal, OJS, Omeka Classic, and Omeka S select v1.2.1. Keep each preset's complete template and package set together when diff --git a/ci/config-management-smoke-assert.py b/ci/config-management-smoke-assert.py index 82c999f..fdbfe32 100644 --- a/ci/config-management-smoke-assert.py +++ b/ci/config-management-smoke-assert.py @@ -300,7 +300,7 @@ def assert_salt_runtime( assert not Path("/tmp/cloud-compose-salt-command-injection").exists() elif expected_name == "drupal-prod": assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-drupal": "v1.5.0", } diff --git a/docs/examples.md b/docs/examples.md index fb4f74c..df15d80 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -136,7 +136,7 @@ healthcheck settings: runtime = { sitectl = { package_versions = { - sitectl = "v1.9.0" + sitectl = "v1.9.1" sitectl-wp = "v2.1.0" } } @@ -182,7 +182,7 @@ runtime = { } sitectl = { package_versions = { - sitectl = "v1.9.0" + sitectl = "v1.9.1" sitectl-wp = "v2.1.0" sitectl-drupal = "v1.5.0" } diff --git a/docs/managed-runtime.md b/docs/managed-runtime.md index fdce740..44f4e67 100644 --- a/docs/managed-runtime.md +++ b/docs/managed-runtime.md @@ -26,7 +26,7 @@ core and plugins independently. The provider-neutral backward-compatible fallback only for an installed package with neither a template selector nor an explicit per-package selector. -All presets pin sitectl v1.9.0 with reviewed application plugins: +All presets pin sitectl v1.9.1 with reviewed application plugins: sitectl-archivesspace v2.1.1, sitectl-drupal v1.5.0, sitectl-isle v1.6.0, sitectl-ojs v1.4.0, sitectl-omeka-classic v1.4.0, sitectl-omeka-s v1.4.0, and sitectl-wp v2.1.0. ISLE includes both the Drupal and ISLE plugins. The @@ -54,7 +54,7 @@ runtime = { "sitectl-isle", ] package_versions = { - sitectl = "v1.9.0" + sitectl = "v1.9.1" sitectl-drupal = "v1.5.0" sitectl-isle = "v1.6.0" } diff --git a/examples/binpack/main.tf b/examples/binpack/main.tf index f22d15f..c709108 100644 --- a/examples/binpack/main.tf +++ b/examples/binpack/main.tf @@ -29,7 +29,7 @@ module "apps" { } sitectl = { package_versions = { - sitectl = "v1.9.0" + sitectl = "v1.9.1" sitectl-wp = "v2.1.0" sitectl-drupal = "v1.5.0" } diff --git a/internal/contracttest/template_version_test.go b/internal/contracttest/template_version_test.go index 92d1215..4b7245c 100644 --- a/internal/contracttest/template_version_test.go +++ b/internal/contracttest/template_version_test.go @@ -30,35 +30,35 @@ func TestTemplateVersionContract(t *testing.T) { expectedVersions := map[string]map[string]string{ "default": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", }, "archivesspace": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-archivesspace": "v2.1.1", }, "drupal": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-drupal": "v1.5.0", }, "isle": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-drupal": "v1.5.0", "sitectl-isle": "v1.6.0", }, "ojs": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-ojs": "v1.4.0", }, "omeka-classic": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-omeka-classic": "v1.4.0", }, "omeka-s": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-omeka-s": "v1.4.0", }, "wp": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-wp": "v2.1.0", }, } @@ -146,7 +146,7 @@ func TestTemplateVersionContract(t *testing.T) { for _, packageName := range []string{"sitectl", "sitectl-wp", "sitectl-drupal"} { requireContains(t, content, packageName, relativePath+" bin-pack package") } - requireContains(t, content, `sitectl = "v1.9.0"`, relativePath+" bin-pack core selector") + requireContains(t, content, `sitectl = "v1.9.1"`, relativePath+" bin-pack core selector") requireContains(t, content, `sitectl-wp = "v2.1.0"`, relativePath+" bin-pack WordPress selector") requireContains(t, content, `sitectl-drupal = "v1.5.0"`, relativePath+" bin-pack Drupal selector") } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index f0bc663..94bc740 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -71,7 +71,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == { - sitectl = "v1.9.0" + sitectl = "v1.9.1" } error_message = "The DigitalOcean entrypoint must preserve an explicit core-only package set." } diff --git a/providers/gcp/template_versions.tftest.hcl b/providers/gcp/template_versions.tftest.hcl index f06698b..ee04d8a 100644 --- a/providers/gcp/template_versions.tftest.hcl +++ b/providers/gcp/template_versions.tftest.hcl @@ -67,7 +67,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == { - sitectl = "v1.9.0" + sitectl = "v1.9.1" } error_message = "The GCP entrypoint must preserve an explicit core-only package set." } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index 6c440dc..a61858c 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -81,7 +81,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == { - sitectl = "v1.9.0" + sitectl = "v1.9.1" } error_message = "The Linode entrypoint must preserve an explicit core-only package set." } diff --git a/template_versions.tftest.hcl b/template_versions.tftest.hcl index 972b920..34a310e 100644 --- a/template_versions.tftest.hcl +++ b/template_versions.tftest.hcl @@ -35,7 +35,7 @@ run "default_template_uses_v1_core" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.9.0" + sitectl = "v1.9.1" } error_message = "The default template must select the released sitectl v1 core." } @@ -63,7 +63,7 @@ run "non_isle_template_uses_v1_release_set" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.9.0" + sitectl = "v1.9.1" sitectl-wp = "v2.1.0" } error_message = "Non-ISLE templates must select their coordinated sitectl v1 release set." @@ -92,7 +92,7 @@ run "isle_template_uses_v1_release_set" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.9.0" + sitectl = "v1.9.1" sitectl-drupal = "v1.5.0" sitectl-isle = "v1.6.0" } @@ -196,7 +196,7 @@ run "custom_package_set_filters_template_versions" { assert { condition = local.sitectl.package_versions == { - sitectl = "v1.9.0" + sitectl = "v1.9.1" sitectl-wp = "v0.6.1" } error_message = "Template selectors for packages omitted by a custom package set must not reach the runtime." @@ -223,7 +223,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { assert { condition = local.sitectl.packages == tolist(["sitectl"]) && local.sitectl.package_versions == { - sitectl = "v1.9.0" + sitectl = "v1.9.1" } error_message = "An explicit core-only package set must not be mistaken for an omitted template package selection." } diff --git a/templates/apps.json b/templates/apps.json index 7fc4b13..f14a602 100644 --- a/templates/apps.json +++ b/templates/apps.json @@ -6,7 +6,7 @@ "packages": ["sitectl"], "extra_env": {}, "package_versions": { - "sitectl": "v1.9.0" + "sitectl": "v1.9.1" } }, "templates": { @@ -17,7 +17,7 @@ "packages": ["sitectl", "sitectl-archivesspace"], "extra_env": {}, "package_versions": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-archivesspace": "v2.1.1" } }, @@ -28,7 +28,7 @@ "packages": ["sitectl", "sitectl-ojs"], "extra_env": {}, "package_versions": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-ojs": "v1.4.0" } }, @@ -41,7 +41,7 @@ "ISLANDORA_TAG": "6.3.19" }, "package_versions": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-drupal": "v1.5.0", "sitectl-isle": "v1.6.0" } @@ -53,7 +53,7 @@ "packages": ["sitectl", "sitectl-drupal"], "extra_env": {}, "package_versions": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-drupal": "v1.5.0" } }, @@ -64,7 +64,7 @@ "packages": ["sitectl", "sitectl-wp"], "extra_env": {}, "package_versions": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-wp": "v2.1.0" } }, @@ -75,7 +75,7 @@ "packages": ["sitectl", "sitectl-omeka-s"], "extra_env": {}, "package_versions": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-omeka-s": "v1.4.0" } }, @@ -86,7 +86,7 @@ "packages": ["sitectl", "sitectl-omeka-classic"], "extra_env": {}, "package_versions": { - "sitectl": "v1.9.0", + "sitectl": "v1.9.1", "sitectl-omeka-classic": "v1.4.0" } } From 9e296bcb456f681e75da7fb85272f1939f7035b9 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 20:36:51 +0000 Subject: [PATCH 65/67] [patch] Invoke reviewed runtime programs by path --- Makefile | 9 +- ansible/roles/cloud_compose/tasks/main.yml | 26 ++ ci/app-filesystem-convergence-contract.sh | 1 + ci/application-env-contract.sh | 8 +- ci/compose-runtime-contract.sh | 16 ++ ci/config-management-input-contract.sh | 14 + ci/config-management-smoke-assert.py | 40 ++- ci/disaster-recovery-contract.sh | 7 +- ci/docker-plugin-install-contract.sh | 1 + ci/filesystem-prep-contract.sh | 16 ++ ci/fixtures/invalid-checked-program.jq | 1 + ci/host-runtime-security.sh | 149 ++++++++++ ci/inline-data-program-contract.sh | 66 +++++ ci/key-rotation-contract.sh | 1 + ci/managed-artifact-contract.sh | 9 +- ci/remote/config-management-verify.sh | 28 ++ ci/rootfs-package-contract.sh | 6 + ci/runtime-config-contract.sh | 40 ++- ci/sitectl-version-contract.sh | 1 + ci/source-trust-contract.sh | 1 + docs/runtime-contracts.md | 28 +- .../digitalocean/runtime_inputs.tftest.hcl | 2 +- modules/gcp/main.tf | 1 + modules/gcp/runtime_contracts.tftest.hcl | 16 +- modules/linode/runtime_inputs.tftest.hcl | 2 +- .../runtime_inputs.tftest.hcl | 12 +- providers/do/template_versions.tftest.hcl | 4 +- providers/linode/template_versions.tftest.hcl | 4 +- .../awk}/compose-secret-files.awk | 0 .../etc/cloud-compose/awk/reconcile-fstab.awk | 20 ++ .../awk/release-checksum-entry.awk | 1 + .../cloud-compose/awk/release-checksum.awk | 14 + .../jq/application-env-validate.jq | 6 + .../cloud-compose/jq/array-values-base64.jq | 1 + .../etc/cloud-compose/jq/compose-app-array.jq | 6 + .../etc/cloud-compose/jq/compose-app-field.jq | 6 + .../jq/compose-app-ingress-array.jq | 6 + .../jq/compose-app-ingress-field.jq | 6 + .../jq/compose-app-verify-args-json.jq | 8 + .../jq/compose-app-verify-args.jq | 6 + .../jq/compose-project-dirs-base64.jq | 1 + .../jq/compose-reject-host-network.jq | 12 + .../jq/compose-services-object-validate.jq | 1 + .../jq/compose-validate-projects.jq | 26 ++ .../jq/gcp-metadata-private-ip.jq | 1 + .../jq/gcp-metadata-public-ip.jq | 1 + .../jq/github-latest-release-tag.jq | 2 + rootfs/etc/cloud-compose/jq/json-length.jq | 1 + .../cloud-compose/jq/json-object-validate.jq | 1 + .../cloud-compose/jq/nonempty-string-field.jq | 1 + .../jq/object-entries-sorted-base64.jq | 1 + .../jq/object-field-delimited.jq | 1 + rootfs/etc/cloud-compose/jq/object-field.jq | 1 + rootfs/etc/cloud-compose/jq/object-has-key.jq | 1 + .../cloud-compose/jq/object-keys-base64.jq | 1 + rootfs/etc/cloud-compose/jq/object-keys.jq | 1 + rootfs/etc/cloud-compose/jq/rotation-audit.jq | 12 + .../cloud-compose/jq/rotation-build-state.jq | 14 + .../cloud-compose/jq/rotation-first-key-id.jq | 1 + .../cloud-compose/jq/rotation-idle-audit.jq | 7 + .../cloud-compose/jq/rotation-jwt-claims.jq | 7 + .../cloud-compose/jq/rotation-key-disabled.jq | 1 + .../cloud-compose/jq/rotation-key-ids-join.jq | 1 + .../jq/rotation-key-names-base64.jq | 1 + .../jq/rotation-key-names-sorted.jq | 1 + .../jq/rotation-key-object-ids-join.jq | 1 + .../cloud-compose/jq/rotation-key-present.jq | 1 + .../jq/rotation-new-key-names.jq | 1 + .../jq/rotation-normalize-user-keys.jq | 15 + .../jq/rotation-remaining-baseline.jq | 1 + .../jq/rotation-validate-state.jq | 20 ++ .../jq/service-account-credentials-valid.jq | 8 + .../jq/service-account-key-id.jq | 3 + .../jq/sitectl-package-version.jq | 1 + .../jq/sitectl-package-versions-validate.jq | 6 + .../cloud-compose/jq/string-array-validate.jq | 1 + .../cloud-compose/jq/string-field-valid.jq | 1 + .../libexec/checked-programs.bash | 271 ++++++++++++++++++ .../libexec/gcp-filesystem-boot.sh | 25 +- .../libexec/linux-vm-cloud-init.sh | 21 +- .../cloud-compose/libexec/rootfs-archive.sh | 9 +- .../cloud-compose/assert-app-initialized.sh | 22 +- .../home/cloud-compose/assert-vault-ready.sh | 23 +- rootfs/home/cloud-compose/compose-apps.sh | 189 ++++++------ .../configure-metadata-firewall.sh | 25 +- .../cloud-compose/converge-app-filesystems.sh | 33 ++- .../home/cloud-compose/default-lifecycle.sh | 25 +- .../cloud-compose/disaster-recovery-lib.sh | 50 +++- rootfs/home/cloud-compose/docker-prune.sh | 23 +- rootfs/home/cloud-compose/host-init.sh | 28 +- .../cloud-compose/install-docker-plugins.sh | 39 +-- .../cloud-compose/libops-managed-runtime.sh | 82 ++++-- rootfs/home/cloud-compose/mariadb-backup.sh | 29 +- rootfs/home/cloud-compose/mount-overlays.sh | 29 +- rootfs/home/cloud-compose/offhost-backup.sh | 55 +++- .../home/cloud-compose/persist-filesystems.sh | 47 +-- .../home/cloud-compose/prepare-app-sources.sh | 29 +- rootfs/home/cloud-compose/profile.sh | 56 +++- rootfs/home/cloud-compose/restore-test.sh | 35 ++- rootfs/home/cloud-compose/rotate-keys-app.sh | 62 ++-- .../home/cloud-compose/rotate-keys-daily.sh | 21 +- .../cloud-compose/rotate-keys-internal.sh | 26 +- rootfs/home/cloud-compose/rotate-keys.sh | 261 +++++++++-------- .../home/cloud-compose/run-rollout-service.sh | 37 ++- .../start-cloud-compose-bootstrap.sh | 22 +- .../cloud-compose/vault-agent-readiness.sh | 23 +- salt/cloud-compose/init.sls | 28 ++ templates/gcp-filesystem-boothook.sh.tftpl | 7 +- 108 files changed, 1978 insertions(+), 400 deletions(-) create mode 100644 ci/fixtures/invalid-checked-program.jq create mode 100644 ci/inline-data-program-contract.sh rename rootfs/{home/cloud-compose => etc/cloud-compose/awk}/compose-secret-files.awk (100%) create mode 100644 rootfs/etc/cloud-compose/awk/reconcile-fstab.awk create mode 100644 rootfs/etc/cloud-compose/awk/release-checksum-entry.awk create mode 100644 rootfs/etc/cloud-compose/awk/release-checksum.awk create mode 100644 rootfs/etc/cloud-compose/jq/application-env-validate.jq create mode 100644 rootfs/etc/cloud-compose/jq/array-values-base64.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-app-array.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-app-field.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-app-ingress-array.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-app-ingress-field.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-app-verify-args-json.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-app-verify-args.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-project-dirs-base64.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-reject-host-network.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-services-object-validate.jq create mode 100644 rootfs/etc/cloud-compose/jq/compose-validate-projects.jq create mode 100644 rootfs/etc/cloud-compose/jq/gcp-metadata-private-ip.jq create mode 100644 rootfs/etc/cloud-compose/jq/gcp-metadata-public-ip.jq create mode 100644 rootfs/etc/cloud-compose/jq/github-latest-release-tag.jq create mode 100644 rootfs/etc/cloud-compose/jq/json-length.jq create mode 100644 rootfs/etc/cloud-compose/jq/json-object-validate.jq create mode 100644 rootfs/etc/cloud-compose/jq/nonempty-string-field.jq create mode 100644 rootfs/etc/cloud-compose/jq/object-entries-sorted-base64.jq create mode 100644 rootfs/etc/cloud-compose/jq/object-field-delimited.jq create mode 100644 rootfs/etc/cloud-compose/jq/object-field.jq create mode 100644 rootfs/etc/cloud-compose/jq/object-has-key.jq create mode 100644 rootfs/etc/cloud-compose/jq/object-keys-base64.jq create mode 100644 rootfs/etc/cloud-compose/jq/object-keys.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-audit.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-build-state.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-first-key-id.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-idle-audit.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-jwt-claims.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-key-disabled.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-key-ids-join.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-key-names-base64.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-key-names-sorted.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-key-object-ids-join.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-key-present.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-new-key-names.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-normalize-user-keys.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-remaining-baseline.jq create mode 100644 rootfs/etc/cloud-compose/jq/rotation-validate-state.jq create mode 100644 rootfs/etc/cloud-compose/jq/service-account-credentials-valid.jq create mode 100644 rootfs/etc/cloud-compose/jq/service-account-key-id.jq create mode 100644 rootfs/etc/cloud-compose/jq/sitectl-package-version.jq create mode 100644 rootfs/etc/cloud-compose/jq/sitectl-package-versions-validate.jq create mode 100644 rootfs/etc/cloud-compose/jq/string-array-validate.jq create mode 100644 rootfs/etc/cloud-compose/jq/string-field-valid.jq create mode 100644 rootfs/etc/cloud-compose/libexec/checked-programs.bash diff --git a/Makefile b/Makefile index ed0b7fb..0443768 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci +.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security inline-data-program-contract cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci .PHONY: terraform-fmt terraform-fmt-check terraform-validate terraform-validate-contract terraform-lint-check terraform-docs terraform-docs-check .PHONY: config-management-cloud-smoke config-management-cloud-smoke-ansible-drupal config-management-cloud-smoke-salt-drupal .PHONY: destroy-config-management-cloud-smoke destroy-config-management-cloud-smoke-ansible-drupal destroy-config-management-cloud-smoke-salt-drupal @@ -16,9 +16,9 @@ export CLOUD_COMPOSE_CI_BIN GO_MODULE_FILES := $(wildcard go.mod go.sum) GO_SOURCES := $(shell find cmd internal -type f -name '*.go') -lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check +lint: terraform-fmt actionlint shell-lint host-runtime-security inline-data-program-contract cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check -lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check +lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security inline-data-program-contract cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check actionlint: go run github.com/rhysd/actionlint/cmd/actionlint@$(ACTIONLINT_VERSION) @@ -105,6 +105,9 @@ rootfs-package-contract: host-runtime-security: bash ci/host-runtime-security.sh +inline-data-program-contract: + bash ci/inline-data-program-contract.sh + cos-jq-portability-contract: bash ci/cos-jq-portability-contract.sh diff --git a/ansible/roles/cloud_compose/tasks/main.yml b/ansible/roles/cloud_compose/tasks/main.yml index fd5e56d..d4ce17e 100644 --- a/ansible/roles/cloud_compose/tasks/main.yml +++ b/ansible/roles/cloud_compose/tasks/main.yml @@ -528,7 +528,9 @@ group: root mode: "0755" loop: + - "{{ cloud_compose_home }}" - /etc/cloud-compose + - /etc/cloud-compose/awk - /etc/cloud-compose/bin - /etc/cloud-compose/jq - /etc/cloud-compose/libexec @@ -581,6 +583,14 @@ mode: "0755" loop: "{{ cloud_compose_shell_scripts.files }}" +- name: Make checked Cloud Compose program resolver root-controlled + ansible.builtin.file: + path: /etc/cloud-compose/libexec/checked-programs.bash + state: file + owner: root + group: root + mode: "0644" + - name: Find checked-in Cloud Compose jq programs ansible.builtin.find: paths: /etc/cloud-compose/jq @@ -597,6 +607,22 @@ mode: "0644" loop: "{{ cloud_compose_jq_programs.files }}" +- name: Find checked-in Cloud Compose awk programs + ansible.builtin.find: + paths: /etc/cloud-compose/awk + patterns: "*.awk" + recurse: false + file_type: file + register: cloud_compose_awk_programs + +- name: Make checked-in Cloud Compose awk programs root-controlled + ansible.builtin.file: + path: "{{ item.path }}" + owner: root + group: root + mode: "0644" + loop: "{{ cloud_compose_awk_programs.files }}" + - name: Write cloud-compose lifecycle dispatchers ansible.builtin.copy: src: /home/cloud-compose/lifecycle-entrypoint.sh diff --git a/ci/app-filesystem-convergence-contract.sh b/ci/app-filesystem-convergence-contract.sh index cc6a5f1..494e7de 100755 --- a/ci/app-filesystem-convergence-contract.sh +++ b/ci/app-filesystem-convergence-contract.sh @@ -3,6 +3,7 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT diff --git a/ci/application-env-contract.sh b/ci/application-env-contract.sh index 303575f..118d629 100755 --- a/ci/application-env-contract.sh +++ b/ci/application-env-contract.sh @@ -18,6 +18,7 @@ require_cmd base64 require_cmd jq profile="$repo_root/rootfs/home/cloud-compose/profile.sh" +jq_program_dir="$repo_root/rootfs/etc/cloud-compose/jq" rollout_service="$repo_root/rootfs/home/cloud-compose/run-rollout-service.sh" host_env="$tmp/host.env" application_env="$tmp/application-env.json" @@ -69,6 +70,7 @@ done # shellcheck disable=SC2016 env -i \ PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir" \ CLOUD_COMPOSE_ENV_FILE="$host_env" \ CLOUD_COMPOSE_APPLICATION_ENV_FILE="$application_env" \ bash --noprofile --norc -c ' @@ -138,7 +140,8 @@ for invalid_application_data in \ '{"SAFE\n":"value"}' \ '{"SAFE\u0000":"value"}'; do printf '%s' "$invalid_application_data" >"$tmp/application-env.invalid.json" - if env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$host_env" \ + if env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir" \ + CLOUD_COMPOSE_ENV_FILE="$host_env" \ CLOUD_COMPOSE_APPLICATION_ENV_FILE="$tmp/application-env.invalid.json" \ bash --noprofile --norc -c ' source "$1" @@ -155,7 +158,8 @@ done jq 'del(.APPLICATION_LITERAL)' "$application_env" >"$tmp/application-env.next.json" # The child shell receives file paths as positional parameters. # shellcheck disable=SC2016 -env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$host_env" \ +env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir" \ + CLOUD_COMPOSE_ENV_FILE="$host_env" \ CLOUD_COMPOSE_APPLICATION_ENV_FILE="$tmp/application-env.next.json" \ bash --noprofile --norc -c ' source "$1" diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh index d72890f..08f85a5 100755 --- a/ci/compose-runtime-contract.sh +++ b/ci/compose-runtime-contract.sh @@ -99,6 +99,7 @@ export COMPOSE_PROJECTS_FILE="$tmp/compose-projects.json" export COMPOSE_APPS_ENV_DIR="$tmp/apps" export COMPOSE_APPS_STATE_DIR="$tmp/state" export CLOUD_COMPOSE_DATA_ROOT="$tmp/data" +export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" mkdir -p "$CLOUD_COMPOSE_DATA_ROOT/project" # shellcheck disable=SC1091 @@ -230,6 +231,21 @@ FAKE_COMPOSE_CONFIG="$(jq -cn '{services:{web:{network_mode:"default"}}}')" export FAKE_COMPOSE_CONFIG reject_host_network_compose_services +checked_jq_program_dir="$CLOUD_COMPOSE_JQ_PROGRAM_DIR" +CLOUD_COMPOSE_JQ_PROGRAM_DIR="$tmp/missing-jq-programs" +if reject_host_network_compose_services >/dev/null 2>&1; then + fail "GCP metadata isolation accepted a missing checked jq program" +fi +mkdir -p "$tmp/invalid-jq-programs" +cp "$repo_root/ci/fixtures/invalid-checked-program.jq" \ + "$tmp/invalid-jq-programs/compose-reject-host-network.jq" +CLOUD_COMPOSE_JQ_PROGRAM_DIR="$tmp/invalid-jq-programs" +if reject_host_network_compose_services >/dev/null 2>&1; then + fail "GCP metadata isolation accepted an invalid checked jq program" +fi +CLOUD_COMPOSE_JQ_PROGRAM_DIR="$checked_jq_program_dir" +export CLOUD_COMPOSE_JQ_PROGRAM_DIR + FAKE_COMPOSE_CONFIG="$(jq -cn '{services:{web:{build:{context:".",network:"host"}}}}')" export FAKE_COMPOSE_CONFIG if reject_host_network_compose_services >/dev/null 2>&1; then diff --git a/ci/config-management-input-contract.sh b/ci/config-management-input-contract.sh index e4b2757..3f23669 100755 --- a/ci/config-management-input-contract.sh +++ b/ci/config-management-input-contract.sh @@ -32,6 +32,17 @@ grep -Fq 'cmd: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh' "$ansible_t grep -Fq -- '- name: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh' "$salt_state" || \ contract_fail "Salt does not apply the shared bootstrap path hardener" +ansible_privileged_block="$(sed -n '/^- name: Secure Cloud Compose privileged program directories/,/^- name:/p' "$ansible_tasks")" +grep -Fq -- '- "{{ cloud_compose_home }}"' <<<"$ansible_privileged_block" || \ + contract_fail "Ansible does not close the account-owned runtime-home window after installing rootfs" +salt_privileged_block="$(sed -n '/^cloud-compose-privileged-program-directories:/,/^[^[:space:]]/p' "$salt_state")" +grep -Fq -- '- {{ home | json }}' <<<"$salt_privileged_block" || \ + contract_fail "Salt does not close the account-owned runtime-home window after installing rootfs" +grep -Fq 'path: /etc/cloud-compose/libexec/checked-programs.bash' "$ansible_tasks" || \ + contract_fail "Ansible does not explicitly secure the checked program resolver" +grep -Fq 'source: salt://rootfs/etc/cloud-compose/libexec/checked-programs.bash' "$salt_state" || \ + contract_fail "Salt does not explicitly secure the checked program resolver" + grep -Fq 'configure-metadata-firewall.sh | deploy-rollout.sh | docker-prune.sh' "$root_program_runner" || \ contract_fail "the trusted root-program runner does not allow deploy-rollout.sh" @@ -49,6 +60,9 @@ for marker in \ '- file: cloud-compose-rootfs' \ '- cmd: cloud-compose-lifecycle-lock' \ '- cmd: cloud-compose-rootfs-script-modes' \ + '- file: cloud-compose-checked-program-resolver' \ + '- cmd: cloud-compose-rootfs-jq-modes' \ + '- cmd: cloud-compose-rootfs-awk-modes' \ '- file: cloud-compose-lifecycle-init' \ '- file: cloud-compose-lifecycle-up' \ '- file: cloud-compose-lifecycle-down' \ diff --git a/ci/config-management-smoke-assert.py b/ci/config-management-smoke-assert.py index fdbfe32..8a98b19 100644 --- a/ci/config-management-smoke-assert.py +++ b/ci/config-management-smoke-assert.py @@ -15,6 +15,7 @@ PRIVILEGED_PROGRAM_ROOT = Path("/etc/cloud-compose") DIAGNOSTICS_PROGRAM = PRIVILEGED_PROGRAM_ROOT / "bin/cloud-compose-diagnostics.sh" BOOTSTRAP_LIBEXEC = Path("/etc/cloud-compose/libexec") +AWK_PROGRAM_DIR = PRIVILEGED_PROGRAM_ROOT / "awk" JQ_PROGRAM_DIR = PRIVILEGED_PROGRAM_ROOT / "jq" @@ -63,8 +64,14 @@ def assert_runtime_files() -> None: assert os.access(path, os.X_OK), path for path in [ + BOOTSTRAP_LIBEXEC / "checked-programs.bash", + AWK_PROGRAM_DIR / "compose-secret-files.awk", + AWK_PROGRAM_DIR / "reconcile-fstab.awk", + AWK_PROGRAM_DIR / "release-checksum.awk", JQ_PROGRAM_DIR / "diagnostics-validate-compose-projects.jq", + JQ_PROGRAM_DIR / "compose-validate-projects.jq", JQ_PROGRAM_DIR / "offhost-validate-manifest.jq", + JQ_PROGRAM_DIR / "rotation-validate-state.jq", JQ_PROGRAM_DIR / "sitectl-verify-args.jq", ]: assert path.exists(), path @@ -97,13 +104,25 @@ def assert_runtime_files() -> None: oct(stat.S_IMODE(metadata.st_mode)), ) + checked_programs = BOOTSTRAP_LIBEXEC / "checked-programs.bash" + checked_programs_metadata = checked_programs.lstat() + assert not checked_programs.is_symlink() + assert checked_programs_metadata.st_uid == 0 + assert checked_programs_metadata.st_gid == 0 + assert checked_programs_metadata.st_nlink == 1 + assert stat.S_IMODE(checked_programs_metadata.st_mode) == 0o644 + for path in [ + RUNTIME_HOME, PRIVILEGED_PROGRAM_ROOT, + AWK_PROGRAM_DIR, DIAGNOSTICS_PROGRAM.parent, BOOTSTRAP_LIBEXEC, JQ_PROGRAM_DIR, ]: - metadata = path.stat() + assert not path.is_symlink(), path + metadata = path.lstat() + assert stat.S_ISDIR(metadata.st_mode), path assert metadata.st_uid == 0, (path, metadata.st_uid) assert metadata.st_gid == 0, (path, metadata.st_gid) assert stat.S_IMODE(metadata.st_mode) == 0o755, ( @@ -124,7 +143,24 @@ def assert_runtime_files() -> None: jq_programs = list(JQ_PROGRAM_DIR.glob("*.jq")) assert jq_programs for path in jq_programs: - metadata = path.stat() + assert not path.is_symlink(), path + metadata = path.lstat() + assert stat.S_ISREG(metadata.st_mode), path + assert metadata.st_nlink == 1, (path, metadata.st_nlink) + assert metadata.st_uid == 0, (path, metadata.st_uid) + assert metadata.st_gid == 0, (path, metadata.st_gid) + assert stat.S_IMODE(metadata.st_mode) == 0o644, ( + path, + oct(stat.S_IMODE(metadata.st_mode)), + ) + + awk_programs = list(AWK_PROGRAM_DIR.glob("*.awk")) + assert awk_programs + for path in awk_programs: + assert not path.is_symlink(), path + metadata = path.lstat() + assert stat.S_ISREG(metadata.st_mode), path + assert metadata.st_nlink == 1, (path, metadata.st_nlink) assert metadata.st_uid == 0, (path, metadata.st_uid) assert metadata.st_gid == 0, (path, metadata.st_gid) assert stat.S_IMODE(metadata.st_mode) == 0o644, ( diff --git a/ci/disaster-recovery-contract.sh b/ci/disaster-recovery-contract.sh index a6c7d5b..4a7c565 100644 --- a/ci/disaster-recovery-contract.sh +++ b/ci/disaster-recovery-contract.sh @@ -63,12 +63,15 @@ bash "$backup_script" [[ "$(grep -c '^backup$' "$tmp/drivers/good-driver.calls")" == "2" ]] || \ fail "nightly flow skipped off-host transfer when the daily dump already existed" -published_receipt_sha="$(sha256sum "$receipt" | awk '{print $1}')" +published_receipt_sha="$(sha256sum -- "$receipt")" +published_receipt_sha="${published_receipt_sha%% *}" export CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="$tmp/drivers/incomplete-driver" if bash "$backup_script" >/dev/null 2>&1; then fail "driver receipt without complete volume coverage was accepted" fi -[[ "$(sha256sum "$receipt" | awk '{print $1}')" == "$published_receipt_sha" ]] || \ +current_receipt_sha="$(sha256sum -- "$receipt")" +current_receipt_sha="${current_receipt_sha%% *}" +[[ "$current_receipt_sha" == "$published_receipt_sha" ]] || \ fail "invalid driver output replaced the last validated receipt" export CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="$tmp/drivers/good-driver" diff --git a/ci/docker-plugin-install-contract.sh b/ci/docker-plugin-install-contract.sh index 1efdf94..8bf765c 100755 --- a/ci/docker-plugin-install-contract.sh +++ b/ci/docker-plugin-install-contract.sh @@ -103,6 +103,7 @@ run_installer() { PATH="${BIN_DIR}:$PATH" \ DOCKER_PLUGIN_INSTALLER="$INSTALLER" \ + CLOUD_COMPOSE_RELEASE_CHECKSUM_PROGRAM="$ROOT_DIR/rootfs/etc/cloud-compose/awk/release-checksum.awk" \ DOCKER_CLI_PLUGIN_DIR="$PLUGIN_DIR" \ DOCKER_COMPOSE_VERSION="$compose_version" \ DOCKER_BUILDX_VERSION="$buildx_version" \ diff --git a/ci/filesystem-prep-contract.sh b/ci/filesystem-prep-contract.sh index aff3b6a..0c37ffc 100644 --- a/ci/filesystem-prep-contract.sh +++ b/ci/filesystem-prep-contract.sh @@ -5,6 +5,7 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" prep_script="$repo_root/rootfs/home/cloud-compose/prepare-filesystem.sh" persist_script="$repo_root/rootfs/home/cloud-compose/persist-filesystems.sh" +export CLOUD_COMPOSE_FSTAB_RECONCILE_PROGRAM="$repo_root/rootfs/etc/cloud-compose/awk/reconcile-fstab.awk" gcp_filesystem_boot="$repo_root/rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh" gcp_filesystem_boothook="$repo_root/templates/gcp-filesystem-boothook.sh.tftpl" gcp_cloud_init_mime="$repo_root/templates/gcp-cloud-init.mime.tftpl" @@ -795,8 +796,23 @@ fi grep -Fq 'FILESYSTEM_PERSIST_SCRIPT_B64' "$gcp_filesystem_boothook" || \ fail "GCP boothook does not bootstrap persistent mount configuration" +grep -Fq 'FSTAB_RECONCILE_AWK_B64' "$gcp_filesystem_boothook" || \ + fail "GCP boothook does not bootstrap the checked fstab reconciliation program" +grep -Fq 'require_root_owned_data_program "$filesystem_reconcile"' "$gcp_filesystem_boot" || \ + fail "GCP filesystem bootstrap does not verify its checked fstab reconciliation program" +grep -Fq 'CLOUD_COMPOSE_FSTAB_RECONCILE_PROGRAM="$filesystem_reconcile" bash "$filesystem_persist"' \ + "$gcp_filesystem_boot" || \ + fail "GCP filesystem bootstrap does not pass the checked fstab reconciliation program as data" grep -Fq 'bash "$filesystem_persist" "$data_device" "$volumes_device"' "$linux_cloud_init" || \ fail "Linux VM cloud-init does not invoke persistent mount configuration" +grep -Fq 'install -m 0600 -- /etc/cloud-compose/awk/reconcile-fstab.awk "$filesystem_reconcile"' \ + "$linux_cloud_init" || \ + fail "embedded Linux bootstrap does not stage the checked fstab reconciliation program" +grep -Fq 'require_root_owned_data_program "$filesystem_reconcile"' "$linux_cloud_init" || \ + fail "Linux bootstrap does not verify its checked fstab reconciliation program" +grep -Fq '"$staged_rootfs/etc/cloud-compose/awk/reconcile-fstab.awk"' \ + "$repo_root/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh" || \ + fail "archive-mode Linux bootstrap does not stage the checked fstab reconciliation program from the verified rootfs" grep -Fq '/var/lib/cloud-compose/mounted-rootfs/mnt/disks' "$linux_cloud_init" || \ fail "Linux VM cloud-init does not copy mounted-root files after mounting durable storage" diff --git a/ci/fixtures/invalid-checked-program.jq b/ci/fixtures/invalid-checked-program.jq new file mode 100644 index 0000000..6e5fbf9 --- /dev/null +++ b/ci/fixtures/invalid-checked-program.jq @@ -0,0 +1 @@ +this is not a valid jq program diff --git a/ci/host-runtime-security.sh b/ci/host-runtime-security.sh index d42701a..bd252d9 100644 --- a/ci/host-runtime-security.sh +++ b/ci/host-runtime-security.sh @@ -3,6 +3,7 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT @@ -16,6 +17,23 @@ assert_contains() { grep -Fq -- "$pattern" "$file" || fail "$file does not contain: $pattern" } +binding_inventory() { + local file="$1" marker="$2" line + local capture=false + + while IFS= read -r line; do + if [[ "$line" == *"$marker"* ]]; then + capture=true + fi + if [[ "$capture" == "true" ]]; then + printf '%s\n' "$line" + if [[ "$line" != *\\ ]]; then + capture=false + fi + fi + done <"$file" +} + mkdir -p "$tmp/bin" cat >"$tmp/profile.sh" <<'EOF' @@ -266,7 +284,11 @@ if [[ -n "$(git -C "$repo_root" ls-files 'rootfs/usr/**')" ]]; then fail "Cloud Compose-owned rootfs programs still target immutable /usr" fi for trusted_program in \ + rootfs/etc/cloud-compose/awk/compose-secret-files.awk \ + rootfs/etc/cloud-compose/awk/reconcile-fstab.awk \ + rootfs/etc/cloud-compose/awk/release-checksum.awk \ rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh \ + rootfs/etc/cloud-compose/libexec/checked-programs.bash \ rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh \ rootfs/etc/cloud-compose/libexec/gcp-cloud-init-post-bootstrap.sh \ rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh \ @@ -279,10 +301,137 @@ for trusted_program in \ rootfs/etc/cloud-compose/libexec/run-bootstrap.sh \ rootfs/etc/cloud-compose/libexec/run-root-program.sh \ rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq \ + rootfs/etc/cloud-compose/jq/compose-validate-projects.jq \ + rootfs/etc/cloud-compose/jq/rotation-validate-state.jq \ rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq; do [[ -f "$repo_root/$trusted_program" ]] || fail "COS-safe trusted program is missing: $trusted_program" done +checked_programs="$repo_root/rootfs/etc/cloud-compose/libexec/checked-programs.bash" +assert_contains "$checked_programs" 'candidate_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)"' +assert_contains "$checked_programs" 'installed_home="$(cloud_compose_installed_home)" || return 2' +assert_contains "$checked_programs" 'Installed Cloud Compose home resolves to the filesystem root' +assert_contains "$checked_programs" '"$alias_target" != "var/home"' +assert_contains "$checked_programs" "stat -c '%u:%g:%a'" +assert_contains "$checked_programs" "stat -c '%u:%g:%a:%h'" +assert_contains "$checked_programs" '"$program" == "$program_dir/$program_name"' +assert_contains "$checked_programs" 'cloud_compose_validate_installed_program_dir "$selected"' +assert_contains "$checked_programs" '$((8#$mode & 0022)) -ne 0' +assert_contains "$checked_programs" '"$links" != 1' +assert_contains "$checked_programs" 'cannot override the installed Cloud Compose program' +assert_contains "$checked_programs" 'cannot override the installed Cloud Compose source program' +assert_contains "$checked_programs" 'readonly "$variable_name"' +if grep -Fq '%F' "$checked_programs"; then + fail "checked program validation relies on localized file-type labels" +fi + +while IFS= read -r checked_consumer; do + assert_contains "$checked_consumer" '/etc/cloud-compose/libexec/checked-programs.bash' + assert_contains "$checked_consumer" 'readlink -f -- /home/cloud-compose' +done < <(grep -RlE --include='*.sh' \ + 'cloud_compose_bind_(program|program_dir|source_program)' \ + "$repo_root/rootfs/home/cloud-compose") + +if grep -REq --include='*.sh' '\$\{![A-Za-z_]' \ + "$repo_root/rootfs/home/cloud-compose"; then + fail "indirect parameter expansion bypasses the installed source/program inventory" +fi + +source_reference_pattern='\$CLOUD_COMPOSE_[A-Z0-9_]*_PATH([^A-Z0-9_]|$)|\$\{CLOUD_COMPOSE_[A-Z0-9_]*_PATH([^A-Z0-9_]|$)' +while IFS= read -r source_consumer; do + source_binding_inventory="$(binding_inventory \ + "$source_consumer" cloud_compose_bind_source_program)" + mapfile -t referenced_source_variables < <( + grep -oE \ + '\$CLOUD_COMPOSE_[A-Z0-9_]*_PATH|\$\{CLOUD_COMPOSE_[A-Z0-9_]*_PATH([^A-Z0-9_]|$)' \ + "$source_consumer" | + sed -E 's/^\$\{//; s/^\$//; s/([^A-Z0-9_]).*$//' | + sort -u + ) + requires_source_binder=false + for source_variable in "${referenced_source_variables[@]}"; do + case "$source_variable" in + CLOUD_COMPOSE_DOCKER_PRUNE_LOCK_PATH) + [[ "$source_consumer" == "$repo_root/rootfs/home/cloud-compose/docker-prune.sh" ]] || \ + fail "$source_variable is registered only as docker-prune lock data" + continue + ;; + CLOUD_COMPOSE_FSTAB_PATH | CLOUD_COMPOSE_FSTAB_LOCK_PATH) + [[ "$source_consumer" == "$repo_root/rootfs/home/cloud-compose/persist-filesystems.sh" ]] || \ + fail "$source_variable is registered only as fstab data" + continue + ;; + esac + requires_source_binder=true + grep -Eq "(^|[^A-Z0-9_])${source_variable}([^A-Z0-9_]|$)" \ + <<<"$source_binding_inventory" || \ + fail "$source_consumer references $source_variable without binding that exact source variable" + done + if [[ "$requires_source_binder" == "true" ]]; then + assert_contains "$source_consumer" 'cloud_compose_bind_source_program' + assert_contains "$source_consumer" '/etc/cloud-compose/libexec/checked-programs.bash' + assert_contains "$source_consumer" 'readlink -f -- /home/cloud-compose' + fi +done < <(grep -RlE --include='*.sh' \ + "$source_reference_pattern" \ + "$repo_root/rootfs/home/cloud-compose") + +program_reference_pattern='\$CLOUD_COMPOSE_[A-Z0-9_]*(PROGRAM|PROGRAM_DIR)([^A-Z0-9_]|$)|\$\{CLOUD_COMPOSE_[A-Z0-9_]*(PROGRAM|PROGRAM_DIR)([^A-Z0-9_]|$)' +while IFS= read -r program_consumer; do + program_dir_binding_inventory="$(binding_inventory \ + "$program_consumer" cloud_compose_bind_program_dir)" + program_binding_inventory="$(binding_inventory \ + "$program_consumer" 'cloud_compose_bind_program ')" + mapfile -t referenced_program_variables < <( + grep -oE \ + '\$CLOUD_COMPOSE_[A-Z0-9_]*(PROGRAM|PROGRAM_DIR)|\$\{CLOUD_COMPOSE_[A-Z0-9_]*(PROGRAM|PROGRAM_DIR)([^A-Z0-9_]|$)' \ + "$program_consumer" | + sed -E 's/^\$\{//; s/^\$//; s/([^A-Z0-9_]).*$//' | + sort -u + ) + for program_variable in "${referenced_program_variables[@]}"; do + case "$program_variable" in + *_PROGRAM_DIR) binding_inventory="$program_dir_binding_inventory" ;; + *) binding_inventory="$program_binding_inventory" ;; + esac + grep -Eq "(^|[^A-Z0-9_])${program_variable}([^A-Z0-9_]|$)" \ + <<<"$binding_inventory" || \ + fail "$program_consumer references $program_variable without binding that exact program variable" + done + assert_contains "$program_consumer" '/etc/cloud-compose/libexec/checked-programs.bash' + assert_contains "$program_consumer" 'readlink -f -- /home/cloud-compose' +done < <(grep -RlE --include='*.sh' \ + "$program_reference_pattern" \ + "$repo_root/rootfs/home/cloud-compose") + +assert_contains "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" \ + '/etc/cloud-compose/awk/compose-secret-files.awk' +[[ ! -e "$repo_root/rootfs/home/cloud-compose/compose-secret-files.awk" ]] || \ + fail "Compose secret extraction program remains under the runtime home" +assert_contains "$rotate_script" 'cloud_compose_bind_source_program' +assert_contains "$repo_root/rootfs/home/cloud-compose/rotate-keys-app.sh" \ + 'CLOUD_COMPOSE_ROTATE_KEYS_PATH' +assert_contains "$repo_root/rootfs/home/cloud-compose/rotate-keys-app.sh" \ + 'CLOUD_COMPOSE_COMPOSE_APPS_PATH' +assert_contains "$managed_runtime" 'cloud_compose_bind_source_program' + +assert_contains "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" \ + '-f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-validate-projects.jq"' +assert_contains "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" \ + 'if [[ "$filter_status" -ne 1 ]]; then' +assert_contains "$repo_root/rootfs/home/cloud-compose/profile.sh" \ + '-f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/application-env-validate.jq"' +assert_contains "$rotate_script" \ + '-f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-validate-state.jq"' +assert_contains "$repo_root/rootfs/home/cloud-compose/rotate-keys-app.sh" \ + '-f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/service-account-credentials-valid.jq"' +assert_contains "$managed_runtime" \ + '-f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/sitectl-package-versions-validate.jq"' +assert_contains "$repo_root/rootfs/home/cloud-compose/persist-filesystems.sh" \ + '-f "$fstab_reconcile_program"' +assert_contains "$repo_root/rootfs/home/cloud-compose/install-docker-plugins.sh" \ + '-f "$release_checksum_program"' + assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh" \ '/etc/cloud-compose/libexec/harden-bootstrap-paths.sh' assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh" \ diff --git a/ci/inline-data-program-contract.sh b/ci/inline-data-program-contract.sh new file mode 100644 index 0000000..6c15dd7 --- /dev/null +++ b/ci/inline-data-program-contract.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +readonly repo_root + +fail() { + echo "inline data program contract: $*" >&2 + exit 1 +} + +unchecked_data_program_invocation() { + local logical_command="$1" + local invocation_pattern + + # Match commands at a statement boundary, in a command/process + # substitution, or after a pipeline/control operator. This intentionally + # ignores package names, filenames, and prose that merely mention jq/awk. + invocation_pattern='(^|[|;&]|[$][(]|<[(])[[:space:]]*(if|elif|while|until|then)?[[:space:]]*!?[[:space:]]*(command[[:space:]]+)?(jq|awk)([[:space:]]|$)' + [[ "$logical_command" =~ $invocation_pattern ]] || return 1 + [[ ! "$logical_command" =~ (^|[[:space:]])-f([[:space:]]|$) ]] +} + +# Keep the scanner itself fail-closed as its shell matching evolves. +unchecked_data_program_invocation 'value="$(jq -r '\''keys[]'\'' input.json)"' || \ + fail "scanner did not recognize inline jq" +unchecked_data_program_invocation 'awk -v key=value '\''$1 == key { print }'\'' input' || \ + fail "scanner did not recognize inline awk" +if unchecked_data_program_invocation 'jq -r -f "$program" input.json'; then + fail "scanner rejected checked jq" +fi +if unchecked_data_program_invocation 'awk -v key=value -f "$program" input'; then + fail "scanner rejected checked awk" +fi + +while IFS= read -r -d '' script; do + IFS= read -r shebang <"$script" || continue + case "$shebang" in + '#!/usr/bin/env bash' | '#!/bin/bash' | '#!/bin/sh') ;; + *) continue ;; + esac + logical_command="" + logical_start=0 + line_number=0 + while IFS= read -r line || [[ -n "$line" ]]; do + ((line_number += 1)) + if [[ -z "$logical_command" ]]; then + logical_start="$line_number" + fi + if [[ "${line: -1}" == "\\" ]]; then + logical_command+="${line::-1} " + continue + fi + logical_command+="$line" + if unchecked_data_program_invocation "$logical_command"; then + fail "${script#$repo_root/}:$logical_start invokes jq/awk without a checked -f program" + fi + logical_command="" + done <"$script" + if [[ -n "$logical_command" ]]; then + fail "${script#$repo_root/}:$logical_start ends with an incomplete continued command" + fi +done < <(find "$repo_root/rootfs" -type f -print0) + +echo "Inline data program contract passed" diff --git a/ci/key-rotation-contract.sh b/ci/key-rotation-contract.sh index 353f7d8..d529f80 100644 --- a/ci/key-rotation-contract.sh +++ b/ci/key-rotation-contract.sh @@ -3,6 +3,7 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" tmp="$(mktemp -d)" trap 'rm -rf -- "$tmp"' EXIT diff --git a/ci/managed-artifact-contract.sh b/ci/managed-artifact-contract.sh index 608b56f..cb0f4a7 100644 --- a/ci/managed-artifact-contract.sh +++ b/ci/managed-artifact-contract.sh @@ -84,8 +84,12 @@ CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh" bash --noprofile --norc -c ' managed_artifact_metadata_matches "$2" 0755 "$3" "$4" ! managed_artifact_metadata_matches "$2" 0755 wrong-owner "$4" ! managed_artifact_metadata_matches "$2" 0755 "$3" wrong-group -' managed-artifact-metadata "$runtime_script" "$target" "$artifact_owner" "$artifact_group" || \ - fail "owner/group metadata does not participate in the installed-spec gate" + test "$(tab_separated_field_count "$5")" = 8 + test "$(tab_separated_field_count "$6")" = 9 +' managed-artifact-metadata "$runtime_script" "$target" "$artifact_owner" "$artifact_group" \ + $'one\ttwo\tthree\tfour\tfive\tsix\tseven\t' \ + $'one\ttwo\tthree\tfour\tfive\tsix\tseven\teight\t' || \ + fail "managed-artifact metadata or tab-field parsing contract failed" chmod 0644 "$target" : >"$curl_log" run_install @@ -141,6 +145,7 @@ assert_rejected $'tool\thttps://example.invalid/tool\t'"$sha"$'\t'"$target"$'\t0 assert_rejected $'tool\thttps://example.invalid/tool\t'"$sha"$'\t'"$target"$'\t0755\troot\troot\t-unit.service' assert_rejected $'tool\thttps://example.invalid/tool\t'"$sha"$'\t'"$target"$'\t0755\troot\troot\tunit;id.service' assert_rejected $'tool\thttps://example.invalid/tool\t'"$sha"$'\t'"$target"$'\t0755\troot\troot' +assert_rejected $'tool\thttps://example.invalid/tool\t'"$sha"$'\t'"$target"$'\t0755\troot\troot\tunit.service\textra' long_name="$(printf 'a%.0s' {1..129})" assert_rejected "$long_name"$'\thttps://example.invalid/tool\t'"$sha"$'\t'"$target"$'\t0755\troot\troot\t' diff --git a/ci/remote/config-management-verify.sh b/ci/remote/config-management-verify.sh index 192b31c..ed355eb 100755 --- a/ci/remote/config-management-verify.sh +++ b/ci/remote/config-management-verify.sh @@ -39,6 +39,34 @@ for program in \ test -x "$program" done +shopt -s nullglob dotglob +awk_programs=(/etc/cloud-compose/awk/*.awk) +jq_programs=(/etc/cloud-compose/jq/*.jq) +shopt -u nullglob dotglob +(( ${#awk_programs[@]} > 0 )) +(( ${#jq_programs[@]} > 0 )) +data_programs=( + /etc/cloud-compose/libexec/checked-programs.bash + "${awk_programs[@]}" + "${jq_programs[@]}" +) +for data_program in "${data_programs[@]}"; do + test ! -L "$data_program" + test -f "$data_program" + test "$(stat -c '%u:%g:%a:%h' -- "$data_program")" = "0:0:644:1" +done + +for program_parent in \ + /home/cloud-compose \ + /etc/cloud-compose \ + /etc/cloud-compose/awk \ + /etc/cloud-compose/jq \ + /etc/cloud-compose/libexec; do + test ! -L "$program_parent" + test -d "$program_parent" + test "$(stat -c '%u:%g:%a' -- "$program_parent")" = "0:0:755" +done + python3 -m json.tool /home/cloud-compose/compose-projects.json >/dev/null python3 -m json.tool /home/cloud-compose/application-env.json >/dev/null python3 "$runtime_state_contract" diff --git a/ci/rootfs-package-contract.sh b/ci/rootfs-package-contract.sh index 0c8ad1a..f7ec1dc 100755 --- a/ci/rootfs-package-contract.sh +++ b/ci/rootfs-package-contract.sh @@ -36,6 +36,12 @@ tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/home/cloud-co tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/home/cloud-compose/default-lifecycle.sh' >/dev/null tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/home/cloud-compose/lifecycle-entrypoint.sh' >/dev/null tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/etc/cloud-compose/jq/sitectl-verify-args.jq' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/etc/cloud-compose/jq/compose-validate-projects.jq' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/etc/cloud-compose/jq/rotation-validate-state.jq' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/etc/cloud-compose/awk/reconcile-fstab.awk' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/etc/cloud-compose/awk/release-checksum.awk' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/etc/cloud-compose/awk/compose-secret-files.awk' >/dev/null +tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/etc/cloud-compose/libexec/checked-programs.bash' >/dev/null for bootstrap_program in \ gcp-cloud-init-finalize.sh \ gcp-cloud-init-post-bootstrap.sh \ diff --git a/ci/runtime-config-contract.sh b/ci/runtime-config-contract.sh index 0fade8e..f6dc862 100644 --- a/ci/runtime-config-contract.sh +++ b/ci/runtime-config-contract.sh @@ -74,7 +74,9 @@ assert_runtime_values() { # The child shell receives values as positional parameters. # shellcheck disable=SC2016 - env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$env_file" bash --noprofile --norc -c ' + env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$env_file" bash --noprofile --norc -c ' source "$1" shift test "$BACKTICKS" = "$1" @@ -249,7 +251,9 @@ update_dir="$tmp/update" mkdir -p "$update_dir" cp "$env_file" "$update_dir/.env" UPDATE_VALUE=$'updated $(touch update-injection) "quote" \\ path\nnext line' -env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$update_dir/.env" \ +env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$update_dir/.env" \ bash --noprofile --norc -c ' source "$1" cd "$2" @@ -272,7 +276,9 @@ DOMAIN=example.org EXPANDED=${DOMAIN}/path SINGLE_QUOTED='literal $DOMAIN' EOF -env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$env_file" \ +env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$env_file" \ bash --noprofile --norc -c ' source "$1" cd "$2" @@ -306,7 +312,9 @@ jq -n \ '$ARGS.named' >"$tmp/application-env.json" # The child shell receives file paths as positional parameters. # shellcheck disable=SC2016 -env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$env_file" \ +env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$env_file" \ CLOUD_COMPOSE_APPLICATION_ENV_FILE="$tmp/application-env.json" \ bash --noprofile --norc -c ' source "$1" @@ -323,7 +331,9 @@ test ! -e "$tmp/command-injection" unsafe_update_dir="$tmp/unsafe-update" mkdir -p "$unsafe_update_dir" ln -s /etc/passwd "$unsafe_update_dir/.env" -if env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$env_file" \ +if env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$env_file" \ bash --noprofile --norc -c ' source "$1" cd "$2" @@ -335,7 +345,9 @@ if env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$env_file" \ exit 1 fi -if env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$env_file" \ +if env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$env_file" \ bash --noprofile --norc -c ' source "$1" cd "$2" @@ -350,7 +362,9 @@ fi printf '%s\n' 'BAD-NAME="value"' >"$tmp/invalid.env" # Source runs in the intentionally isolated child shell. # shellcheck disable=SC2016 -if env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$tmp/invalid.env" \ +if env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$tmp/invalid.env" \ bash --noprofile --norc -c 'source "$1"' cloud-compose-env \ "$repo_root/rootfs/home/cloud-compose/profile.sh" >/dev/null 2>&1; then echo "Runtime environment loader accepted an unsafe variable name" >&2 @@ -365,7 +379,9 @@ for invalid_value in \ printf '%s\n' "$invalid_value" >"$tmp/invalid.env" # Source runs in the intentionally isolated child shell. # shellcheck disable=SC2016 - if env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$tmp/invalid.env" \ + if env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$tmp/invalid.env" \ bash --noprofile --norc -c 'source "$1"' cloud-compose-env \ "$repo_root/rootfs/home/cloud-compose/profile.sh" >/dev/null 2>&1; then echo "Runtime environment loader accepted data outside the encoding contract: $invalid_value" >&2 @@ -374,7 +390,9 @@ for invalid_value in \ done retry_log="$tmp/retry.log" -if env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$env_file" MAX_RETRIES=1 \ +if env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$env_file" MAX_RETRIES=1 \ bash --noprofile --norc -c ' source "$1" retry_until_success /bin/false "https://example.invalid/?token=must-not-appear" @@ -398,7 +416,9 @@ for assignment in \ 'SLEEP_INCREMENT=1+1' \ 'SLEEP_INCREMENT=a[$(touch /tmp/cloud-compose-retry-injection)]'; do rm -f /tmp/cloud-compose-retry-injection - if env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$env_file" "$assignment" \ + if env -i PATH=/usr/bin:/bin \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" \ + CLOUD_COMPOSE_ENV_FILE="$env_file" "$assignment" \ bash --noprofile --norc -c ' source "$1" retry_until_success /bin/true diff --git a/ci/sitectl-version-contract.sh b/ci/sitectl-version-contract.sh index f868d64..a2ba393 100644 --- a/ci/sitectl-version-contract.sh +++ b/ci/sitectl-version-contract.sh @@ -4,6 +4,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "$0")/.." && pwd)" runtime_script="$repo_root/rootfs/home/cloud-compose/libops-managed-runtime.sh" +export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" tmp="$(mktemp -d "${TMPDIR:-/tmp}/cloud-compose-sitectl-versions.XXXXXX")" trap 'rm -rf "$tmp"' EXIT diff --git a/ci/source-trust-contract.sh b/ci/source-trust-contract.sh index 035d48c..5a941b7 100644 --- a/ci/source-trust-contract.sh +++ b/ci/source-trust-contract.sh @@ -3,6 +3,7 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq" tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index e014e07..7917933 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -172,14 +172,33 @@ failures remain available to the existing warning-level collector. Root systemd jobs enter through `/etc/cloud-compose/libexec`, validate the root-owned home scripts and control inputs, and only then execute their allowlisted `/home/cloud-compose` program. The privileged entrypoints, -diagnostics command, and checked-in `jq` programs live below -`/etc/cloud-compose/{libexec,bin,jq}` because COS permits cloud-init to rebuild +diagnostics command, and checked-in `jq` and AWK programs live below +`/etc/cloud-compose/{libexec,bin,jq,awk}` because COS permits cloud-init to rebuild that stateless tree while its `/usr` filesystem is immutable. Application services retain their unprivileged execution model. Compose diagnostics run as the application account and select the verified plugin copy below the executable data disk explicitly; they do not fall back to the compatibility copy below COS's `noexec` `/home` mount. +An installed script resolved below `/home/cloud-compose` always uses the fixed +`/etc/cloud-compose/jq` and `/etc/cloud-compose/awk` programs. Environment +overrides for those programs, the shared profile, key-rotation entrypoint, or +any other sourced runtime helper are rejected in installed mode. Repository +and CI fixtures may select checked programs only while the owning script resolves +outside the installed home. Before an installed program is consumed, its +canonical root-owned parent chain must contain no symlink or group/world-writable +directory. The only accepted logical alias is the operating-system-owned +`/home -> /var/home` link used by Fedora CoreOS. The program itself must be a +root-owned, single-link, non-writable regular file. Configuration-management +adapters make `/home/cloud-compose` root-owned immediately after copying rootfs, +before any root execution; mutable application state remains in its explicitly +account-owned child/data paths. A checked directory binding validates every +direct `jq` or AWK program in that installed directory, so adding a new filter +cannot bypass per-file ownership, link, and mode checks. Production rootfs shell +scripts invoke `jq` and AWK only with checked `-f` program files; the static CI +contract rejects embedded filters so review and integrity checks cannot be +bypassed by a later one-line program. + Terraform and cloud-init are transport and orchestration boundaries, not the home of bootstrap implementations. The GCP and provider-neutral templates transfer checked-in programs from `rootfs/etc/cloud-compose/libexec` and invoke @@ -197,7 +216,10 @@ root-controlled program files before they are sourced at their documented points in the bootstrap sequence. Keep substantive shell, Python, `jq`, and similar programs in reviewed files; do not interpolate them into Terraform heredocs, cloud-init command strings, Compose commands, or configuration- -management task bodies. +management task bodies. The early filesystem path stages its checked-in fstab +AWK program from the same Terraform source or already verified rootfs archive, +requires an unlinked root-owned mode-0600 file, and invokes it as data with +`awk -f` before the full stateless rootfs is available. ## Sitectl diff --git a/modules/digitalocean/runtime_inputs.tftest.hcl b/modules/digitalocean/runtime_inputs.tftest.hcl index d77207b..61ebac1 100644 --- a/modules/digitalocean/runtime_inputs.tftest.hcl +++ b/modules/digitalocean/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "digitalocean" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index 371b5cd..13fcfa2 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -448,6 +448,7 @@ EOT gcp_filesystem_boothook = templatefile("${path.module}/../../templates/gcp-filesystem-boothook.sh.tftpl", { FILESYSTEM_PREP_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/prepare-filesystem.sh"), FILESYSTEM_PERSIST_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/persist-filesystems.sh"), + FSTAB_RECONCILE_AWK_B64 = filebase64("${local.rootFs}/etc/cloud-compose/awk/reconcile-fstab.awk"), GCP_FILESYSTEM_BOOT_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/libexec/gcp-filesystem-boot.sh"), FRESH_FILESYSTEM_IDENTITY = "v1:gcp-disk-id:${google_compute_disk.data.disk_id}", USE_OVERLAY = local.use_overlay, diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index 9d60c72..bc2a53d 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "cloudinit" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } @@ -104,9 +104,17 @@ run "disables_privileged_services_by_default" { local.cloud_init_yaml, filebase64("${path.module}/../../rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh"), ) && + strcontains( + local.cloud_init_yaml, + filebase64("${path.module}/../../rootfs/etc/cloud-compose/awk/reconcile-fstab.awk"), + ) && + strcontains( + local.write_files_content, + "- path: \"/etc/cloud-compose/jq/compose-validate-projects.jq\"", + ) && strcontains(local.cloud_init_yaml, "Content-Type: text/cloud-boothook") ) - error_message = "GCP's early every-boot filesystem program and root runtime environment must carry the same immutable data-disk identity." + error_message = "GCP's early every-boot filesystem programs and root runtime environment must carry the same immutable data-disk identity and checked fstab reconciler." } assert { @@ -567,7 +575,7 @@ run "renders_verified_archive_before_downstream_overlay" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } @@ -752,7 +760,7 @@ run "rejects_archive_without_checksum" { override_data { target = data.http.rootfs_contract[0] values = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } diff --git a/modules/linode/runtime_inputs.tftest.hcl b/modules/linode/runtime_inputs.tftest.hcl index 69818ef..6f0b6ef 100644 --- a/modules/linode/runtime_inputs.tftest.hcl +++ b/modules/linode/runtime_inputs.tftest.hcl @@ -2,7 +2,7 @@ mock_provider "linode" {} mock_provider "http" { mock_data "http" { defaults = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl index 3333cb8..50c478f 100644 --- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl +++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl @@ -1,7 +1,7 @@ mock_provider "http" { mock_data "http" { defaults = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } @@ -44,9 +44,17 @@ run "renders_safe_ssh_values" { strcontains( local.write_files_content, "- path: \"/etc/cloud-compose/libexec/linux-vm-cloud-init.sh\"", + ) && + strcontains( + local.write_files_content, + "- path: \"/etc/cloud-compose/jq/compose-validate-projects.jq\"", + ) && + strcontains( + local.write_files_content, + "- path: \"/etc/cloud-compose/awk/reconcile-fstab.awk\"", ) ) - error_message = "Cloud-init must install one root-owned diagnostics program with exact passwordless sudo commands." + error_message = "Cloud-init must install root-owned diagnostics and checked jq/awk programs with exact passwordless sudo commands." } } diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl index 94bc740..a577a17 100644 --- a/providers/do/template_versions.tftest.hcl +++ b/providers/do/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } @@ -52,7 +52,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.digitalocean.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl index a61858c..d32bfc8 100644 --- a/providers/linode/template_versions.tftest.hcl +++ b/providers/linode/template_versions.tftest.hcl @@ -6,7 +6,7 @@ run "custom_package_set_merges_only_applicable_template_versions" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } @@ -57,7 +57,7 @@ run "explicit_core_only_package_set_disables_template_plugins" { override_data { target = module.linode.module.runtime.data.http.rootfs_contract[0] values = { - response_body = "379b1887c46f9e4aedf105a509ccc888338da9d80170e35f05e25f54dd826151\n" + response_body = "c33470299657aca69837d7ce2cee73659aa5fd9a3297dcaad4444b50b54cdde2\n" status_code = 200 } } diff --git a/rootfs/home/cloud-compose/compose-secret-files.awk b/rootfs/etc/cloud-compose/awk/compose-secret-files.awk similarity index 100% rename from rootfs/home/cloud-compose/compose-secret-files.awk rename to rootfs/etc/cloud-compose/awk/compose-secret-files.awk diff --git a/rootfs/etc/cloud-compose/awk/reconcile-fstab.awk b/rootfs/etc/cloud-compose/awk/reconcile-fstab.awk new file mode 100644 index 0000000..36b907c --- /dev/null +++ b/rootfs/etc/cloud-compose/awk/reconcile-fstab.awk @@ -0,0 +1,20 @@ +$0 == begin { managed = 1; next } +$0 == end { managed = 0; next } +!managed { + if (data_provider_mount != "" && $2 == data_provider_mount) { + if ($1 == data_device) next + conflict = 1 + } + if (volumes_provider_mount != "" && $2 == volumes_provider_mount) { + if ($1 == volumes_device) next + conflict = 1 + } + if ($2 == "/mnt/disks/data" || + $2 == "/mnt/disks/volumes" || + $2 == "/mnt/disks/data/docker/volumes" || + $2 == "/mnt/disks/prod-readonly") { + conflict = 1 + } + print +} +END { if (managed || conflict) exit 42 } diff --git a/rootfs/etc/cloud-compose/awk/release-checksum-entry.awk b/rootfs/etc/cloud-compose/awk/release-checksum-entry.awk new file mode 100644 index 0000000..9b68308 --- /dev/null +++ b/rootfs/etc/cloud-compose/awk/release-checksum-entry.awk @@ -0,0 +1 @@ +$2 == archive { print } diff --git a/rootfs/etc/cloud-compose/awk/release-checksum.awk b/rootfs/etc/cloud-compose/awk/release-checksum.awk new file mode 100644 index 0000000..a00540a --- /dev/null +++ b/rootfs/etc/cloud-compose/awk/release-checksum.awk @@ -0,0 +1,14 @@ +{ + filename = $2 + sub(/^\*/, "", filename) + if (filename == asset) { + checksum = $1 + matches++ + } +} +END { + if (matches != 1) { + exit 1 + } + print checksum +} diff --git a/rootfs/etc/cloud-compose/jq/application-env-validate.jq b/rootfs/etc/cloud-compose/jq/application-env-validate.jq new file mode 100644 index 0000000..6c8b533 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/application-env-validate.jq @@ -0,0 +1,6 @@ +type == "object" and +all(to_entries[]; + (.key | explode | index(0) == null) and + (.value | type == "string") and + (.value | explode | index(0) == null) +) diff --git a/rootfs/etc/cloud-compose/jq/array-values-base64.jq b/rootfs/etc/cloud-compose/jq/array-values-base64.jq new file mode 100644 index 0000000..fec3d58 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/array-values-base64.jq @@ -0,0 +1 @@ +.[] | @base64 diff --git a/rootfs/etc/cloud-compose/jq/compose-app-array.jq b/rootfs/etc/cloud-compose/jq/compose-app-array.jq new file mode 100644 index 0000000..bf25de2 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-app-array.jq @@ -0,0 +1,6 @@ +(.[$app][$field] // []) as $values | +if ($values | type) != "array" or any($values[]; type != "string") then + error("invalid string array") +else + $values +end diff --git a/rootfs/etc/cloud-compose/jq/compose-app-field.jq b/rootfs/etc/cloud-compose/jq/compose-app-field.jq new file mode 100644 index 0000000..e5aa9b8 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-app-field.jq @@ -0,0 +1,6 @@ +(.[$app][$field] // "" | tostring) as $value | +if $value | (explode | index(0) != null) or contains("\n") or contains("\r") then + error("invalid scalar field") +else + $value +end diff --git a/rootfs/etc/cloud-compose/jq/compose-app-ingress-array.jq b/rootfs/etc/cloud-compose/jq/compose-app-ingress-array.jq new file mode 100644 index 0000000..d7dfc3f --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-app-ingress-array.jq @@ -0,0 +1,6 @@ +(.[$app].ingress[$field] // []) as $values | +if ($values | type) != "array" or any($values[]; type != "string") then + error("invalid ingress string array") +else + $values +end diff --git a/rootfs/etc/cloud-compose/jq/compose-app-ingress-field.jq b/rootfs/etc/cloud-compose/jq/compose-app-ingress-field.jq new file mode 100644 index 0000000..8f9424c --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-app-ingress-field.jq @@ -0,0 +1,6 @@ +(.[$app].ingress[$field] // "" | tostring) as $value | +if $value | (explode | index(0) != null) or contains("\n") or contains("\r") then + error("invalid ingress scalar field") +else + $value +end diff --git a/rootfs/etc/cloud-compose/jq/compose-app-verify-args-json.jq b/rootfs/etc/cloud-compose/jq/compose-app-verify-args-json.jq new file mode 100644 index 0000000..272f433 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-app-verify-args-json.jq @@ -0,0 +1,8 @@ +(.[$app].sitectl_verify_args // []) as $values | +if ($values | type) != "array" or any($values[]; + type != "string" or (explode | index(0) != null) or contains("\n") or contains("\r") +) then + error("invalid verify args") +else + $values +end diff --git a/rootfs/etc/cloud-compose/jq/compose-app-verify-args.jq b/rootfs/etc/cloud-compose/jq/compose-app-verify-args.jq new file mode 100644 index 0000000..bb4a258 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-app-verify-args.jq @@ -0,0 +1,6 @@ +(.[$app].sitectl_verify_args // []) as $values | +if ($values | type) != "array" or any($values[]; type != "string") then + error("invalid verify args") +else + $values | join(" ") +end diff --git a/rootfs/etc/cloud-compose/jq/compose-project-dirs-base64.jq b/rootfs/etc/cloud-compose/jq/compose-project-dirs-base64.jq new file mode 100644 index 0000000..f9f67fe --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-project-dirs-base64.jq @@ -0,0 +1 @@ +.[] | .project_dir | @base64 diff --git a/rootfs/etc/cloud-compose/jq/compose-reject-host-network.jq b/rootfs/etc/cloud-compose/jq/compose-reject-host-network.jq new file mode 100644 index 0000000..c8d9c8d --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-reject-host-network.jq @@ -0,0 +1,12 @@ +any(.services[]; + (.network_mode // "") == "host" or + ( + (.build | type) == "object" and + ( + (.build.network // "") == "host" or + any((.build.entitlements // [])[]; + . == "network.host" or . == "security.insecure" + ) + ) + ) +) diff --git a/rootfs/etc/cloud-compose/jq/compose-services-object-validate.jq b/rootfs/etc/cloud-compose/jq/compose-services-object-validate.jq new file mode 100644 index 0000000..696aa4b --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-services-object-validate.jq @@ -0,0 +1 @@ +.services | type == "object" diff --git a/rootfs/etc/cloud-compose/jq/compose-validate-projects.jq b/rootfs/etc/cloud-compose/jq/compose-validate-projects.jq new file mode 100644 index 0000000..99e1efd --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/compose-validate-projects.jq @@ -0,0 +1,26 @@ +type == "object" and length > 0 and +all(to_entries[]; . as $entry | + ($entry.key | explode | index(0) == null) and + ($entry.value | type == "object") and + (all($entry.value | .. | select(type == "string"); + explode | index(0) == null)) and + ($entry.value.docker_compose_repo | type == "string" and length > 0) and + ($entry.value.docker_compose_branch | type == "string" and length > 0) and + ($entry.value.project_dir | type == "string" and length > 0) and + ($entry.value.compose_project_name | type == "string" and length > 0) and + (all(["init_commands", "up_commands", "down_commands", "rollout_commands"][]; + . as $field | + ($entry.value[$field] == null) or + (($entry.value[$field] | type) == "array" and + all($entry.value[$field][]; type == "string")) + )) and + (($entry.value.sitectl_verify_args == null) or + (($entry.value.sitectl_verify_args | type) == "array" and + all($entry.value.sitectl_verify_args[]; + type == "string" and + (explode | index(0) == null) and + (contains("\n") | not) and + (contains("\r") | not) + ))) and + (($entry.value.ingress == null) or ($entry.value.ingress | type == "object")) +) diff --git a/rootfs/etc/cloud-compose/jq/gcp-metadata-private-ip.jq b/rootfs/etc/cloud-compose/jq/gcp-metadata-private-ip.jq new file mode 100644 index 0000000..4693313 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/gcp-metadata-private-ip.jq @@ -0,0 +1 @@ +.instance.networkInterfaces[0].ip diff --git a/rootfs/etc/cloud-compose/jq/gcp-metadata-public-ip.jq b/rootfs/etc/cloud-compose/jq/gcp-metadata-public-ip.jq new file mode 100644 index 0000000..2a84839 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/gcp-metadata-public-ip.jq @@ -0,0 +1 @@ +.instance.networkInterfaces[0].accessConfigs[0].externalIp diff --git a/rootfs/etc/cloud-compose/jq/github-latest-release-tag.jq b/rootfs/etc/cloud-compose/jq/github-latest-release-tag.jq new file mode 100644 index 0000000..42f0fd4 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/github-latest-release-tag.jq @@ -0,0 +1,2 @@ +(.tag_name | select(type == "string" and length > 0 and (explode | index(0) == null))), +"\u001f" diff --git a/rootfs/etc/cloud-compose/jq/json-length.jq b/rootfs/etc/cloud-compose/jq/json-length.jq new file mode 100644 index 0000000..7f5e3b6 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/json-length.jq @@ -0,0 +1 @@ +length diff --git a/rootfs/etc/cloud-compose/jq/json-object-validate.jq b/rootfs/etc/cloud-compose/jq/json-object-validate.jq new file mode 100644 index 0000000..a25876a --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/json-object-validate.jq @@ -0,0 +1 @@ +type == "object" diff --git a/rootfs/etc/cloud-compose/jq/nonempty-string-field.jq b/rootfs/etc/cloud-compose/jq/nonempty-string-field.jq new file mode 100644 index 0000000..58b565d --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/nonempty-string-field.jq @@ -0,0 +1 @@ +.[$field] | select(type == "string" and length > 0) diff --git a/rootfs/etc/cloud-compose/jq/object-entries-sorted-base64.jq b/rootfs/etc/cloud-compose/jq/object-entries-sorted-base64.jq new file mode 100644 index 0000000..617be57 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/object-entries-sorted-base64.jq @@ -0,0 +1 @@ +to_entries | sort_by(.key)[] | @base64 diff --git a/rootfs/etc/cloud-compose/jq/object-field-delimited.jq b/rootfs/etc/cloud-compose/jq/object-field-delimited.jq new file mode 100644 index 0000000..d1b8407 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/object-field-delimited.jq @@ -0,0 +1 @@ +(.[$field]), "\u001f" diff --git a/rootfs/etc/cloud-compose/jq/object-field.jq b/rootfs/etc/cloud-compose/jq/object-field.jq new file mode 100644 index 0000000..fff061d --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/object-field.jq @@ -0,0 +1 @@ +.[$field] diff --git a/rootfs/etc/cloud-compose/jq/object-has-key.jq b/rootfs/etc/cloud-compose/jq/object-has-key.jq new file mode 100644 index 0000000..f9943a1 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/object-has-key.jq @@ -0,0 +1 @@ +has($key) diff --git a/rootfs/etc/cloud-compose/jq/object-keys-base64.jq b/rootfs/etc/cloud-compose/jq/object-keys-base64.jq new file mode 100644 index 0000000..2975208 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/object-keys-base64.jq @@ -0,0 +1 @@ +keys[] | @base64 diff --git a/rootfs/etc/cloud-compose/jq/object-keys.jq b/rootfs/etc/cloud-compose/jq/object-keys.jq new file mode 100644 index 0000000..d7d2124 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/object-keys.jq @@ -0,0 +1 @@ +keys[] diff --git a/rootfs/etc/cloud-compose/jq/rotation-audit.jq b/rootfs/etc/cloud-compose/jq/rotation-audit.jq new file mode 100644 index 0000000..6eeb3a4 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-audit.jq @@ -0,0 +1,12 @@ +{ + version: 1, + phase: $phase, + current_key_id: $current_key_id, + new_key_id: $new_key_id, + recovery_required: ($phase == "creating" or $phase == "creating-fresh"), + candidate_key_ids: [$candidate_names[] | split("/")[-1]], + created_at: $created_at, + ready_at: $ready_at, + disabled_at: $disabled_at, + grace_remaining_seconds: $grace_remaining +} diff --git a/rootfs/etc/cloud-compose/jq/rotation-build-state.jq b/rootfs/etc/cloud-compose/jq/rotation-build-state.jq new file mode 100644 index 0000000..fd619a2 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-build-state.jq @@ -0,0 +1,14 @@ +{ + version: 2, + phase: $phase, + service_account: $service_account, + project_id: $project_id, + credentials_file: $credentials_file, + current_key_id: $current_key_id, + new_key_id: $new_key_id, + new_key_name: $new_key_name, + baseline_key_names: $baseline_key_names, + created_at: $created_at, + ready_at: $ready_at, + disabled_at: $disabled_at +} diff --git a/rootfs/etc/cloud-compose/jq/rotation-first-key-id.jq b/rootfs/etc/cloud-compose/jq/rotation-first-key-id.jq new file mode 100644 index 0000000..34529ca --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-first-key-id.jq @@ -0,0 +1 @@ +.[0] | split("/")[-1] diff --git a/rootfs/etc/cloud-compose/jq/rotation-idle-audit.jq b/rootfs/etc/cloud-compose/jq/rotation-idle-audit.jq new file mode 100644 index 0000000..9c9b20e --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-idle-audit.jq @@ -0,0 +1,7 @@ +{ + version: 1, + phase: "idle", + recovery_required: false, + candidate_key_ids: [], + grace_remaining_seconds: 0 +} diff --git a/rootfs/etc/cloud-compose/jq/rotation-jwt-claims.jq b/rootfs/etc/cloud-compose/jq/rotation-jwt-claims.jq new file mode 100644 index 0000000..e431aa8 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-jwt-claims.jq @@ -0,0 +1,7 @@ +{ + iss: $iss, + scope: "https://www.googleapis.com/auth/cloud-platform", + aud: $aud, + iat: $iat, + exp: ($iat + 3600) +} diff --git a/rootfs/etc/cloud-compose/jq/rotation-key-disabled.jq b/rootfs/etc/cloud-compose/jq/rotation-key-disabled.jq new file mode 100644 index 0000000..9ce9b69 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-key-disabled.jq @@ -0,0 +1 @@ +any(.[]; .name == $name and .disabled == true) diff --git a/rootfs/etc/cloud-compose/jq/rotation-key-ids-join.jq b/rootfs/etc/cloud-compose/jq/rotation-key-ids-join.jq new file mode 100644 index 0000000..1ede644 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-key-ids-join.jq @@ -0,0 +1 @@ +[.[] | split("/")[-1]] | join(", ") diff --git a/rootfs/etc/cloud-compose/jq/rotation-key-names-base64.jq b/rootfs/etc/cloud-compose/jq/rotation-key-names-base64.jq new file mode 100644 index 0000000..9ee5619 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-key-names-base64.jq @@ -0,0 +1 @@ +.[].name | @base64 diff --git a/rootfs/etc/cloud-compose/jq/rotation-key-names-sorted.jq b/rootfs/etc/cloud-compose/jq/rotation-key-names-sorted.jq new file mode 100644 index 0000000..7704945 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-key-names-sorted.jq @@ -0,0 +1 @@ +[.[].name] | sort diff --git a/rootfs/etc/cloud-compose/jq/rotation-key-object-ids-join.jq b/rootfs/etc/cloud-compose/jq/rotation-key-object-ids-join.jq new file mode 100644 index 0000000..d9ce6ea --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-key-object-ids-join.jq @@ -0,0 +1 @@ +[.[].name | split("/")[-1]] | sort | join(", ") diff --git a/rootfs/etc/cloud-compose/jq/rotation-key-present.jq b/rootfs/etc/cloud-compose/jq/rotation-key-present.jq new file mode 100644 index 0000000..cd6febc --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-key-present.jq @@ -0,0 +1 @@ +any(.[]; .name == $name) diff --git a/rootfs/etc/cloud-compose/jq/rotation-new-key-names.jq b/rootfs/etc/cloud-compose/jq/rotation-new-key-names.jq new file mode 100644 index 0000000..60a7881 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-new-key-names.jq @@ -0,0 +1 @@ +$after - $before | sort diff --git a/rootfs/etc/cloud-compose/jq/rotation-normalize-user-keys.jq b/rootfs/etc/cloud-compose/jq/rotation-normalize-user-keys.jq new file mode 100644 index 0000000..685da02 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-normalize-user-keys.jq @@ -0,0 +1,15 @@ +(.keys // []) as $keys | +if ($keys | type) != "array" then error("invalid key list") else + [$keys[] | select(.keyType == "USER_MANAGED")] as $user_keys | + if ($user_keys | length) > 10 then error("too many user-managed keys") + elif any($user_keys[]; + (.name | type) != "string" or + (.name | startswith($prefix) | not) or + ((.disabled // false) | type) != "boolean") + then error("invalid user-managed key") + elif ([$user_keys[].name] | unique | length) != ($user_keys | length) + then error("duplicate user-managed key") + else + [$user_keys[] | {name: .name, disabled: (.disabled // false)}] | sort_by(.name) + end +end diff --git a/rootfs/etc/cloud-compose/jq/rotation-remaining-baseline.jq b/rootfs/etc/cloud-compose/jq/rotation-remaining-baseline.jq new file mode 100644 index 0000000..cd22ed3 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-remaining-baseline.jq @@ -0,0 +1 @@ +$after | map(select(. as $name | $before | index($name))) | sort diff --git a/rootfs/etc/cloud-compose/jq/rotation-validate-state.jq b/rootfs/etc/cloud-compose/jq/rotation-validate-state.jq new file mode 100644 index 0000000..cf4b1fa --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/rotation-validate-state.jq @@ -0,0 +1,20 @@ +select( + .version == 2 and + (.phase == "reconciling" or .phase == "creating-fresh" or + .phase == "creating" or .phase == "staged" or .phase == "authenticated" or + .phase == "ready" or .phase == "grace" or .phase == "rolling-back" or + .phase == "rollback" or .phase == "revoke-new") and + .service_account == $service_account and + .project_id == $project_id and + .credentials_file == $credentials_file and + (.current_key_id | type == "string" and (explode | index(0) == null)) and + (.new_key_id | type == "string" and (explode | index(0) == null)) and + (.new_key_name | type == "string" and (explode | index(0) == null)) and + (.baseline_key_names | type == "array") and + (.baseline_key_names | length <= 10 and . == (sort | unique)) and + all(.baseline_key_names[]; + type == "string" and (explode | index(0) == null)) and + (.created_at | type == "number" and . >= 0 and floor == .) and + (.ready_at | type == "number" and . >= 0 and floor == .) and + (.disabled_at | type == "number" and . >= 0 and floor == .) +) diff --git a/rootfs/etc/cloud-compose/jq/service-account-credentials-valid.jq b/rootfs/etc/cloud-compose/jq/service-account-credentials-valid.jq new file mode 100644 index 0000000..cccc453 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/service-account-credentials-valid.jq @@ -0,0 +1,8 @@ +.type == "service_account" and +.private_key_id == $key_id and +.client_email == $service_account and +.project_id == $project_id and +.token_uri == "https://oauth2.googleapis.com/token" and +(.private_key | type == "string" and + startswith("-----BEGIN PRIVATE KEY-----") and + contains("-----END PRIVATE KEY-----")) diff --git a/rootfs/etc/cloud-compose/jq/service-account-key-id.jq b/rootfs/etc/cloud-compose/jq/service-account-key-id.jq new file mode 100644 index 0000000..5539109 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/service-account-key-id.jq @@ -0,0 +1,3 @@ +(.private_key_id | + select(type == "string" and length > 0 and (explode | index(0) == null))), +"\u001f" diff --git a/rootfs/etc/cloud-compose/jq/sitectl-package-version.jq b/rootfs/etc/cloud-compose/jq/sitectl-package-version.jq new file mode 100644 index 0000000..42eaeb7 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/sitectl-package-version.jq @@ -0,0 +1 @@ +.[$package] // $fallback diff --git a/rootfs/etc/cloud-compose/jq/sitectl-package-versions-validate.jq b/rootfs/etc/cloud-compose/jq/sitectl-package-versions-validate.jq new file mode 100644 index 0000000..6c8b533 --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/sitectl-package-versions-validate.jq @@ -0,0 +1,6 @@ +type == "object" and +all(to_entries[]; + (.key | explode | index(0) == null) and + (.value | type == "string") and + (.value | explode | index(0) == null) +) diff --git a/rootfs/etc/cloud-compose/jq/string-array-validate.jq b/rootfs/etc/cloud-compose/jq/string-array-validate.jq new file mode 100644 index 0000000..5083d4c --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/string-array-validate.jq @@ -0,0 +1 @@ +type == "array" and all(.[]; type == "string") diff --git a/rootfs/etc/cloud-compose/jq/string-field-valid.jq b/rootfs/etc/cloud-compose/jq/string-field-valid.jq new file mode 100644 index 0000000..60c5b0d --- /dev/null +++ b/rootfs/etc/cloud-compose/jq/string-field-valid.jq @@ -0,0 +1 @@ +.[$field] | type == "string" and length > 0 diff --git a/rootfs/etc/cloud-compose/libexec/checked-programs.bash b/rootfs/etc/cloud-compose/libexec/checked-programs.bash new file mode 100644 index 0000000..c338767 --- /dev/null +++ b/rootfs/etc/cloud-compose/libexec/checked-programs.bash @@ -0,0 +1,271 @@ +# Shared trust boundary for jq, AWK, and sourced shell programs consumed by Cloud Compose. +# This file is sourced from checked runtime scripts; it is not an entrypoint. + +cloud_compose_program_owner_path() { + local owner_source="$1" + + readlink -f -- "$owner_source" +} + +cloud_compose_validate_root_parent() { + local current="$1" metadata owner group mode + + if [[ -L "$current" || ! -d "$current" ]]; then + echo "Installed Cloud Compose program parent is missing or redirected: $current" >&2 + return 1 + fi + metadata="$(stat -c '%u:%g:%a' -- "$current")" || return 1 + IFS=: read -r owner group mode <<<"$metadata" + if [[ "$owner" != 0 || "$group" != 0 || ! "$mode" =~ ^[0-7]{3,4}$ || + $((8#$mode & 0022)) -ne 0 ]]; then + echo "Installed Cloud Compose program parent is not root-controlled: $current" >&2 + return 1 + fi +} + +cloud_compose_installed_home() { + local resolved_home alias_metadata alias_target + local -a parents + + if [[ -L /home ]]; then + alias_metadata="$(stat -c '%u:%g:%h' -- /home)" || return 1 + alias_target="$(readlink -- /home)" || return 1 + if [[ "$alias_metadata" != "0:0:1" || + ( "$alias_target" != "var/home" && "$alias_target" != "/var/home" ) ]]; then + echo "Installed Cloud Compose home uses an unsafe operating-system alias" >&2 + return 1 + fi + elif [[ ! -d /home ]]; then + echo "Installed Cloud Compose home parent is missing" >&2 + return 1 + fi + + resolved_home="$(readlink -f -- /home)" || return 1 + case "$resolved_home" in + /home) parents=(/ /home /home/cloud-compose) ;; + /var/home) parents=(/ /var /var/home /var/home/cloud-compose) ;; + *) + echo "Installed Cloud Compose home resolves outside a supported root-controlled path" >&2 + return 1 + ;; + esac + for parent in "${parents[@]}"; do + cloud_compose_validate_root_parent "$parent" || return 1 + done + printf '%s/cloud-compose\n' "$resolved_home" +} + +cloud_compose_owner_is_installed() { + local owner_path="$1" candidate_home installed_home + + candidate_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" + if [[ "$candidate_home" == "/" ]]; then + echo "Installed Cloud Compose home resolves to the filesystem root" >&2 + return 2 + fi + if [[ -z "$candidate_home" || + "$owner_path" != "${candidate_home%/}/"* ]]; then + return 1 + fi + installed_home="$(cloud_compose_installed_home)" || return 2 + [[ "$candidate_home" == "$installed_home" ]] || return 2 +} + +cloud_compose_validate_installed_program() { + local program="$1" program_dir="$2" program_name current metadata owner group mode links + local -a parents + + case "$program_dir" in + /etc/cloud-compose/jq | /etc/cloud-compose/awk) ;; + *) + echo "Unsupported installed Cloud Compose program directory: $program_dir" >&2 + return 1 + ;; + esac + program_name="${program##*/}" + [[ "$program" == "$program_dir/$program_name" && + "$program_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*\.(jq|awk)$ && + "$program" != *$'\n'* && + "$program" != *$'\r'* && ! "$program" =~ (^|/)\.\.?(/|$) ]] || { + echo "Unsafe installed Cloud Compose program path: $program" >&2 + return 1 + } + + parents=(/ /etc /etc/cloud-compose "$program_dir") + for current in "${parents[@]}"; do + cloud_compose_validate_root_parent "$current" || return 1 + done + + if [[ -L "$program" || ! -f "$program" ]]; then + echo "Installed Cloud Compose program is missing or redirected: $program" >&2 + return 1 + fi + metadata="$(stat -c '%u:%g:%a:%h' -- "$program")" || return 1 + IFS=: read -r owner group mode links <<<"$metadata" + if [[ "$owner" != 0 || "$group" != 0 || "$links" != 1 || + ! "$mode" =~ ^[0-7]{3,4}$ || + $((8#$mode & 0022)) -ne 0 ]]; then + echo "Installed Cloud Compose program is not a single-link, root-controlled regular file: $program" >&2 + return 1 + fi +} + +cloud_compose_validate_installed_program_dir() ( + local program_dir="$1" program + local -a programs + + shopt -s nullglob dotglob + case "$program_dir" in + /etc/cloud-compose/jq) programs=("$program_dir"/*.jq) ;; + /etc/cloud-compose/awk) programs=("$program_dir"/*.awk) ;; + *) + echo "Unsupported installed Cloud Compose program directory: $program_dir" >&2 + return 1 + ;; + esac + ((${#programs[@]} > 0)) || { + echo "Installed Cloud Compose program directory is empty: $program_dir" >&2 + return 1 + } + for program in "${programs[@]}"; do + cloud_compose_validate_installed_program "$program" "$program_dir" || return 1 + done +) + +cloud_compose_validate_installed_source() { + local program="$1" resolved_program installed_home metadata owner group mode links + + [[ "$program" =~ ^/home/cloud-compose/[A-Za-z0-9][A-Za-z0-9._-]*\.sh$ ]] || { + echo "Unsafe installed Cloud Compose source program path: $program" >&2 + return 1 + } + installed_home="$(cloud_compose_installed_home)" || return 1 + resolved_program="$(readlink -f -- "$program")" || return 1 + [[ "$resolved_program" == "$installed_home/${program##*/}" ]] || { + echo "Installed Cloud Compose source program resolves outside its fixed home" >&2 + return 1 + } + if [[ -L "$program" || ! -f "$program" ]]; then + echo "Installed Cloud Compose source program is missing or redirected: $program" >&2 + return 1 + fi + metadata="$(stat -c '%u:%g:%a:%h' -- "$program")" || return 1 + IFS=: read -r owner group mode links <<<"$metadata" + if [[ "$owner" != 0 || "$group" != 0 || "$links" != 1 || + ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then + echo "Installed Cloud Compose source program is not a single-link, root-controlled regular file: $program" >&2 + return 1 + fi +} + +cloud_compose_bind_program_dir() { + local owner_source="$1" variable_name="$2" installed_dir="$3" repository_dir="$4" + shift 4 + local owner_path selected name program installed_status=0 + + owner_path="$(cloud_compose_program_owner_path "$owner_source")" || return 1 + if cloud_compose_owner_is_installed "$owner_path"; then + installed_status=0 + else + installed_status=$? + fi + if ((installed_status > 1)); then + return 1 + fi + if ((installed_status == 0)); then + if [[ -v $variable_name && "${!variable_name}" != "$installed_dir" ]]; then + echo "$variable_name cannot override the installed Cloud Compose program directory" >&2 + return 1 + fi + selected="$installed_dir" + cloud_compose_validate_installed_program_dir "$selected" || return 1 + for name in "$@"; do + [[ "$name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*\.(jq|awk)$ ]] || { + echo "Unsafe Cloud Compose program name: $name" >&2 + return 1 + } + program="$selected/$name" + cloud_compose_validate_installed_program "$program" "$selected" || return 1 + done + if [[ ! -v $variable_name ]]; then + printf -v "$variable_name" '%s' "$selected" + fi + readonly "$variable_name" + else + if [[ -v $variable_name && -n "${!variable_name}" ]]; then + selected="${!variable_name}" + else + selected="$repository_dir" + fi + printf -v "$variable_name" '%s' "$selected" + fi +} + +cloud_compose_bind_program() { + local owner_source="$1" variable_name="$2" installed_program="$3" repository_program="$4" + local owner_path selected program_dir installed_status=0 + + owner_path="$(cloud_compose_program_owner_path "$owner_source")" || return 1 + if cloud_compose_owner_is_installed "$owner_path"; then + installed_status=0 + else + installed_status=$? + fi + if ((installed_status > 1)); then + return 1 + fi + if ((installed_status == 0)); then + if [[ -v $variable_name && "${!variable_name}" != "$installed_program" ]]; then + echo "$variable_name cannot override the installed Cloud Compose program" >&2 + return 1 + fi + selected="$installed_program" + program_dir="${installed_program%/*}" + cloud_compose_validate_installed_program "$selected" "$program_dir" || return 1 + if [[ ! -v $variable_name ]]; then + printf -v "$variable_name" '%s' "$selected" + fi + readonly "$variable_name" + else + if [[ -v $variable_name && -n "${!variable_name}" ]]; then + selected="${!variable_name}" + else + selected="$repository_program" + fi + printf -v "$variable_name" '%s' "$selected" + fi +} + +cloud_compose_bind_source_program() { + local owner_source="$1" variable_name="$2" installed_program="$3" repository_program="$4" + local owner_path selected installed_status=0 + + owner_path="$(cloud_compose_program_owner_path "$owner_source")" || return 1 + if cloud_compose_owner_is_installed "$owner_path"; then + installed_status=0 + else + installed_status=$? + fi + if ((installed_status > 1)); then + return 1 + fi + if ((installed_status == 0)); then + if [[ -v $variable_name && "${!variable_name}" != "$installed_program" ]]; then + echo "$variable_name cannot override the installed Cloud Compose source program" >&2 + return 1 + fi + selected="$installed_program" + cloud_compose_validate_installed_source "$selected" || return 1 + if [[ ! -v $variable_name ]]; then + printf -v "$variable_name" '%s' "$selected" + fi + readonly "$variable_name" + else + if [[ -v $variable_name && -n "${!variable_name}" ]]; then + selected="${!variable_name}" + else + selected="$repository_program" + fi + printf -v "$variable_name" '%s' "$selected" + fi +} diff --git a/rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh b/rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh index 8d5a373..55a7904 100755 --- a/rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh +++ b/rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh @@ -2,8 +2,8 @@ set -euo pipefail -if [[ "$#" -ne 4 ]]; then - echo "usage: gcp-filesystem-boot.sh FRESH_FILESYSTEM_IDENTITY USE_OVERLAY PREP_PROGRAM PERSIST_PROGRAM" >&2 +if [[ "$#" -ne 5 ]]; then + echo "usage: gcp-filesystem-boot.sh FRESH_FILESYSTEM_IDENTITY USE_OVERLAY PREP_PROGRAM PERSIST_PROGRAM FSTAB_RECONCILE_PROGRAM" >&2 exit 2 fi @@ -11,6 +11,21 @@ fresh_filesystem_identity="$1" use_overlay="$2" filesystem_prep="$3" filesystem_persist="$4" +filesystem_reconcile="$5" + +require_root_owned_data_program() { + local path="$1" metadata + + if [[ -L "$path" || ! -f "$path" ]]; then + echo "Checked filesystem data program is missing or unsafe: $path" >&2 + return 1 + fi + metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$path")" || return 1 + if [[ "$metadata" != "0:0:600:1:regular file" ]]; then + echo "Checked filesystem data program is not an unlinked root-owned mode-0600 file: $path" >&2 + return 1 + fi +} case "$use_overlay" in true | false) ;; @@ -20,6 +35,8 @@ case "$use_overlay" in ;; esac +require_root_owned_data_program "$filesystem_reconcile" + rm -f /run/cloud-compose-filesystems-ready bash "$filesystem_prep" /dev/disk/by-id/google-data /mnt/disks/data \ --publish-fresh-marker "$fresh_filesystem_identity" @@ -41,12 +58,12 @@ if [[ "$use_overlay" == "true" ]]; then mount -o ro "$(readlink -f /dev/disk/by-id/google-prod-volumes)" \ /mnt/disks/prod-readonly fi - bash "$filesystem_persist" \ + CLOUD_COMPOSE_FSTAB_RECONCILE_PROGRAM="$filesystem_reconcile" bash "$filesystem_persist" \ /dev/disk/by-id/google-data \ /dev/disk/by-id/google-docker-volumes \ /dev/disk/by-id/google-prod-volumes else - bash "$filesystem_persist" \ + CLOUD_COMPOSE_FSTAB_RECONCILE_PROGRAM="$filesystem_reconcile" bash "$filesystem_persist" \ /dev/disk/by-id/google-data \ /dev/disk/by-id/google-docker-volumes fi diff --git a/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh b/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh index 7d921dd..68375ec 100755 --- a/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh +++ b/rootfs/etc/cloud-compose/libexec/linux-vm-cloud-init.sh @@ -17,6 +17,21 @@ readonly archive_program="$bootstrap_dir/rootfs-archive.sh" readonly overlay_dir=/var/lib/cloud-compose/rootfs-overlay readonly filesystem_prep=/run/cloud-compose-prepare-filesystem readonly filesystem_persist=/run/cloud-compose-persist-filesystems +readonly filesystem_reconcile=/run/cloud-compose-reconcile-fstab.awk + +require_root_owned_data_program() { + local path="$1" metadata + + if [[ -L "$path" || ! -f "$path" ]]; then + echo "Checked filesystem data program is missing or unsafe: $path" >&2 + return 1 + fi + metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$path")" || return 1 + if [[ "$metadata" != "0:0:600:1:regular file" ]]; then + echo "Checked filesystem data program is not an unlinked root-owned mode-0600 file: $path" >&2 + return 1 + fi +} for boolean_name in rootfs_archive_enabled rollout_enabled; do case "${!boolean_name}" in @@ -46,8 +61,11 @@ fi if [[ "$rootfs_archive_enabled" == "false" ]]; then install -m 0600 -- /home/cloud-compose/prepare-filesystem.sh "$filesystem_prep" install -m 0600 -- /home/cloud-compose/persist-filesystems.sh "$filesystem_persist" + install -m 0600 -- /etc/cloud-compose/awk/reconcile-fstab.awk "$filesystem_reconcile" fi +require_root_owned_data_program "$filesystem_reconcile" + bash "$filesystem_prep" "$data_device" /mnt/disks/data --publish-fresh-marker bash "$filesystem_prep" "$volumes_device" /mnt/disks/volumes mkdir -p /mnt/disks/data/docker/volumes @@ -60,7 +78,8 @@ for required_mount in /mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/ exit 1 fi done -bash "$filesystem_persist" "$data_device" "$volumes_device" +CLOUD_COMPOSE_FSTAB_RECONCILE_PROGRAM="$filesystem_reconcile" \ + bash "$filesystem_persist" "$data_device" "$volumes_device" if [[ "$rootfs_archive_enabled" == "true" ]]; then bash "$archive_program" install-staged "$overlay_dir" diff --git a/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh b/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh index 79955f1..ff361db 100755 --- a/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh +++ b/rootfs/etc/cloud-compose/libexec/rootfs-archive.sh @@ -7,6 +7,7 @@ readonly staged_rootfs="$stage_root/rootfs" readonly bootstrap_dir=/var/lib/cloud-compose/bootstrap readonly filesystem_prep=/run/cloud-compose-prepare-filesystem readonly filesystem_persist=/run/cloud-compose-persist-filesystems +readonly filesystem_reconcile=/run/cloud-compose-reconcile-fstab.awk fail() { echo "rootfs archive: $*" >&2 @@ -232,14 +233,18 @@ prepare_archive() { rm -rf -- "$extract_dir" [[ -f "$staged_rootfs/home/cloud-compose/prepare-filesystem.sh" && - -f "$staged_rootfs/home/cloud-compose/persist-filesystems.sh" ]] || \ - fail "verified rootfs archive is missing filesystem preparation scripts" + -f "$staged_rootfs/home/cloud-compose/persist-filesystems.sh" && + -f "$staged_rootfs/etc/cloud-compose/awk/reconcile-fstab.awk" ]] || \ + fail "verified rootfs archive is missing filesystem preparation programs" install -m 0600 -- \ "$staged_rootfs/home/cloud-compose/prepare-filesystem.sh" \ "$filesystem_prep" install -m 0600 -- \ "$staged_rootfs/home/cloud-compose/persist-filesystems.sh" \ "$filesystem_persist" + install -m 0600 -- \ + "$staged_rootfs/etc/cloud-compose/awk/reconcile-fstab.awk" \ + "$filesystem_reconcile" } install_staged_archive() { diff --git a/rootfs/home/cloud-compose/assert-app-initialized.sh b/rootfs/home/cloud-compose/assert-app-initialized.sh index 276e4aa..bf08842 100644 --- a/rootfs/home/cloud-compose/assert-app-initialized.sh +++ b/rootfs/home/cloud-compose/assert-app-initialized.sh @@ -2,8 +2,28 @@ set -euo pipefail +_cc_assert_initialized_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_assert_initialized_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_assert_initialized_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_assert_initialized_source _cc_assert_initialized_dir _cc_assert_initialized_installed_home +if [[ -n "$_cc_assert_initialized_installed_home" && + ( "$_cc_assert_initialized_installed_home" == "/" || + "$_cc_assert_initialized_source" == "${_cc_assert_initialized_installed_home%/}/"* ) ]]; then + _cc_assert_initialized_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_assert_initialized_checked_programs="$_cc_assert_initialized_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_assert_initialized_checked_programs +# shellcheck disable=SC1090 +source "$_cc_assert_initialized_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_assert_initialized_source" CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH \ + /home/cloud-compose/bootstrap-helpers.sh "$_cc_assert_initialized_dir/bootstrap-helpers.sh" +bootstrap_helpers_path="$CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH" +readonly bootstrap_helpers_path + # shellcheck disable=SC1090 -source "${CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH:-/home/cloud-compose/bootstrap-helpers.sh}" +source "$bootstrap_helpers_path" durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/var/lib/cloud-compose/bootstrap-complete}" boot_marker="${CLOUD_COMPOSE_APP_INIT_MARKER:-/run/cloud-compose-app-init-complete}" diff --git a/rootfs/home/cloud-compose/assert-vault-ready.sh b/rootfs/home/cloud-compose/assert-vault-ready.sh index 4662a49..33f3f48 100644 --- a/rootfs/home/cloud-compose/assert-vault-ready.sh +++ b/rootfs/home/cloud-compose/assert-vault-ready.sh @@ -2,7 +2,28 @@ set -euo pipefail -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-/home/cloud-compose/profile.sh}" +_cc_assert_vault_ready_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_assert_vault_ready_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_assert_vault_ready_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_assert_vault_ready_source _cc_assert_vault_ready_dir _cc_assert_vault_ready_installed_home +if [[ -n "$_cc_assert_vault_ready_installed_home" && + ( "$_cc_assert_vault_ready_installed_home" == "/" || + "$_cc_assert_vault_ready_source" == "${_cc_assert_vault_ready_installed_home%/}/"* ) ]]; then + _cc_assert_vault_ready_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_assert_vault_ready_checked_programs="$_cc_assert_vault_ready_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_assert_vault_ready_checked_programs +# shellcheck disable=SC1090 +source "$_cc_assert_vault_ready_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_assert_vault_ready_source" \ + CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh \ + "$_cc_assert_vault_ready_dir/profile.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +readonly profile_path + # shellcheck disable=SC1090 source "$profile_path" diff --git a/rootfs/home/cloud-compose/compose-apps.sh b/rootfs/home/cloud-compose/compose-apps.sh index d0efb89..9b67610 100644 --- a/rootfs/home/cloud-compose/compose-apps.sh +++ b/rootfs/home/cloud-compose/compose-apps.sh @@ -6,8 +6,47 @@ COMPOSE_PROJECTS_FILE="${COMPOSE_PROJECTS_FILE:-/home/cloud-compose/compose-proj COMPOSE_APPS_ENV_DIR="${COMPOSE_APPS_ENV_DIR:-/home/cloud-compose/apps}" COMPOSE_APPS_STATE_DIR="${COMPOSE_APPS_STATE_DIR:-/home/cloud-compose/state}" CLOUD_COMPOSE_DATA_ROOT="${CLOUD_COMPOSE_DATA_ROOT:-/mnt/disks/data}" -compose_apps_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -COMPOSE_SECRET_FILES_PROGRAM="${CLOUD_COMPOSE_COMPOSE_SECRET_FILES_PROGRAM:-$compose_apps_dir/compose-secret-files.awk}" +_cc_compose_apps_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_compose_apps_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_compose_apps_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_compose_apps_source _cc_compose_apps_dir _cc_compose_apps_installed_home +if [[ -n "$_cc_compose_apps_installed_home" && + ( "$_cc_compose_apps_installed_home" == "/" || + "$_cc_compose_apps_source" == "${_cc_compose_apps_installed_home%/}/"* ) ]]; then + _cc_compose_apps_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_compose_apps_checked_programs="$_cc_compose_apps_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_compose_apps_checked_programs +# shellcheck disable=SC1090 +source "$_cc_compose_apps_checked_programs" +cloud_compose_bind_program_dir \ + "$_cc_compose_apps_source" \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR \ + /etc/cloud-compose/jq \ + "$_cc_compose_apps_dir/../../etc/cloud-compose/jq" \ + compose-validate-projects.jq \ + compose-app-field.jq \ + compose-app-array.jq \ + compose-app-verify-args.jq \ + compose-app-verify-args-json.jq \ + compose-app-ingress-field.jq \ + compose-app-ingress-array.jq \ + compose-reject-host-network.jq \ + compose-project-dirs-base64.jq \ + compose-services-object-validate.jq \ + array-values-base64.jq \ + object-has-key.jq \ + object-keys-base64.jq \ + object-keys.jq \ + string-array-validate.jq +cloud_compose_bind_program \ + "$_cc_compose_apps_source" \ + CLOUD_COMPOSE_COMPOSE_SECRET_FILES_PROGRAM \ + /etc/cloud-compose/awk/compose-secret-files.awk \ + "$_cc_compose_apps_dir/../../etc/cloud-compose/awk/compose-secret-files.awk" +COMPOSE_SECRET_FILES_PROGRAM="$CLOUD_COMPOSE_COMPOSE_SECRET_FILES_PROGRAM" +readonly COMPOSE_SECRET_FILES_PROGRAM readonly COMPOSE_LIFECYCLE_EXECUTOR="/etc/cloud-compose/libexec/run-lifecycle-program.sh" run_compose_lifecycle_executor() { @@ -66,32 +105,8 @@ validate_compose_projects_manifest() { return 1 fi - if ! jq -e ' - type == "object" and length > 0 and - all(to_entries[]; . as $entry | - ($entry.key | explode | index(0) == null) and - ($entry.value | type == "object") and - (all($entry.value | .. | select(type == "string"); - explode | index(0) == null)) and - ($entry.value.docker_compose_repo | type == "string" and length > 0) and - ($entry.value.docker_compose_branch | type == "string" and length > 0) and - ($entry.value.project_dir | type == "string" and length > 0) and - ($entry.value.compose_project_name | type == "string" and length > 0) and - (all(["init_commands", "up_commands", "down_commands", "rollout_commands"][]; - . as $field | - ($entry.value[$field] == null) or - (($entry.value[$field] | type) == "array" and all($entry.value[$field][]; type == "string")) - )) and - (($entry.value.sitectl_verify_args == null) or - (($entry.value.sitectl_verify_args | type) == "array" and all($entry.value.sitectl_verify_args[]; - type == "string" and - (explode | index(0) == null) and - (contains("\n") | not) and - (contains("\r") | not) - ))) and - (($entry.value.ingress == null) or ($entry.value.ingress | type == "object")) - ) - ' "$COMPOSE_PROJECTS_FILE" >/dev/null; then + if ! jq -e -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-validate-projects.jq" \ + "$COMPOSE_PROJECTS_FILE" >/dev/null; then echo "Invalid cloud-compose project manifest: $COMPOSE_PROJECTS_FILE" >&2 return 1 fi @@ -106,7 +121,8 @@ validate_compose_projects_manifest() { )" || return 1 app="${app%$'\x1f'}" validate_compose_app_name "$app" || return 1 - done < <(jq -r 'keys[] | @base64' "$COMPOSE_PROJECTS_FILE") + done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-keys-base64.jq" \ + "$COMPOSE_PROJECTS_FILE") while IFS= read -r encoded_project_dir; do project_dir="$( @@ -117,7 +133,8 @@ validate_compose_projects_manifest() { )" || return 1 project_dir="${project_dir%$'\x1f'}" validate_compose_project_dir "$project_dir" || return 1 - done < <(jq -r '.[] | .project_dir | @base64' "$COMPOSE_PROJECTS_FILE") + done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-project-dirs-base64.jq" \ + "$COMPOSE_PROJECTS_FILE") } # Converge one already-existing manifest project without traversing or changing @@ -248,7 +265,8 @@ compose_app_exists() { validate_compose_app_name "$app" || return 1 validate_compose_projects_manifest || return 1 - jq -e --arg app "$app" 'has($app)' "$COMPOSE_PROJECTS_FILE" >/dev/null || { + jq -e --arg key "$app" -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-has-key.jq" \ + "$COMPOSE_PROJECTS_FILE" >/dev/null || { echo "Cloud-compose app is not present in the manifest: $app" >&2 return 1 } @@ -260,7 +278,8 @@ compose_app_names_array() { local -n "result=$result_name" validate_compose_projects_manifest || return 1 - names="$(jq -er 'keys[]' "$COMPOSE_PROJECTS_FILE")" || return 1 + names="$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-keys.jq" \ + "$COMPOSE_PROJECTS_FILE")" || return 1 result=() while IFS= read -r app; do validate_compose_app_name "$app" || return 1 @@ -280,14 +299,9 @@ compose_app_field() { local field="$2" compose_app_exists "$app" || return 1 - jq -er --arg app "$app" --arg field "$field" ' - (.[$app][$field] // "" | tostring) as $value | - if $value | (explode | index(0) != null) or contains("\n") or contains("\r") then - error("invalid scalar field") - else - $value - end - ' "$COMPOSE_PROJECTS_FILE" + jq -er --arg app "$app" --arg field "$field" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-app-field.jq" \ + "$COMPOSE_PROJECTS_FILE" } compose_app_array_values() { @@ -298,20 +312,16 @@ compose_app_array_values() { local -n "result=$result_name" compose_app_exists "$app" || return 1 - payload="$(jq -ce --arg app "$app" --arg field "$field" ' - (.[$app][$field] // []) as $values | - if ($values | type) != "array" or any($values[]; type != "string") then - error("invalid string array") - else - $values - end - ' "$COMPOSE_PROJECTS_FILE")" || { + payload="$(jq -ce --arg app "$app" --arg field "$field" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-app-array.jq" \ + "$COMPOSE_PROJECTS_FILE")" || { echo "Invalid $field array for cloud-compose app $app" >&2 return 1 } result=() - encoded_lines="$(jq -r '.[] | @base64' <<<"$payload")" || return 1 + encoded_lines="$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/array-values-base64.jq" \ + <<<"$payload")" || return 1 if [[ -z "$encoded_lines" ]]; then return 0 fi @@ -343,30 +353,18 @@ compose_app_verify_args() { local app="$1" compose_app_exists "$app" || return 1 - jq -er --arg app "$app" ' - (.[$app].sitectl_verify_args // []) as $values | - if ($values | type) != "array" or any($values[]; type != "string") then - error("invalid verify args") - else - $values | join(" ") - end - ' "$COMPOSE_PROJECTS_FILE" + jq -er --arg app "$app" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-app-verify-args.jq" \ + "$COMPOSE_PROJECTS_FILE" } compose_app_verify_args_json() { local app="$1" compose_app_exists "$app" || return 1 - jq -cer --arg app "$app" ' - (.[$app].sitectl_verify_args // []) as $values | - if ($values | type) != "array" or any($values[]; - type != "string" or (explode | index(0) != null) or contains("\n") or contains("\r") - ) then - error("invalid verify args") - else - $values - end - ' "$COMPOSE_PROJECTS_FILE" + jq -cer --arg app "$app" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-app-verify-args-json.jq" \ + "$COMPOSE_PROJECTS_FILE" } compose_app_ingress_field() { @@ -374,14 +372,9 @@ compose_app_ingress_field() { local field="$2" compose_app_exists "$app" || return 1 - jq -er --arg app "$app" --arg field "$field" ' - (.[$app].ingress[$field] // "" | tostring) as $value | - if $value | (explode | index(0) != null) or contains("\n") or contains("\r") then - error("invalid ingress scalar field") - else - $value - end - ' "$COMPOSE_PROJECTS_FILE" + jq -er --arg app "$app" --arg field "$field" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-app-ingress-field.jq" \ + "$COMPOSE_PROJECTS_FILE" } compose_app_ingress_array_values() { @@ -392,20 +385,16 @@ compose_app_ingress_array_values() { local -n "result=$result_name" compose_app_exists "$app" || return 1 - payload="$(jq -ce --arg app "$app" --arg field "$field" ' - (.[$app].ingress[$field] // []) as $values | - if ($values | type) != "array" or any($values[]; type != "string") then - error("invalid ingress string array") - else - $values - end - ' "$COMPOSE_PROJECTS_FILE")" || { + payload="$(jq -ce --arg app "$app" --arg field "$field" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-app-ingress-array.jq" \ + "$COMPOSE_PROJECTS_FILE")" || { echo "Invalid ingress $field array for cloud-compose app $app" >&2 return 1 } result=() - encoded_lines="$(jq -r '.[] | @base64' <<<"$payload")" || return 1 + encoded_lines="$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/array-values-base64.jq" \ + <<<"$payload")" || return 1 if [[ -z "$encoded_lines" ]]; then return 0 fi @@ -692,7 +681,7 @@ configure_sitectl_verify_argv() { # not invoke sitectl (for example a source-policy contract). The exported # wrapper fails at the actual call site if the executable is missing. SITECTL_EXECUTABLE="$(type -P sitectl || true)" - if ! jq -e 'type == "array" and all(.[]; type == "string")' \ + if ! jq -e -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/string-array-validate.jq" \ <<<"${SITECTL_VERIFY_ARGS_JSON:-[]}" >/dev/null; then echo "Invalid sitectl verify argument JSON for ${APP_NAME:-unknown app}" >&2 return 1 @@ -715,7 +704,8 @@ configure_sitectl_verify_argv() { [[ -n "$encoded" ]] || continue decoded="$(printf '%s' "$encoded" | base64 -d)" || return 1 configured_verify_args+=("$decoded") - done < <(jq -r '.[] | @base64' <<<"${SITECTL_VERIFY_ARGS_JSON:-[]}") + done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/array-values-base64.jq" \ + <<<"${SITECTL_VERIFY_ARGS_JSON:-[]}") fi "$executable" "$@" "${configured_verify_args[@]}" } @@ -921,7 +911,7 @@ verify_clean_compose_checkout() { } reject_host_network_compose_services() { - local config_json + local config_json filter_status if [[ "${CLOUD_COMPOSE_PROVIDER:-}" != "gcp" ]]; then return 0 @@ -934,26 +924,21 @@ reject_host_network_compose_services() { echo "Could not render Compose configuration for metadata-isolation validation" >&2 return 1 } - if ! jq -e '.services | type == "object"' <<<"$config_json" >/dev/null; then + if ! jq -e -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-services-object-validate.jq" \ + <<<"$config_json" >/dev/null; then echo "Docker Compose returned an invalid service configuration" >&2 return 1 fi - if jq -e ' - any(.services[]; - (.network_mode // "") == "host" or - ( - (.build | type) == "object" and - ( - (.build.network // "") == "host" or - any((.build.entitlements // [])[]; - . == "network.host" or . == "security.insecure" - ) - ) - ) - ) - ' <<<"$config_json" >/dev/null; then + if jq -e -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/compose-reject-host-network.jq" \ + <<<"$config_json" >/dev/null; then echo "Host runtime/build networking and insecure BuildKit entitlements are not allowed on GCP because they bypass container metadata isolation" >&2 return 1 + else + filter_status=$? + fi + if [[ "$filter_status" -ne 1 ]]; then + echo "Could not evaluate Compose network isolation with the checked jq program" >&2 + return 1 fi } diff --git a/rootfs/home/cloud-compose/configure-metadata-firewall.sh b/rootfs/home/cloud-compose/configure-metadata-firewall.sh index 257734e..70ee5c2 100644 --- a/rootfs/home/cloud-compose/configure-metadata-firewall.sh +++ b/rootfs/home/cloud-compose/configure-metadata-firewall.sh @@ -2,6 +2,28 @@ set -euo pipefail +_cc_configure_metadata_firewall_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_configure_metadata_firewall_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_configure_metadata_firewall_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_configure_metadata_firewall_source _cc_configure_metadata_firewall_dir _cc_configure_metadata_firewall_installed_home +if [[ -n "$_cc_configure_metadata_firewall_installed_home" && + ( "$_cc_configure_metadata_firewall_installed_home" == "/" || + "$_cc_configure_metadata_firewall_source" == "${_cc_configure_metadata_firewall_installed_home%/}/"* ) ]]; then + _cc_configure_metadata_firewall_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_configure_metadata_firewall_checked_programs="$_cc_configure_metadata_firewall_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_configure_metadata_firewall_checked_programs +# shellcheck disable=SC1090 +source "$_cc_configure_metadata_firewall_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_configure_metadata_firewall_source" \ + CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh \ + "$_cc_configure_metadata_firewall_dir/profile.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +readonly profile_path + mode="${1:-full}" case "$mode" in full | pre-docker) @@ -12,9 +34,6 @@ case "$mode" in ;; esac -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" - # shellcheck disable=SC1090 source "$profile_path" diff --git a/rootfs/home/cloud-compose/converge-app-filesystems.sh b/rootfs/home/cloud-compose/converge-app-filesystems.sh index 02ac070..11f861b 100644 --- a/rootfs/home/cloud-compose/converge-app-filesystems.sh +++ b/rootfs/home/cloud-compose/converge-app-filesystems.sh @@ -2,15 +2,40 @@ set -euo pipefail +_cc_converge_app_filesystems_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_converge_app_filesystems_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_converge_app_filesystems_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_converge_app_filesystems_source _cc_converge_app_filesystems_dir _cc_converge_app_filesystems_installed_home +if [[ -n "$_cc_converge_app_filesystems_installed_home" && + ( "$_cc_converge_app_filesystems_installed_home" == "/" || + "$_cc_converge_app_filesystems_source" == "${_cc_converge_app_filesystems_installed_home%/}/"* ) ]]; then + _cc_converge_app_filesystems_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_converge_app_filesystems_checked_programs="$_cc_converge_app_filesystems_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_converge_app_filesystems_checked_programs +# shellcheck disable=SC1090 +source "$_cc_converge_app_filesystems_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_converge_app_filesystems_source" \ + CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh \ + "$_cc_converge_app_filesystems_dir/profile.sh" +cloud_compose_bind_source_program \ + "$_cc_converge_app_filesystems_source" \ + CLOUD_COMPOSE_COMPOSE_APPS_PATH \ + /home/cloud-compose/compose-apps.sh \ + "$_cc_converge_app_filesystems_dir/compose-apps.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +compose_apps_path="$CLOUD_COMPOSE_COMPOSE_APPS_PATH" +readonly profile_path compose_apps_path + if ((EUID != 0)); then echo "Compose application filesystem convergence must run as root" >&2 exit 1 fi -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" -compose_apps_path="${CLOUD_COMPOSE_COMPOSE_APPS_PATH:-$script_dir/compose-apps.sh}" -cd "$script_dir" +cd "$_cc_converge_app_filesystems_dir" # shellcheck disable=SC1090 source "$profile_path" # shellcheck disable=SC1090 diff --git a/rootfs/home/cloud-compose/default-lifecycle.sh b/rootfs/home/cloud-compose/default-lifecycle.sh index 845a0a9..424a793 100644 --- a/rootfs/home/cloud-compose/default-lifecycle.sh +++ b/rootfs/home/cloud-compose/default-lifecycle.sh @@ -2,6 +2,28 @@ set -euo pipefail +_cc_default_lifecycle_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_default_lifecycle_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_default_lifecycle_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_default_lifecycle_source _cc_default_lifecycle_dir _cc_default_lifecycle_installed_home +if [[ -n "$_cc_default_lifecycle_installed_home" && + ( "$_cc_default_lifecycle_installed_home" == "/" || + "$_cc_default_lifecycle_source" == "${_cc_default_lifecycle_installed_home%/}/"* ) ]]; then + _cc_default_lifecycle_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_default_lifecycle_checked_programs="$_cc_default_lifecycle_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_default_lifecycle_checked_programs +# shellcheck disable=SC1090 +source "$_cc_default_lifecycle_checked_programs" +cloud_compose_bind_program \ + "$_cc_default_lifecycle_source" \ + CLOUD_COMPOSE_SITECTL_VERIFY_ARGS_PROGRAM \ + /etc/cloud-compose/jq/sitectl-verify-args.jq \ + "$_cc_default_lifecycle_dir/../../etc/cloud-compose/jq/sitectl-verify-args.jq" +sitectl_verify_args_program="$CLOUD_COMPOSE_SITECTL_VERIFY_ARGS_PROGRAM" +readonly sitectl_verify_args_program + action="${1:-}" if [[ "$#" -ne 1 ]]; then echo "usage: default-lifecycle.sh init|up|down|rollout" >&2 @@ -28,11 +50,10 @@ run_sitectl() { verify_nonproduction() { local encoded decoded encoded_args - local verify_args_program="${CLOUD_COMPOSE_SITECTL_VERIFY_ARGS_PROGRAM:-/etc/cloud-compose/jq/sitectl-verify-args.jq}" local -a verify_args=() if [[ "${SITECTL_ENVIRONMENT:?SITECTL_ENVIRONMENT is required}" != "production" ]]; then - if ! encoded_args="$(jq -r -f "$verify_args_program" \ + if ! encoded_args="$(jq -r -f "$sitectl_verify_args_program" \ <<<"${SITECTL_VERIFY_ARGS_JSON:-[]}")"; then echo "SITECTL_VERIFY_ARGS_JSON must be an array of strings" >&2 return 1 diff --git a/rootfs/home/cloud-compose/disaster-recovery-lib.sh b/rootfs/home/cloud-compose/disaster-recovery-lib.sh index 8cba3df..3175b5f 100644 --- a/rootfs/home/cloud-compose/disaster-recovery-lib.sh +++ b/rootfs/home/cloud-compose/disaster-recovery-lib.sh @@ -3,9 +3,45 @@ # Shared validation for the provider-neutral disaster-recovery driver contract. # The caller must enable `set -euo pipefail` before sourcing this file. +_cc_dr_library_source="$(readlink -f -- "${BASH_SOURCE[0]}")" || { + echo "Could not resolve the Cloud Compose disaster-recovery library path" >&2 + return 1 2>/dev/null || exit 1 +} +_cc_dr_library_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || { + echo "Could not resolve the Cloud Compose disaster-recovery library directory" >&2 + return 1 2>/dev/null || exit 1 +} +_cc_dr_library_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_dr_library_source _cc_dr_library_dir _cc_dr_library_installed_home +if [[ -n "$_cc_dr_library_installed_home" && + ( "$_cc_dr_library_installed_home" == "/" || + "$_cc_dr_library_source" == "${_cc_dr_library_installed_home%/}/"* ) ]]; then + _cc_dr_library_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_dr_library_checked_programs="$_cc_dr_library_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_dr_library_checked_programs +# shellcheck disable=SC1090 +if ! source "$_cc_dr_library_checked_programs"; then + echo "Could not load the checked Cloud Compose program resolver" >&2 + return 1 2>/dev/null || exit 1 +fi +if ! cloud_compose_bind_program_dir \ + "$_cc_dr_library_source" \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR \ + /etc/cloud-compose/jq \ + "$_cc_dr_library_dir/../../etc/cloud-compose/jq" \ + dr-validate-backup-receipt.jq \ + dr-backup-completed-at.jq \ + dr-backup-remote-id.jq \ + dr-validate-restore-proof.jq \ + dr-restore-completed-at.jq \ + dr-restore-recovery-id.jq; then + return 1 2>/dev/null || exit 1 +fi + CLOUD_COMPOSE_DR_STATE_ROOT="${CLOUD_COMPOSE_DR_STATE_ROOT:-/mnt/disks/data/.cloud-compose-disaster-recovery}" CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="${CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER:-/etc/cloud-compose/libexec/offhost-backup-driver}" -CLOUD_COMPOSE_JQ_PROGRAM_DIR="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/etc/cloud-compose/jq}" cloud_compose_dr_is_required() { case "${CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED:-false}" in @@ -46,6 +82,18 @@ cloud_compose_dr_validate_remote_id() { fi } +cloud_compose_dr_sha256_file() { + local path="$1" output digest + + output="$(sha256sum -- "$path")" || return 1 + digest="${output%% *}" + if [[ ! "$digest" =~ ^[0-9a-f]{64}$ ]]; then + echo "sha256sum returned an invalid digest for: $path" >&2 + return 1 + fi + printf '%s\n' "$digest" +} + cloud_compose_dr_validate_driver() { local driver="${1:-$CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER}" local current component metadata owner mode kind links resolved diff --git a/rootfs/home/cloud-compose/docker-prune.sh b/rootfs/home/cloud-compose/docker-prune.sh index 22e21e4..4ef168e 100755 --- a/rootfs/home/cloud-compose/docker-prune.sh +++ b/rootfs/home/cloud-compose/docker-prune.sh @@ -2,8 +2,27 @@ set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" +_cc_docker_prune_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_docker_prune_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_docker_prune_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_docker_prune_source _cc_docker_prune_dir _cc_docker_prune_installed_home +if [[ -n "$_cc_docker_prune_installed_home" && + ( "$_cc_docker_prune_installed_home" == "/" || + "$_cc_docker_prune_source" == "${_cc_docker_prune_installed_home%/}/"* ) ]]; then + _cc_docker_prune_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_docker_prune_checked_programs="$_cc_docker_prune_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_docker_prune_checked_programs +# shellcheck disable=SC1090 +source "$_cc_docker_prune_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_docker_prune_source" \ + CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh \ + "$_cc_docker_prune_dir/profile.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +readonly profile_path # shellcheck disable=SC1090 source "$profile_path" diff --git a/rootfs/home/cloud-compose/host-init.sh b/rootfs/home/cloud-compose/host-init.sh index 673a93d..1799a99 100644 --- a/rootfs/home/cloud-compose/host-init.sh +++ b/rootfs/home/cloud-compose/host-init.sh @@ -5,6 +5,28 @@ set -euo pipefail # shellcheck disable=SC1091 source /home/cloud-compose/profile.sh +_cc_host_init_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_host_init_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_host_init_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_host_init_source _cc_host_init_dir _cc_host_init_installed_home +if [[ -n "$_cc_host_init_installed_home" && + ( "$_cc_host_init_installed_home" == "/" || + "$_cc_host_init_source" == "${_cc_host_init_installed_home%/}/"* ) ]]; then + _cc_host_init_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_host_init_checked_programs="$_cc_host_init_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_host_init_checked_programs +# shellcheck disable=SC1090 +source "$_cc_host_init_checked_programs" +cloud_compose_bind_program_dir \ + "$_cc_host_init_source" \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR \ + /etc/cloud-compose/jq \ + "$_cc_host_init_dir/../../etc/cloud-compose/jq" \ + gcp-metadata-public-ip.jq \ + gcp-metadata-private-ip.jq + cleanup() { if [ -n "${metadata_file:-}" ]; then rm -f "$metadata_file" @@ -23,9 +45,11 @@ if [ "${CLOUD_COMPOSE_PROVIDER:-}" = "gcp" ]; then "http://metadata.google.internal/computeMetadata/v1/?recursive=true" >"$metadata_file" update_runtime_env_file .env GCP_PUBLIC_IP \ - "$(jq -er '.instance.networkInterfaces[0].accessConfigs[0].externalIp' "$metadata_file")" + "$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/gcp-metadata-public-ip.jq" \ + "$metadata_file")" update_runtime_env_file .env GCP_PRIVATE_IP \ - "$(jq -er '.instance.networkInterfaces[0].ip' "$metadata_file")" + "$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/gcp-metadata-private-ip.jq" \ + "$metadata_file")" fi if [ "${LIBOPS_INTERNAL_SERVICES_ENABLED:-false}" = "true" ]; then diff --git a/rootfs/home/cloud-compose/install-docker-plugins.sh b/rootfs/home/cloud-compose/install-docker-plugins.sh index 50b7671..23855d9 100644 --- a/rootfs/home/cloud-compose/install-docker-plugins.sh +++ b/rootfs/home/cloud-compose/install-docker-plugins.sh @@ -2,6 +2,28 @@ set -euo pipefail +_cc_docker_plugins_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_docker_plugins_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_docker_plugins_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_docker_plugins_source _cc_docker_plugins_dir _cc_docker_plugins_installed_home +if [[ -n "$_cc_docker_plugins_installed_home" && + ( "$_cc_docker_plugins_installed_home" == "/" || + "$_cc_docker_plugins_source" == "${_cc_docker_plugins_installed_home%/}/"* ) ]]; then + _cc_docker_plugins_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_docker_plugins_checked_programs="$_cc_docker_plugins_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_docker_plugins_checked_programs +# shellcheck disable=SC1090 +source "$_cc_docker_plugins_checked_programs" +cloud_compose_bind_program \ + "$_cc_docker_plugins_source" \ + CLOUD_COMPOSE_RELEASE_CHECKSUM_PROGRAM \ + /etc/cloud-compose/awk/release-checksum.awk \ + "$_cc_docker_plugins_dir/../../etc/cloud-compose/awk/release-checksum.awk" +release_checksum_program="$CLOUD_COMPOSE_RELEASE_CHECKSUM_PROGRAM" +readonly release_checksum_program + docker_arch() { case "$(uname -m)" in x86_64 | amd64) echo "x86_64" ;; @@ -28,22 +50,7 @@ release_checksum() { local manifest="$1" local asset="$2" - awk -v asset="$asset" ' - { - filename = $2 - sub(/^\*/, "", filename) - if (filename == asset) { - checksum = $1 - matches++ - } - } - END { - if (matches != 1) { - exit 1 - } - print checksum - } - ' "$manifest" + awk -v asset="$asset" -f "$release_checksum_program" "$manifest" } validate_release_version() { diff --git a/rootfs/home/cloud-compose/libops-managed-runtime.sh b/rootfs/home/cloud-compose/libops-managed-runtime.sh index 9ef8b0a..71cfb54 100644 --- a/rootfs/home/cloud-compose/libops-managed-runtime.sh +++ b/rootfs/home/cloud-compose/libops-managed-runtime.sh @@ -2,8 +2,47 @@ set -euo pipefail +_cc_managed_runtime_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_managed_runtime_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_managed_runtime_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_managed_runtime_source _cc_managed_runtime_dir _cc_managed_runtime_installed_home +if [[ -n "$_cc_managed_runtime_installed_home" && + ( "$_cc_managed_runtime_installed_home" == "/" || + "$_cc_managed_runtime_source" == "${_cc_managed_runtime_installed_home%/}/"* ) ]]; then + _cc_managed_runtime_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_managed_runtime_checked_programs="$_cc_managed_runtime_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_managed_runtime_checked_programs +# shellcheck disable=SC1090 +source "$_cc_managed_runtime_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_managed_runtime_source" \ + CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh \ + "$_cc_managed_runtime_dir/profile.sh" + +# shellcheck disable=SC1090 +source "$CLOUD_COMPOSE_PROFILE_PATH" + +# Reload the fixed resolver after the profile before binding data programs. # shellcheck disable=SC1090 -source "${CLOUD_COMPOSE_PROFILE_PATH:-/home/cloud-compose/profile.sh}" +source "$_cc_managed_runtime_checked_programs" +cloud_compose_bind_program_dir \ + "$_cc_managed_runtime_source" \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR \ + /etc/cloud-compose/jq \ + "$_cc_managed_runtime_dir/../../etc/cloud-compose/jq" \ + github-latest-release-tag.jq \ + sitectl-package-version.jq \ + sitectl-package-versions-validate.jq \ + object-field-delimited.jq \ + object-keys-base64.jq +cloud_compose_bind_program \ + "$_cc_managed_runtime_source" \ + CLOUD_COMPOSE_RELEASE_CHECKSUM_ENTRY_PROGRAM \ + /etc/cloud-compose/awk/release-checksum-entry.awk \ + "$_cc_managed_runtime_dir/../../etc/cloud-compose/awk/release-checksum-entry.awk" LOG_PREFIX="[libops-managed-runtime]" STATE_DIR="/mnt/disks/data/libops-managed" @@ -201,10 +240,8 @@ latest_release_tag() { return 1 fi - if ! tag="$(jq -jr ' - (.tag_name | select(type == "string" and length > 0 and (explode | index(0) == null))), - "\u001f" - ' "$metadata")"; then + if ! tag="$(jq -jr -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/github-latest-release-tag.jq" \ + "$metadata")"; then log "latest release metadata for ${package} did not contain a tag" rm -f "$metadata" return 1 @@ -325,7 +362,8 @@ install_release_package() { --connect-timeout 10 --max-time 300 -o "${tmp}/checksums.txt" -- "${base_url}/checksums.txt" checksum_file="${tmp}/checksums.selected.txt" - awk -v archive="$archive" '$2 == archive { print }' "${tmp}/checksums.txt" >"$checksum_file" + awk -v archive="$archive" -f "$CLOUD_COMPOSE_RELEASE_CHECKSUM_ENTRY_PROGRAM" \ + "${tmp}/checksums.txt" >"$checksum_file" if [[ "$(wc -l <"$checksum_file")" -ne 1 ]]; then log "release checksums must contain exactly one entry for ${archive}" rm -rf -- "$tmp" @@ -408,7 +446,7 @@ sitectl_package_version() { local fallback="${SITECTL_VERSION:-latest}" jq -er --arg package "$package" --arg fallback "$fallback" \ - '.[$package] // $fallback' \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/sitectl-package-version.jq" \ <<<"$(sitectl_package_versions_json)" } @@ -423,14 +461,8 @@ validate_sitectl_configuration() { fi versions_json="$(sitectl_package_versions_json)" - if ! jq -e ' - type == "object" and - all(to_entries[]; - (.key | explode | index(0) == null) and - (.value | type == "string") and - (.value | explode | index(0) == null) - ) - ' <<<"$versions_json" >/dev/null; then + if ! jq -e -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/sitectl-package-versions-validate.jq" \ + <<<"$versions_json" >/dev/null; then log "SITECTL_PACKAGE_VERSIONS must be a JSON object of sitectl package names to latest or exact semantic-version release tags" return 1 fi @@ -455,7 +487,9 @@ validate_sitectl_configuration() { log "SITECTL_PACKAGE_VERSIONS contains an invalid package name: ${override}" return 1 fi - override_version="$(jq -jr --arg package "$override" '(.[$package]), "\u001f"' <<<"$versions_json")" || return 1 + override_version="$(jq -jr --arg field "$override" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field-delimited.jq" \ + <<<"$versions_json")" || return 1 override_version="${override_version%$'\x1f'}" if ! valid_sitectl_version "$override_version"; then log "SITECTL_PACKAGE_VERSIONS contains an invalid release tag for ${override}: ${override_version}" @@ -465,7 +499,8 @@ validate_sitectl_configuration() { log "SITECTL_PACKAGE_VERSIONS contains an uninstalled package: ${override}" return 1 fi - done < <(jq -r 'keys[] | @base64' <<<"$versions_json") + done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-keys-base64.jq" \ + <<<"$versions_json") } validate_stale_managed_sitectl_package() { @@ -802,6 +837,17 @@ write_artifact_state() { mv -f -- "$state_tmp" "$state_file" } +tab_separated_field_count() { + local value="$1" + local count=1 + + while [[ "$value" == *$'\t'* ]]; do + value="${value#*$'\t'}" + ((count += 1)) + done + printf '%s\n' "$count" +} + install_managed_artifacts() { local line name url sha path mode owner group restart index local state_file failed_state download_tmp install_tmp target_dir target_name backup @@ -820,7 +866,7 @@ install_managed_artifacts() { if [ -z "$line" ] || [[ "$line" == \#* ]]; then continue fi - field_count="$(awk -F '\t' '{ print NF }' <<<"$line")" + field_count="$(tab_separated_field_count "$line")" if [[ "$field_count" != "8" ]]; then log "managed artifact manifest row must contain exactly eight tab-separated fields" return 1 diff --git a/rootfs/home/cloud-compose/mariadb-backup.sh b/rootfs/home/cloud-compose/mariadb-backup.sh index ebdcd3c..c7513a7 100644 --- a/rootfs/home/cloud-compose/mariadb-backup.sh +++ b/rootfs/home/cloud-compose/mariadb-backup.sh @@ -2,11 +2,34 @@ set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" -compose_apps_path="${CLOUD_COMPOSE_COMPOSE_APPS_PATH:-$script_dir/compose-apps.sh}" +_cc_mariadb_backup_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_mariadb_backup_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_mariadb_backup_source script_dir _cc_mariadb_backup_installed_home +if [[ -n "$_cc_mariadb_backup_installed_home" && + ( "$_cc_mariadb_backup_installed_home" == "/" || + "$_cc_mariadb_backup_source" == "${_cc_mariadb_backup_installed_home%/}/"* ) ]]; then + _cc_mariadb_backup_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_mariadb_backup_checked_programs="$script_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_mariadb_backup_checked_programs +# shellcheck disable=SC1090 +source "$_cc_mariadb_backup_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_mariadb_backup_source" CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh "$script_dir/profile.sh" +cloud_compose_bind_source_program \ + "$_cc_mariadb_backup_source" CLOUD_COMPOSE_COMPOSE_APPS_PATH \ + /home/cloud-compose/compose-apps.sh "$script_dir/compose-apps.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +compose_apps_path="$CLOUD_COMPOSE_COMPOSE_APPS_PATH" +readonly profile_path compose_apps_path # shellcheck disable=SC1090 source "$profile_path" +# Reload the fixed resolver before sourcing the Compose library. +# shellcheck disable=SC1090 +source "$_cc_mariadb_backup_checked_programs" # shellcheck disable=SC1090 source "$compose_apps_path" diff --git a/rootfs/home/cloud-compose/mount-overlays.sh b/rootfs/home/cloud-compose/mount-overlays.sh index 81ba681..8d96c14 100755 --- a/rootfs/home/cloud-compose/mount-overlays.sh +++ b/rootfs/home/cloud-compose/mount-overlays.sh @@ -2,8 +2,33 @@ set -euo pipefail -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-/home/cloud-compose/profile.sh}" -overlay_script="${CLOUD_COMPOSE_OVERLAY_INIT_PATH:-/home/cloud-compose/overlay-init.sh}" +_cc_mount_overlays_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_mount_overlays_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_mount_overlays_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_mount_overlays_source _cc_mount_overlays_dir _cc_mount_overlays_installed_home +if [[ -n "$_cc_mount_overlays_installed_home" && + ( "$_cc_mount_overlays_installed_home" == "/" || + "$_cc_mount_overlays_source" == "${_cc_mount_overlays_installed_home%/}/"* ) ]]; then + _cc_mount_overlays_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_mount_overlays_checked_programs="$_cc_mount_overlays_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_mount_overlays_checked_programs +# shellcheck disable=SC1090 +source "$_cc_mount_overlays_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_mount_overlays_source" \ + CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh \ + "$_cc_mount_overlays_dir/profile.sh" +cloud_compose_bind_source_program \ + "$_cc_mount_overlays_source" \ + CLOUD_COMPOSE_OVERLAY_INIT_PATH \ + /home/cloud-compose/overlay-init.sh \ + "$_cc_mount_overlays_dir/overlay-init.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +overlay_script="$CLOUD_COMPOSE_OVERLAY_INIT_PATH" +readonly profile_path overlay_script # shellcheck disable=SC1090 source "$profile_path" diff --git a/rootfs/home/cloud-compose/offhost-backup.sh b/rootfs/home/cloud-compose/offhost-backup.sh index e84ea8b..35a5989 100644 --- a/rootfs/home/cloud-compose/offhost-backup.sh +++ b/rootfs/home/cloud-compose/offhost-backup.sh @@ -2,16 +2,52 @@ set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" -compose_apps_path="${CLOUD_COMPOSE_COMPOSE_APPS_PATH:-$script_dir/compose-apps.sh}" -dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}" -jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/etc/cloud-compose/jq}" +_cc_offhost_backup_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_offhost_backup_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_offhost_backup_source script_dir _cc_offhost_backup_installed_home +if [[ -n "$_cc_offhost_backup_installed_home" && + ( "$_cc_offhost_backup_installed_home" == "/" || + "$_cc_offhost_backup_source" == "${_cc_offhost_backup_installed_home%/}/"* ) ]]; then + _cc_offhost_backup_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_offhost_backup_checked_programs="$script_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_offhost_backup_checked_programs +# shellcheck disable=SC1090 +source "$_cc_offhost_backup_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_offhost_backup_source" CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh "$script_dir/profile.sh" +cloud_compose_bind_source_program \ + "$_cc_offhost_backup_source" CLOUD_COMPOSE_COMPOSE_APPS_PATH \ + /home/cloud-compose/compose-apps.sh "$script_dir/compose-apps.sh" +cloud_compose_bind_source_program \ + "$_cc_offhost_backup_source" CLOUD_COMPOSE_DR_LIBRARY_PATH \ + /home/cloud-compose/disaster-recovery-lib.sh "$script_dir/disaster-recovery-lib.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +compose_apps_path="$CLOUD_COMPOSE_COMPOSE_APPS_PATH" +dr_library_path="$CLOUD_COMPOSE_DR_LIBRARY_PATH" +readonly profile_path compose_apps_path dr_library_path # shellcheck disable=SC1090 source "$profile_path" # shellcheck disable=SC1090 source "$compose_apps_path" -CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir" +# Restore the fixed binding functions before the final sourced dependency. +# shellcheck disable=SC1090 +source "$_cc_offhost_backup_checked_programs" +cloud_compose_bind_program_dir \ + "$_cc_offhost_backup_source" \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR \ + /etc/cloud-compose/jq \ + "$script_dir/../../etc/cloud-compose/jq" \ + offhost-validate-compose-config.jq \ + offhost-validate-bind-roots.jq \ + offhost-bind-sources.jq \ + offhost-build-application-coverage.jq \ + offhost-build-manifest.jq \ + offhost-validate-manifest.jq \ + offhost-manifest-app-digests.jq # shellcheck disable=SC1090 source "$dr_library_path" @@ -105,7 +141,7 @@ for app in "${apps[@]}"; do echo "Required local MariaDB recovery artifact is invalid for ${app}" >&2 exit 1 fi - dump_sha256="$(sha256sum "$staged_dump" | awk '{print $1}')" + dump_sha256="$(cloud_compose_dr_sha256_file "$staged_dump")" dump_bytes="$(wc -c <"$staged_dump")" ( @@ -170,14 +206,15 @@ while IFS=$'\t' read -r manifest_app manifest_sha; do fi done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-manifest-app-digests.jq" "$staged_manifest") -manifest_sha256="$(sha256sum "$staged_manifest" | awk '{print $1}')" +manifest_sha256="$(cloud_compose_dr_sha256_file "$staged_manifest")" staged_receipt="$staging_dir/receipt.json" cloud_compose_dr_run_driver "$driver" backup \ --manifest "$staged_manifest" \ --manifest-sha256 "$manifest_sha256" \ --operation-id "$operation_id" \ --receipt "$staged_receipt" -if [[ "$(sha256sum "$staged_manifest" | awk '{print $1}')" != "$manifest_sha256" ]]; then +post_driver_manifest_sha256="$(cloud_compose_dr_sha256_file "$staged_manifest")" +if [[ "$post_driver_manifest_sha256" != "$manifest_sha256" ]]; then echo "Off-host backup driver modified the immutable coverage manifest" >&2 exit 1 fi diff --git a/rootfs/home/cloud-compose/persist-filesystems.sh b/rootfs/home/cloud-compose/persist-filesystems.sh index 03a761a..fab4bf7 100755 --- a/rootfs/home/cloud-compose/persist-filesystems.sh +++ b/rootfs/home/cloud-compose/persist-filesystems.sh @@ -2,6 +2,29 @@ set -euo pipefail +_cc_persist_filesystems_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_persist_filesystems_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_persist_filesystems_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_persist_filesystems_source _cc_persist_filesystems_dir _cc_persist_filesystems_installed_home +if [[ -n "$_cc_persist_filesystems_installed_home" && + ( "$_cc_persist_filesystems_installed_home" == "/" || + "$_cc_persist_filesystems_source" == "${_cc_persist_filesystems_installed_home%/}/"* ) ]]; then + # shellcheck disable=SC1091 + source /etc/cloud-compose/libexec/checked-programs.bash + cloud_compose_bind_program \ + "$_cc_persist_filesystems_source" \ + CLOUD_COMPOSE_FSTAB_RECONCILE_PROGRAM \ + /etc/cloud-compose/awk/reconcile-fstab.awk \ + /etc/cloud-compose/awk/reconcile-fstab.awk + fstab_reconcile_program="$CLOUD_COMPOSE_FSTAB_RECONCILE_PROGRAM" +else + # Early boot executes a verified root-owned copy from /run and passes its + # separately verified awk program explicitly. Repository contracts use the + # same override without weakening the installed /home path. + fstab_reconcile_program="${CLOUD_COMPOSE_FSTAB_RECONCILE_PROGRAM:-$_cc_persist_filesystems_dir/../../etc/cloud-compose/awk/reconcile-fstab.awk}" +fi +readonly fstab_reconcile_program + log() { printf '[filesystem-persist] %s\n' "$*" >&2 } @@ -185,28 +208,8 @@ main() { trap 'rm -f -- "$tmp"' EXIT awk -v begin="$begin_marker" -v end="$end_marker" \ -v data_device="$data_device" -v data_provider_mount="$data_provider_mount" \ - -v volumes_device="$volumes_device" -v volumes_provider_mount="$volumes_provider_mount" ' - $0 == begin { managed = 1; next } - $0 == end { managed = 0; next } - !managed { - if (data_provider_mount != "" && $2 == data_provider_mount) { - if ($1 == data_device) next - conflict = 1 - } - if (volumes_provider_mount != "" && $2 == volumes_provider_mount) { - if ($1 == volumes_device) next - conflict = 1 - } - if ($2 == "/mnt/disks/data" || - $2 == "/mnt/disks/volumes" || - $2 == "/mnt/disks/data/docker/volumes" || - $2 == "/mnt/disks/prod-readonly") { - conflict = 1 - } - print - } - END { if (managed || conflict) exit 42 } - ' "$fstab_path" >"$tmp" || { + -v volumes_device="$volumes_device" -v volumes_provider_mount="$volumes_provider_mount" \ + -f "$fstab_reconcile_program" "$fstab_path" >"$tmp" || { status=$? if [[ "$status" -eq 42 ]]; then log "fstab contains an unterminated managed block or an unmanaged cloud-compose mount target" diff --git a/rootfs/home/cloud-compose/prepare-app-sources.sh b/rootfs/home/cloud-compose/prepare-app-sources.sh index 6b98879..3d6b15d 100644 --- a/rootfs/home/cloud-compose/prepare-app-sources.sh +++ b/rootfs/home/cloud-compose/prepare-app-sources.sh @@ -2,12 +2,35 @@ set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" -compose_apps_path="${CLOUD_COMPOSE_COMPOSE_APPS_PATH:-$script_dir/compose-apps.sh}" +_cc_prepare_sources_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_prepare_sources_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_prepare_sources_source script_dir _cc_prepare_sources_installed_home +if [[ -n "$_cc_prepare_sources_installed_home" && + ( "$_cc_prepare_sources_installed_home" == "/" || + "$_cc_prepare_sources_source" == "${_cc_prepare_sources_installed_home%/}/"* ) ]]; then + _cc_prepare_sources_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_prepare_sources_checked_programs="$script_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_prepare_sources_checked_programs +# shellcheck disable=SC1090 +source "$_cc_prepare_sources_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_prepare_sources_source" CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh "$script_dir/profile.sh" +cloud_compose_bind_source_program \ + "$_cc_prepare_sources_source" CLOUD_COMPOSE_COMPOSE_APPS_PATH \ + /home/cloud-compose/compose-apps.sh "$script_dir/compose-apps.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +compose_apps_path="$CLOUD_COMPOSE_COMPOSE_APPS_PATH" +readonly profile_path compose_apps_path cd "$script_dir" # shellcheck disable=SC1090 source "$profile_path" +# Reload the fixed resolver before sourcing the Compose library. +# shellcheck disable=SC1090 +source "$_cc_prepare_sources_checked_programs" # shellcheck disable=SC1090 source "$compose_apps_path" diff --git a/rootfs/home/cloud-compose/profile.sh b/rootfs/home/cloud-compose/profile.sh index 7602d40..7695c52 100644 --- a/rootfs/home/cloud-compose/profile.sh +++ b/rootfs/home/cloud-compose/profile.sh @@ -1,5 +1,29 @@ #!/usr/bin/env bash +_cc_profile_source="$(readlink -f -- "${BASH_SOURCE[0]}")" || { + echo "Could not resolve the Cloud Compose profile path" >&2 + return 1 2>/dev/null || exit 1 +} +_cc_profile_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || { + echo "Could not resolve the Cloud Compose profile directory" >&2 + return 1 2>/dev/null || exit 1 +} +_cc_profile_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_profile_source _cc_profile_dir _cc_profile_installed_home +if [[ -n "$_cc_profile_installed_home" && + ( "$_cc_profile_installed_home" == "/" || + "$_cc_profile_source" == "${_cc_profile_installed_home%/}/"* ) ]]; then + _cc_profile_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_profile_checked_programs="$_cc_profile_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_profile_checked_programs +# shellcheck disable=SC1090 +if ! source "$_cc_profile_checked_programs"; then + echo "Could not load the checked Cloud Compose program resolver" >&2 + return 1 2>/dev/null || exit 1 +fi + decode_runtime_env_value() { local encoded="$1" local output="" character next index @@ -106,6 +130,17 @@ if ! load_runtime_env "${CLOUD_COMPOSE_ENV_FILE:-/home/cloud-compose/.env}"; the exit 1 fi +if ! cloud_compose_bind_program_dir \ + "$_cc_profile_source" \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR \ + /etc/cloud-compose/jq \ + "$_cc_profile_dir/../../etc/cloud-compose/jq" \ + application-env-validate.jq \ + object-entries-sorted-base64.jq \ + object-field-delimited.jq; then + return 1 2>/dev/null || exit 1 +fi + if ((EUID == 0)); then # Root-owned systemd/bootstrap paths must never resolve commands from the # cloud-compose-writable tool directory. @@ -329,14 +364,8 @@ sync_compose_application_env() ( echo "Refusing unsafe Compose environment path: $env_file" >&2 return 1 fi - if ! jq -e ' - type == "object" and - all(to_entries[]; - (.key | explode | index(0) == null) and - (.value | type == "string") and - (.value | explode | index(0) == null) - ) - ' "$application_env_file" >/dev/null; then + if ! jq -e -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/application-env-validate.jq" \ + "$application_env_file" >/dev/null; then echo "Invalid Compose application environment data: $application_env_file" >&2 return 1 fi @@ -349,7 +378,8 @@ sync_compose_application_env() ( } trap 'rm -f -- "$entries_file" "$tmp_file"' EXIT - jq -r 'to_entries | sort_by(.key)[] | @base64' "$application_env_file" >"$entries_file" || return 1 + jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-entries-sorted-base64.jq" \ + "$application_env_file" >"$entries_file" || return 1 if [[ -f "$env_file" ]]; then while IFS= read -r line || [[ -n "$line" ]]; do @@ -372,13 +402,17 @@ sync_compose_application_env() ( while IFS= read -r encoded || [[ -n "$encoded" ]]; do [[ -n "$encoded" ]] || continue entry_json="$(printf '%s' "$encoded" | base64 -d)" || return 1 - name="$(jq -jr '.key, "\u001f"' <<<"$entry_json")" || return 1 + name="$(jq -jr --arg field key \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field-delimited.jq" \ + <<<"$entry_json")" || return 1 name="${name%$'\x1f'}" if [[ ! "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then echo "Invalid Compose application environment data: $application_env_file" >&2 return 1 fi - value="$(jq -jr '.value, "\u001f"' <<<"$entry_json")" || return 1 + value="$(jq -jr --arg field value \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field-delimited.jq" \ + <<<"$entry_json")" || return 1 value="${value%$'\x1f'}" printf '# cloud-compose application: %s\n' "$name" >>"$tmp_file" write_runtime_env_assignment "$name" "$value" >>"$tmp_file" || return 1 diff --git a/rootfs/home/cloud-compose/restore-test.sh b/rootfs/home/cloud-compose/restore-test.sh index 92c766f..260114d 100644 --- a/rootfs/home/cloud-compose/restore-test.sh +++ b/rootfs/home/cloud-compose/restore-test.sh @@ -2,13 +2,34 @@ set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" -dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}" -jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/etc/cloud-compose/jq}" +_cc_restore_test_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_restore_test_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_restore_test_source script_dir _cc_restore_test_installed_home +if [[ -n "$_cc_restore_test_installed_home" && + ( "$_cc_restore_test_installed_home" == "/" || + "$_cc_restore_test_source" == "${_cc_restore_test_installed_home%/}/"* ) ]]; then + _cc_restore_test_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_restore_test_checked_programs="$script_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_restore_test_checked_programs +# shellcheck disable=SC1090 +source "$_cc_restore_test_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_restore_test_source" CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh "$script_dir/profile.sh" +cloud_compose_bind_source_program \ + "$_cc_restore_test_source" CLOUD_COMPOSE_DR_LIBRARY_PATH \ + /home/cloud-compose/disaster-recovery-lib.sh "$script_dir/disaster-recovery-lib.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +dr_library_path="$CLOUD_COMPOSE_DR_LIBRARY_PATH" +readonly profile_path dr_library_path # shellcheck disable=SC1090 source "$profile_path" -CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir" +# Restore the fixed binding functions before the final sourced dependency. +# shellcheck disable=SC1090 +source "$_cc_restore_test_checked_programs" # shellcheck disable=SC1090 source "$dr_library_path" @@ -60,9 +81,9 @@ if [[ -L "$manifest_path" || ! -f "$manifest_path" ]]; then exit 1 fi cloud_compose_dr_validate_json_file "$manifest_path" "Off-host backup manifest" -manifest_sha256="$(sha256sum "$manifest_path" | awk '{print $1}')" +manifest_sha256="$(cloud_compose_dr_sha256_file "$manifest_path")" cloud_compose_dr_validate_backup_receipt "$receipt_path" "$operation_id" "$manifest_sha256" -receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" +receipt_sha256="$(cloud_compose_dr_sha256_file "$receipt_path")" test_id="$(date -u +%Y%m%dT%H%M%SZ)-$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" staging_dir="$(mktemp -d "$staging_root/.restore-${test_id}.XXXXXX")" diff --git a/rootfs/home/cloud-compose/rotate-keys-app.sh b/rootfs/home/cloud-compose/rotate-keys-app.sh index 1b15f9d..6977760 100644 --- a/rootfs/home/cloud-compose/rotate-keys-app.sh +++ b/rootfs/home/cloud-compose/rotate-keys-app.sh @@ -2,10 +2,33 @@ set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" -rotate_keys_script="${CLOUD_COMPOSE_ROTATE_KEYS_PATH:-$script_dir/rotate-keys.sh}" -compose_apps_path="${CLOUD_COMPOSE_COMPOSE_APPS_PATH:-/home/cloud-compose/compose-apps.sh}" +_cc_rotate_keys_app_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_rotate_keys_app_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_rotate_keys_app_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_rotate_keys_app_source _cc_rotate_keys_app_dir _cc_rotate_keys_app_installed_home +if [[ -n "$_cc_rotate_keys_app_installed_home" && + ( "$_cc_rotate_keys_app_installed_home" == "/" || + "$_cc_rotate_keys_app_source" == "${_cc_rotate_keys_app_installed_home%/}/"* ) ]]; then + _cc_rotate_keys_app_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_rotate_keys_app_checked_programs="$_cc_rotate_keys_app_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_rotate_keys_app_checked_programs +# shellcheck disable=SC1090 +source "$_cc_rotate_keys_app_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_rotate_keys_app_source" CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh "$_cc_rotate_keys_app_dir/profile.sh" +cloud_compose_bind_source_program \ + "$_cc_rotate_keys_app_source" CLOUD_COMPOSE_ROTATE_KEYS_PATH \ + /home/cloud-compose/rotate-keys.sh "$_cc_rotate_keys_app_dir/rotate-keys.sh" +cloud_compose_bind_source_program \ + "$_cc_rotate_keys_app_source" CLOUD_COMPOSE_COMPOSE_APPS_PATH \ + /home/cloud-compose/compose-apps.sh "$_cc_rotate_keys_app_dir/compose-apps.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +rotate_keys_script="$CLOUD_COMPOSE_ROTATE_KEYS_PATH" +compose_apps_path="$CLOUD_COMPOSE_COMPOSE_APPS_PATH" +readonly profile_path rotate_keys_script compose_apps_path # shellcheck disable=SC1090 source "$profile_path" @@ -22,6 +45,17 @@ esac # shellcheck disable=SC1090 source "$compose_apps_path" +# Reload the owner-relative resolver after both sourced dependencies. +# shellcheck disable=SC1090 +source "$_cc_rotate_keys_app_checked_programs" +cloud_compose_bind_program_dir \ + "$_cc_rotate_keys_app_source" \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR \ + /etc/cloud-compose/jq \ + "$_cc_rotate_keys_app_dir/../../etc/cloud-compose/jq" \ + service-account-key-id.jq \ + service-account-credentials-valid.jq + APP_CREDENTIALS_FILE="${APP_CREDENTIALS_FILE:-/mnt/disks/data/cloud-compose/app/GOOGLE_APPLICATION_CREDENTIALS}" ROTATION_APP_CREDENTIAL_OWNER="${ROTATION_CREDENTIAL_OWNER-100}" ROTATION_CENTRAL_CREDENTIAL_OWNER="${ROTATION_CENTRAL_CREDENTIAL_OWNER-root}" @@ -98,11 +132,8 @@ require_inactive_app_service() { app_credential_key_id() { local file="$1" key_id - key_id="$(jq -jr ' - (.private_key_id | - select(type == "string" and length > 0 and (explode | index(0) == null))), - "\u001f" - ' "$file")" || return 1 + key_id="$(jq -jr -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/service-account-key-id.jq" \ + "$file")" || return 1 key_id="${key_id%$'\x1f'}" [[ "$key_id" =~ ^[A-Za-z0-9_-]+$ ]] || return 1 printf '%s\n' "$key_id" @@ -116,16 +147,9 @@ validate_app_credentials() { jq -e \ --arg key_id "$key_id" \ --arg service_account "$GCP_APP_SERVICE_ACCOUNT_EMAIL" \ - --arg project_id "$GCP_PROJECT" ' - .type == "service_account" and - .private_key_id == $key_id and - .client_email == $service_account and - .project_id == $project_id and - .token_uri == "https://oauth2.googleapis.com/token" and - (.private_key | type == "string" and - startswith("-----BEGIN PRIVATE KEY-----") and - contains("-----END PRIVATE KEY-----")) - ' "$file" >/dev/null + --arg project_id "$GCP_PROJECT" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/service-account-credentials-valid.jq" \ + "$file" >/dev/null } restore_central_app_credentials() { diff --git a/rootfs/home/cloud-compose/rotate-keys-daily.sh b/rootfs/home/cloud-compose/rotate-keys-daily.sh index fb30a69..8d8e60e 100644 --- a/rootfs/home/cloud-compose/rotate-keys-daily.sh +++ b/rootfs/home/cloud-compose/rotate-keys-daily.sh @@ -2,8 +2,25 @@ set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" +_cc_rotate_keys_daily_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_rotate_keys_daily_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_rotate_keys_daily_source script_dir _cc_rotate_keys_daily_installed_home +if [[ -n "$_cc_rotate_keys_daily_installed_home" && + ( "$_cc_rotate_keys_daily_installed_home" == "/" || + "$_cc_rotate_keys_daily_source" == "${_cc_rotate_keys_daily_installed_home%/}/"* ) ]]; then + _cc_rotate_keys_daily_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_rotate_keys_daily_checked_programs="$script_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_rotate_keys_daily_checked_programs +# shellcheck disable=SC1090 +source "$_cc_rotate_keys_daily_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_rotate_keys_daily_source" CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh "$script_dir/profile.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +readonly profile_path # shellcheck disable=SC1090 source "$profile_path" diff --git a/rootfs/home/cloud-compose/rotate-keys-internal.sh b/rootfs/home/cloud-compose/rotate-keys-internal.sh index f8aca1e..6641109 100644 --- a/rootfs/home/cloud-compose/rotate-keys-internal.sh +++ b/rootfs/home/cloud-compose/rotate-keys-internal.sh @@ -2,9 +2,29 @@ set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" -rotate_keys_script="${CLOUD_COMPOSE_ROTATE_KEYS_PATH:-$script_dir/rotate-keys.sh}" +_cc_rotate_keys_internal_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_rotate_keys_internal_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_rotate_keys_internal_source script_dir _cc_rotate_keys_internal_installed_home +if [[ -n "$_cc_rotate_keys_internal_installed_home" && + ( "$_cc_rotate_keys_internal_installed_home" == "/" || + "$_cc_rotate_keys_internal_source" == "${_cc_rotate_keys_internal_installed_home%/}/"* ) ]]; then + _cc_rotate_keys_internal_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_rotate_keys_internal_checked_programs="$script_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_rotate_keys_internal_checked_programs +# shellcheck disable=SC1090 +source "$_cc_rotate_keys_internal_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_rotate_keys_internal_source" CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh "$script_dir/profile.sh" +cloud_compose_bind_source_program \ + "$_cc_rotate_keys_internal_source" CLOUD_COMPOSE_ROTATE_KEYS_PATH \ + /home/cloud-compose/rotate-keys.sh "$script_dir/rotate-keys.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +rotate_keys_script="$CLOUD_COMPOSE_ROTATE_KEYS_PATH" +readonly profile_path rotate_keys_script # shellcheck disable=SC1090 source "$profile_path" diff --git a/rootfs/home/cloud-compose/rotate-keys.sh b/rootfs/home/cloud-compose/rotate-keys.sh index 9735301..7161577 100644 --- a/rootfs/home/cloud-compose/rotate-keys.sh +++ b/rootfs/home/cloud-compose/rotate-keys.sh @@ -2,12 +2,64 @@ set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}" +_cc_rotate_keys_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_rotate_keys_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_rotate_keys_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_rotate_keys_source _cc_rotate_keys_dir _cc_rotate_keys_installed_home +if [[ -n "$_cc_rotate_keys_installed_home" && + ( "$_cc_rotate_keys_installed_home" == "/" || + "$_cc_rotate_keys_source" == "${_cc_rotate_keys_installed_home%/}/"* ) ]]; then + _cc_rotate_keys_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_rotate_keys_checked_programs="$_cc_rotate_keys_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_rotate_keys_checked_programs +# shellcheck disable=SC1090 +source "$_cc_rotate_keys_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_rotate_keys_source" \ + CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh \ + "$_cc_rotate_keys_dir/profile.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +readonly profile_path # shellcheck disable=SC1090 source "$profile_path" +# Reload the fixed resolver after the profile so an installed environment +# cannot replace the binding functions used for checked data programs. +# shellcheck disable=SC1090 +source "$_cc_rotate_keys_checked_programs" +cloud_compose_bind_program_dir \ + "$_cc_rotate_keys_source" \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR \ + /etc/cloud-compose/jq \ + "$_cc_rotate_keys_dir/../../etc/cloud-compose/jq" \ + service-account-key-id.jq \ + rotation-build-state.jq \ + rotation-validate-state.jq \ + rotation-normalize-user-keys.jq \ + service-account-credentials-valid.jq \ + rotation-jwt-claims.jq \ + rotation-remaining-baseline.jq \ + rotation-new-key-names.jq \ + rotation-audit.jq \ + rotation-first-key-id.jq \ + rotation-idle-audit.jq \ + rotation-key-disabled.jq \ + rotation-key-ids-join.jq \ + rotation-key-names-base64.jq \ + rotation-key-names-sorted.jq \ + rotation-key-object-ids-join.jq \ + rotation-key-present.jq \ + array-values-base64.jq \ + json-length.jq \ + nonempty-string-field.jq \ + object-field-delimited.jq \ + object-field.jq \ + string-field-valid.jq + log_info() { printf '[key-rotation] %s\n' "$1" >&2 } @@ -179,11 +231,8 @@ valid_iam_key_id() { credential_key_id() { local file="$1" key_id - key_id="$(jq -jr ' - (.private_key_id | - select(type == "string" and length > 0 and (explode | index(0) == null))), - "\u001f" - ' "$file")" || return 1 + key_id="$(jq -jr -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/service-account-key-id.jq" \ + "$file")" || return 1 key_id="${key_id%$'\x1f'}" valid_iam_key_id "$key_id" || return 1 printf '%s\n' "$key_id" @@ -263,20 +312,8 @@ write_state() { --argjson created_at "$STATE_CREATED_AT" \ --argjson ready_at "$STATE_READY_AT" \ --argjson disabled_at "$STATE_DISABLED_AT" \ - '{ - version: 2, - phase: $phase, - service_account: $service_account, - project_id: $project_id, - credentials_file: $credentials_file, - current_key_id: $current_key_id, - new_key_id: $new_key_id, - new_key_name: $new_key_name, - baseline_key_names: $baseline_key_names, - created_at: $created_at, - ready_at: $ready_at, - disabled_at: $disabled_at - }' >"$state_tmp"; then + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-build-state.jq" \ + >"$state_tmp"; then rm -f -- "$state_tmp" return 1 fi @@ -298,43 +335,32 @@ load_state() { payload="$(jq -ce \ --arg service_account "$SERVICE_ACCOUNT" \ --arg project_id "$PROJECT_ID" \ - --arg credentials_file "$CREDENTIALS_FILE" ' - select( - .version == 2 and - (.phase == "reconciling" or .phase == "creating-fresh" or - .phase == "creating" or .phase == "staged" or .phase == "authenticated" or - .phase == "ready" or .phase == "grace" or .phase == "rolling-back" or - .phase == "rollback" or .phase == "revoke-new") and - .service_account == $service_account and - .project_id == $project_id and - .credentials_file == $credentials_file and - (.current_key_id | type == "string" and (explode | index(0) == null)) and - (.new_key_id | type == "string" and (explode | index(0) == null)) and - (.new_key_name | type == "string" and (explode | index(0) == null)) and - (.baseline_key_names | type == "array") and - (.baseline_key_names | length <= 10 and . == (sort | unique)) and - all(.baseline_key_names[]; - type == "string" and (explode | index(0) == null)) and - (.created_at | type == "number" and . >= 0 and floor == .) and - (.ready_at | type == "number" and . >= 0 and floor == .) and - (.disabled_at | type == "number" and . >= 0 and floor == .) - ) - ' "$PENDING_STATE")" || { + --arg credentials_file "$CREDENTIALS_FILE" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-validate-state.jq" \ + "$PENDING_STATE")" || { log_error "Pending rotation state is invalid or belongs to another target: $PENDING_STATE" return 1 } - STATE_PHASE="$(jq -r '.phase' <<<"$payload")" - STATE_CURRENT_KEY_ID="$(jq -jr '(.current_key_id), "\u001f"' <<<"$payload")" + STATE_PHASE="$(jq -r --arg field phase \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field.jq" <<<"$payload")" + STATE_CURRENT_KEY_ID="$(jq -jr --arg field current_key_id \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field-delimited.jq" <<<"$payload")" STATE_CURRENT_KEY_ID="${STATE_CURRENT_KEY_ID%$'\x1f'}" - STATE_NEW_KEY_ID="$(jq -jr '(.new_key_id), "\u001f"' <<<"$payload")" + STATE_NEW_KEY_ID="$(jq -jr --arg field new_key_id \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field-delimited.jq" <<<"$payload")" STATE_NEW_KEY_ID="${STATE_NEW_KEY_ID%$'\x1f'}" - STATE_NEW_KEY_NAME="$(jq -jr '(.new_key_name), "\u001f"' <<<"$payload")" + STATE_NEW_KEY_NAME="$(jq -jr --arg field new_key_name \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field-delimited.jq" <<<"$payload")" STATE_NEW_KEY_NAME="${STATE_NEW_KEY_NAME%$'\x1f'}" - STATE_BASELINE_KEY_NAMES="$(jq -c '.baseline_key_names' <<<"$payload")" - STATE_CREATED_AT="$(jq -r '.created_at' <<<"$payload")" - STATE_READY_AT="$(jq -r '.ready_at' <<<"$payload")" - STATE_DISABLED_AT="$(jq -r '.disabled_at' <<<"$payload")" + STATE_BASELINE_KEY_NAMES="$(jq -c --arg field baseline_key_names \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field.jq" <<<"$payload")" + STATE_CREATED_AT="$(jq -r --arg field created_at \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field.jq" <<<"$payload")" + STATE_READY_AT="$(jq -r --arg field ready_at \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field.jq" <<<"$payload")" + STATE_DISABLED_AT="$(jq -r --arg field disabled_at \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field.jq" <<<"$payload")" if [[ -n "$STATE_CURRENT_KEY_ID" && ! "$STATE_CURRENT_KEY_ID" =~ ^[A-Za-z0-9_-]+$ ]]; then log_error "Pending rotation state contains an invalid previous key ID" @@ -352,7 +378,8 @@ load_state() { log_error "Pending rotation state contains an invalid baseline key name" return 1 fi - done < <(jq -r '.[] | @base64' <<<"$STATE_BASELINE_KEY_NAMES") + done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/array-values-base64.jq" \ + <<<"$STATE_BASELINE_KEY_NAMES") case "$STATE_PHASE" in staged | authenticated | ready | grace | rolling-back | rollback | revoke-new) @@ -383,7 +410,9 @@ fetch_access_token() { log_error "Failed to get access token from metadata server" return 1 } - ACCESS_TOKEN="$(jq -er '.access_token | select(type == "string" and length > 0)' <<<"$token_response")" || { + ACCESS_TOKEN="$(jq -er --arg field access_token \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/nonempty-string-field.jq" \ + <<<"$token_response")" || { log_error "Metadata server returned an invalid access-token response" return 1 } @@ -412,23 +441,9 @@ list_user_keys() { log_error "Failed to list service-account keys" return 1 fi - normalized_keys="$(jq -cer --arg prefix "$SA_RESOURCE/keys/" ' - (.keys // []) as $keys | - if ($keys | type) != "array" then error("invalid key list") else - [$keys[] | select(.keyType == "USER_MANAGED")] as $user_keys | - if ($user_keys | length) > 10 then error("too many user-managed keys") - elif any($user_keys[]; - (.name | type) != "string" or - (.name | startswith($prefix) | not) or - ((.disabled // false) | type) != "boolean") - then error("invalid user-managed key") - elif ([$user_keys[].name] | unique | length) != ($user_keys | length) - then error("duplicate user-managed key") - else - [$user_keys[] | {name: .name, disabled: (.disabled // false)}] | sort_by(.name) - end - end - ' <<<"$keys_response")" || { + normalized_keys="$(jq -cer --arg prefix "$SA_RESOURCE/keys/" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-normalize-user-keys.jq" \ + <<<"$keys_response")" || { log_error "IAM returned an invalid user-managed key inventory" return 1 } @@ -444,21 +459,26 @@ list_user_keys() { log_error "IAM returned an invalid user-managed key name" return 1 fi - done < <(jq -r '.[].name | @base64' <<<"$normalized_keys") + done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-key-names-base64.jq" \ + <<<"$normalized_keys") printf '%s\n' "$normalized_keys" } list_user_key_names() { - list_user_keys | jq -c '[.[].name] | sort' + list_user_keys | jq -c -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-key-names-sorted.jq" } key_remote_status() { local key_name="$1" keys keys="$(list_user_keys)" || return 1 - if ! jq -e --arg name "$key_name" 'any(.[]; .name == $name)' <<<"$keys" >/dev/null; then + if ! jq -e --arg name "$key_name" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-key-present.jq" \ + <<<"$keys" >/dev/null; then printf 'absent\n' - elif jq -e --arg name "$key_name" 'any(.[]; .name == $name and .disabled == true)' <<<"$keys" >/dev/null; then + elif jq -e --arg name "$key_name" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-key-disabled.jq" \ + <<<"$keys" >/dev/null; then printf 'disabled\n' else printf 'enabled\n' @@ -550,14 +570,9 @@ validate_credentials_key() { jq -e \ --arg key_id "$expected_key_id" \ --arg service_account "$SERVICE_ACCOUNT" \ - --arg project_id "$PROJECT_ID" ' - .type == "service_account" and - .private_key_id == $key_id and - .client_email == $service_account and - .project_id == $project_id and - .token_uri == "https://oauth2.googleapis.com/token" and - (.private_key | type == "string" and startswith("-----BEGIN PRIVATE KEY-----") and contains("-----END PRIVATE KEY-----")) - ' "$file" >/dev/null + --arg project_id "$PROJECT_ID" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/service-account-credentials-valid.jq" \ + "$file" >/dev/null } install_credentials_file() { @@ -618,8 +633,10 @@ authenticate_credentials_once() { log_error "openssl is required to authenticate replacement credentials" return 1 } - email="$(jq -er '.client_email' "$file")" || return 1 - token_uri="$(jq -er '.token_uri' "$file")" || return 1 + email="$(jq -er --arg field client_email \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field.jq" "$file")" || return 1 + token_uri="$(jq -er --arg field token_uri \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field.jq" "$file")" || return 1 private_key_tmp="$(mktemp "${credentials_dir}/.auth-key.XXXXXX")" || return 1 request_tmp="$(mktemp "${credentials_dir}/.auth-request.XXXXXX")" || { rm -f -- "$private_key_tmp" @@ -630,7 +647,9 @@ authenticate_credentials_once() { return 1 } chmod 0600 "$private_key_tmp" "$request_tmp" "$response_tmp" - jq -er '.private_key' "$file" >"$private_key_tmp" || { + jq -er --arg field private_key \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/object-field.jq" \ + "$file" >"$private_key_tmp" || { rm -f -- "$private_key_tmp" "$request_tmp" "$response_tmp" return 1 } @@ -641,7 +660,7 @@ authenticate_credentials_once() { --arg iss "$email" \ --arg aud "$token_uri" \ --argjson iat "$now" \ - '{iss: $iss, scope: "https://www.googleapis.com/auth/cloud-platform", aud: $aud, iat: $iat, exp: ($iat + 3600)}' | base64url)" + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-jwt-claims.jq" | base64url)" signing_input="${header}.${payload}" signature="$(printf '%s' "$signing_input" | openssl dgst -sha256 -sign "$private_key_tmp" | base64url)" || { rm -f -- "$private_key_tmp" "$request_tmp" "$response_tmp" @@ -657,7 +676,9 @@ authenticate_credentials_once() { "$token_uri" || curl_status=$? rm -f -- "$private_key_tmp" "$request_tmp" if ((curl_status != 0)) || - ! jq -e '.access_token | type == "string" and length > 0' "$response_tmp" >/dev/null; then + ! jq -e --arg field access_token \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/string-field-valid.jq" \ + "$response_tmp" >/dev/null; then rm -f -- "$response_tmp" return 1 fi @@ -752,7 +773,8 @@ create_replacement_key() { "https://iam.googleapis.com/v1/$SA_RESOURCE/keys")" || curl_status=$? rm -f -- "$ACCESS_HEADER_FILE" - new_key_name="$(jq -er '.name | select(type == "string" and length > 0)' \ + new_key_name="$(jq -er --arg field name \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/nonempty-string-field.jq" \ "$response_tmp" 2>/dev/null)" || new_key_name="" if [[ -z "$new_key_name" ]]; then cleanup_ephemeral_create_response @@ -774,7 +796,8 @@ create_replacement_key() { STATE_NEW_KEY_ID="$new_key_id" STATE_NEW_KEY_NAME="$new_key_name" - private_key_data="$(jq -er '.privateKeyData | select(type == "string" and length > 0)' \ + private_key_data="$(jq -er --arg field privateKeyData \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/nonempty-string-field.jq" \ "$response_tmp" 2>/dev/null)" || private_key_data="" cleanup_ephemeral_create_response if [[ -z "$private_key_data" ]]; then @@ -837,25 +860,30 @@ finish_orphan_reconciliation() { log_error "Failed to delete fresh-filesystem orphan key ${key_name##*/}; reconciliation will retry" return 1 fi - done < <(jq -r '.[] | @base64' <<<"$STATE_BASELINE_KEY_NAMES") + done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/array-values-base64.jq" \ + <<<"$STATE_BASELINE_KEY_NAMES") current_names="$(list_user_key_names)" || return 1 remaining_baseline="$(jq -cn \ --argjson before "$STATE_BASELINE_KEY_NAMES" \ --argjson after "$current_names" \ - '$after | map(select(. as $name | $before | index($name))) | sort')" || return 1 + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-remaining-baseline.jq")" || return 1 unexpected="$(jq -cn \ --argjson before "$STATE_BASELINE_KEY_NAMES" \ --argjson after "$current_names" \ - '$after - $before | sort')" || return 1 + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-new-key-names.jq")" || return 1 - if [[ "$(jq -r 'length' <<<"$unexpected")" != "0" ]]; then - unexpected_ids="$(jq -r '[.[] | split("/")[-1]] | join(", ")' <<<"$unexpected")" || return 1 + if [[ "$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/json-length.jq" \ + <<<"$unexpected")" != "0" ]]; then + unexpected_ids="$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-key-ids-join.jq" \ + <<<"$unexpected")" || return 1 log_error "Unexpected concurrent user-managed keys appeared during fresh-filesystem reconciliation: $unexpected_ids" return 1 fi - if [[ "$(jq -r 'length' <<<"$remaining_baseline")" != "0" ]]; then - remaining_ids="$(jq -r '[.[] | split("/")[-1]] | join(", ")' <<<"$remaining_baseline")" || return 1 + if [[ "$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/json-length.jq" \ + <<<"$remaining_baseline")" != "0" ]]; then + remaining_ids="$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-key-ids-join.jq" \ + <<<"$remaining_baseline")" || return 1 log_error "Deleted orphan keys are still visible and will be retried: $remaining_ids" return 1 fi @@ -993,9 +1021,11 @@ prepare_rotation() { fetch_access_token || return 1 baseline_key_names="$(list_user_key_names)" || return 1 - baseline_count="$(jq -r 'length' <<<"$baseline_key_names")" || return 1 + baseline_count="$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/json-length.jq" \ + <<<"$baseline_key_names")" || return 1 if ((10#$baseline_count >= 10)); then - baseline_ids="$(jq -r '[.[] | split("/")[-1]] | join(", ")' <<<"$baseline_key_names")" || return 1 + baseline_ids="$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-key-ids-join.jq" \ + <<<"$baseline_key_names")" || return 1 log_error "Service account already has the 10 user-managed keys allowed by IAM: $baseline_ids" return 1 fi @@ -1202,11 +1232,14 @@ retire_credentials() { # remaining IDs; never guess which externally managed key to delete. fetch_access_token || return 1 remaining_keys="$(list_user_keys)" || return 1 - if [[ "$(jq -r 'length' <<<"$remaining_keys")" == "0" ]]; then + if [[ "$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/json-length.jq" \ + <<<"$remaining_keys")" == "0" ]]; then log_info "No local credential or remote user-managed key remains; retirement is complete" return 0 fi - remaining_key_ids="$(jq -r '[.[].name | split("/")[-1]] | sort | join(", ")' <<<"$remaining_keys")" || return 1 + remaining_key_ids="$(jq -r \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-key-object-ids-join.jq" \ + <<<"$remaining_keys")" || return 1 log_error "No local credential is available to identify the managed key, but remote user-managed keys remain: $remaining_key_ids" log_error "Audit and explicitly revoke the remaining key IDs before disabling managed credentials" return 1 @@ -1241,14 +1274,14 @@ creation_candidates() { fetch_access_token || return 1 current_names="$(list_user_key_names)" || return 1 jq -cn --argjson before "$STATE_BASELINE_KEY_NAMES" --argjson after "$current_names" \ - '$after - $before | sort' + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-new-key-names.jq" } rotation_audit() { local candidates='[]' grace_remaining=0 elapsed if [[ ! -e "$PENDING_STATE" && ! -L "$PENDING_STATE" ]]; then - jq -n '{version: 1, phase: "idle", recovery_required: false, candidate_key_ids: [], grace_remaining_seconds: 0}' + jq -n -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-idle-audit.jq" return 0 fi load_state || return 1 @@ -1269,19 +1302,8 @@ rotation_audit() { --argjson created_at "$STATE_CREATED_AT" \ --argjson ready_at "$STATE_READY_AT" \ --argjson disabled_at "$STATE_DISABLED_AT" \ - --argjson grace_remaining "$grace_remaining" ' - { - version: 1, - phase: $phase, - current_key_id: $current_key_id, - new_key_id: $new_key_id, - recovery_required: ($phase == "creating" or $phase == "creating-fresh"), - candidate_key_ids: [$candidate_names[] | split("/")[-1]], - created_at: $created_at, - ready_at: $ready_at, - disabled_at: $disabled_at, - grace_remaining_seconds: $grace_remaining - }' + --argjson grace_remaining "$grace_remaining" \ + -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-audit.jq" } recover_ambiguous_creation() { @@ -1299,7 +1321,8 @@ recover_ambiguous_creation() { return 1 fi candidates="$(creation_candidates)" || return 1 - if [[ "$(jq -r 'length' <<<"$candidates")" == "0" ]]; then + if [[ "$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/json-length.jq" \ + <<<"$candidates")" == "0" ]]; then if [[ "$STATE_PHASE" == "creating-fresh" ]]; then STATE_PHASE=reconciling STATE_BASELINE_KEY_NAMES='[]' @@ -1311,12 +1334,14 @@ recover_ambiguous_creation() { fi return 0 fi - if [[ "$(jq -r 'length' <<<"$candidates")" != "1" ]]; then + if [[ "$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/json-length.jq" \ + <<<"$candidates")" != "1" ]]; then rotation_audit log_error "Recovery is ambiguous because more than one post-baseline key exists; no key was changed" return 1 fi - candidate_id="$(jq -r '.[0] | split("/")[-1]' <<<"$candidates")" + candidate_id="$(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/rotation-first-key-id.jq" \ + <<<"$candidates")" if [[ -z "$RECOVERY_KEY_ID" || "$RECOVERY_KEY_ID" != "$candidate_id" ]]; then rotation_audit log_error "Recovery requires the single audited candidate key ID as the final argument; no key was changed" diff --git a/rootfs/home/cloud-compose/run-rollout-service.sh b/rootfs/home/cloud-compose/run-rollout-service.sh index 413973f..7857425 100644 --- a/rootfs/home/cloud-compose/run-rollout-service.sh +++ b/rootfs/home/cloud-compose/run-rollout-service.sh @@ -2,8 +2,40 @@ set -euo pipefail +_cc_run_rollout_service_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_run_rollout_service_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_run_rollout_service_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_run_rollout_service_source _cc_run_rollout_service_dir _cc_run_rollout_service_installed_home +if [[ -n "$_cc_run_rollout_service_installed_home" && + ( "$_cc_run_rollout_service_installed_home" == "/" || + "$_cc_run_rollout_service_source" == "${_cc_run_rollout_service_installed_home%/}/"* ) ]]; then + _cc_run_rollout_service_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_run_rollout_service_checked_programs="$_cc_run_rollout_service_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_run_rollout_service_checked_programs +# shellcheck disable=SC1090 +source "$_cc_run_rollout_service_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_run_rollout_service_source" \ + CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh \ + "$_cc_run_rollout_service_dir/profile.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +readonly profile_path + # shellcheck disable=SC1090,SC1091 -source "${CLOUD_COMPOSE_PROFILE_PATH:-/home/cloud-compose/profile.sh}" +source "$profile_path" + +# Reload the fixed resolver after the profile before binding data programs. +# shellcheck disable=SC1090 +source "$_cc_run_rollout_service_checked_programs" +cloud_compose_bind_program_dir \ + "$_cc_run_rollout_service_source" \ + CLOUD_COMPOSE_JQ_PROGRAM_DIR \ + /etc/cloud-compose/jq \ + "$_cc_run_rollout_service_dir/../../etc/cloud-compose/jq" \ + json-object-validate.jq unset BASH_ENV ENV LD_PRELOAD LD_LIBRARY_PATH PORT="${ROLLOUT_PORT:?ROLLOUT_PORT is required}" @@ -24,7 +56,8 @@ if [[ "$JWT_AUD" == *$'\n'* || "$JWT_AUD" == *$'\r'* || -z "$JWT_AUD" ]]; then exit 2 fi if [[ -n "$CUSTOM_CLAIMS" ]] && - ! jq -e 'type == "object"' <<<"$CUSTOM_CLAIMS" >/dev/null; then + ! jq -e -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/json-object-validate.jq" \ + <<<"$CUSTOM_CLAIMS" >/dev/null; then echo "ROLLOUT_CUSTOM_CLAIMS must be empty or a JSON object" >&2 exit 2 fi diff --git a/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh b/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh index 79f2480..1016806 100644 --- a/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh +++ b/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh @@ -2,8 +2,28 @@ set -euo pipefail +_cc_start_bootstrap_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_start_bootstrap_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_start_bootstrap_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_start_bootstrap_source _cc_start_bootstrap_dir _cc_start_bootstrap_installed_home +if [[ -n "$_cc_start_bootstrap_installed_home" && + ( "$_cc_start_bootstrap_installed_home" == "/" || + "$_cc_start_bootstrap_source" == "${_cc_start_bootstrap_installed_home%/}/"* ) ]]; then + _cc_start_bootstrap_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_start_bootstrap_checked_programs="$_cc_start_bootstrap_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_start_bootstrap_checked_programs +# shellcheck disable=SC1090 +source "$_cc_start_bootstrap_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_start_bootstrap_source" CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH \ + /home/cloud-compose/bootstrap-helpers.sh "$_cc_start_bootstrap_dir/bootstrap-helpers.sh" +bootstrap_helpers_path="$CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH" +readonly bootstrap_helpers_path + # shellcheck disable=SC1090 -source "${CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH:-/home/cloud-compose/bootstrap-helpers.sh}" +source "$bootstrap_helpers_path" durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/var/lib/cloud-compose/bootstrap-complete}" wait_seconds="${CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS:-10800}" diff --git a/rootfs/home/cloud-compose/vault-agent-readiness.sh b/rootfs/home/cloud-compose/vault-agent-readiness.sh index af9f7df..8549eb7 100644 --- a/rootfs/home/cloud-compose/vault-agent-readiness.sh +++ b/rootfs/home/cloud-compose/vault-agent-readiness.sh @@ -2,7 +2,28 @@ set -euo pipefail -profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-/home/cloud-compose/profile.sh}" +_cc_vault_agent_readiness_source="$(readlink -f -- "${BASH_SOURCE[0]}")" +_cc_vault_agent_readiness_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +_cc_vault_agent_readiness_installed_home="$(readlink -f -- /home/cloud-compose 2>/dev/null || true)" +readonly _cc_vault_agent_readiness_source _cc_vault_agent_readiness_dir _cc_vault_agent_readiness_installed_home +if [[ -n "$_cc_vault_agent_readiness_installed_home" && + ( "$_cc_vault_agent_readiness_installed_home" == "/" || + "$_cc_vault_agent_readiness_source" == "${_cc_vault_agent_readiness_installed_home%/}/"* ) ]]; then + _cc_vault_agent_readiness_checked_programs=/etc/cloud-compose/libexec/checked-programs.bash +else + _cc_vault_agent_readiness_checked_programs="$_cc_vault_agent_readiness_dir/../../etc/cloud-compose/libexec/checked-programs.bash" +fi +readonly _cc_vault_agent_readiness_checked_programs +# shellcheck disable=SC1090 +source "$_cc_vault_agent_readiness_checked_programs" +cloud_compose_bind_source_program \ + "$_cc_vault_agent_readiness_source" \ + CLOUD_COMPOSE_PROFILE_PATH \ + /home/cloud-compose/profile.sh \ + "$_cc_vault_agent_readiness_dir/profile.sh" +profile_path="$CLOUD_COMPOSE_PROFILE_PATH" +readonly profile_path + # shellcheck disable=SC1090 source "$profile_path" diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls index de3f914..9cd2151 100644 --- a/salt/cloud-compose/init.sls +++ b/salt/cloud-compose/init.sls @@ -661,7 +661,9 @@ cloud-compose-rootfs: cloud-compose-privileged-program-directories: file.directory: - names: + - {{ home | json }} - /etc/cloud-compose + - /etc/cloud-compose/awk - /etc/cloud-compose/bin - /etc/cloud-compose/jq - /etc/cloud-compose/libexec @@ -692,6 +694,17 @@ cloud-compose-rootfs-script-modes: - file: cloud-compose-rootfs - file: cloud-compose-privileged-program-directories +cloud-compose-checked-program-resolver: + file.managed: + - name: /etc/cloud-compose/libexec/checked-programs.bash + - source: salt://rootfs/etc/cloud-compose/libexec/checked-programs.bash + - user: root + - group: root + - mode: '0644' + - require: + - file: cloud-compose-rootfs + - file: cloud-compose-privileged-program-directories + cloud-compose-rootfs-jq-modes: cmd.run: - name: find /etc/cloud-compose/jq -maxdepth 1 -type f -name '*.jq' -exec chown root:root {} + -exec chmod 0644 {} + @@ -700,6 +713,14 @@ cloud-compose-rootfs-jq-modes: - file: cloud-compose-rootfs - file: cloud-compose-privileged-program-directories +cloud-compose-rootfs-awk-modes: + cmd.run: + - name: find /etc/cloud-compose/awk -maxdepth 1 -type f -name '*.awk' -exec chown root:root {} + -exec chmod 0644 {} + + - unless: test -z "$(find /etc/cloud-compose/awk -maxdepth 1 -type f -name '*.awk' \( ! -user root -o ! -group root -o ! -perm 0644 \) -print -quit)" + - require: + - file: cloud-compose-rootfs + - file: cloud-compose-privileged-program-directories + {% for lifecycle in ['init', 'up', 'down', 'rollout'] %} cloud-compose-lifecycle-{{ lifecycle }}: file.managed: @@ -779,7 +800,9 @@ cloud-compose-bootstrap-paths-hardened: - file: cloud-compose-project-manifest - file: cloud-compose-managed-runtime-artifacts - cmd: cloud-compose-rootfs-script-modes + - file: cloud-compose-checked-program-resolver - cmd: cloud-compose-rootfs-jq-modes + - cmd: cloud-compose-rootfs-awk-modes {% for lifecycle in ['init', 'up', 'down', 'rollout'] %} - file: cloud-compose-lifecycle-{{ lifecycle }} {% endfor %} @@ -805,6 +828,9 @@ cloud-compose-rollout-service: - file: cloud-compose-rootfs - cmd: cloud-compose-lifecycle-lock - cmd: cloud-compose-rootfs-script-modes + - file: cloud-compose-checked-program-resolver + - cmd: cloud-compose-rootfs-jq-modes + - cmd: cloud-compose-rootfs-awk-modes - file: cloud-compose-lifecycle-init - file: cloud-compose-lifecycle-up - file: cloud-compose-lifecycle-down @@ -838,7 +864,9 @@ cloud-compose-bootstrap: - file: cloud-compose-project-manifest - file: cloud-compose-managed-runtime-artifacts - cmd: cloud-compose-rootfs-script-modes + - file: cloud-compose-checked-program-resolver - cmd: cloud-compose-rootfs-jq-modes + - cmd: cloud-compose-rootfs-awk-modes - cmd: cloud-compose-bootstrap-paths-hardened {% for lifecycle in ['init', 'up', 'down', 'rollout'] %} - file: cloud-compose-lifecycle-{{ lifecycle }} diff --git a/templates/gcp-filesystem-boothook.sh.tftpl b/templates/gcp-filesystem-boothook.sh.tftpl index 08d5235..cd35666 100644 --- a/templates/gcp-filesystem-boothook.sh.tftpl +++ b/templates/gcp-filesystem-boothook.sh.tftpl @@ -6,14 +6,17 @@ umask 077 readonly filesystem_boot=/run/cloud-compose-gcp-filesystem-boot readonly filesystem_prep=/run/cloud-compose-prepare-filesystem readonly filesystem_persist=/run/cloud-compose-persist-filesystems +readonly filesystem_reconcile=/run/cloud-compose-reconcile-fstab.awk printf '%s' '${GCP_FILESYSTEM_BOOT_SCRIPT_B64}' | base64 -d >"$filesystem_boot" printf '%s' '${FILESYSTEM_PREP_SCRIPT_B64}' | base64 -d >"$filesystem_prep" printf '%s' '${FILESYSTEM_PERSIST_SCRIPT_B64}' | base64 -d >"$filesystem_persist" -chmod 0600 "$filesystem_boot" "$filesystem_prep" "$filesystem_persist" +printf '%s' '${FSTAB_RECONCILE_AWK_B64}' | base64 -d >"$filesystem_reconcile" +chmod 0600 "$filesystem_boot" "$filesystem_prep" "$filesystem_persist" "$filesystem_reconcile" bash "$filesystem_boot" \ ${jsonencode(FRESH_FILESYSTEM_IDENTITY)} \ ${jsonencode(USE_OVERLAY ? "true" : "false")} \ "$filesystem_prep" \ - "$filesystem_persist" + "$filesystem_persist" \ + "$filesystem_reconcile" From 7357502c9cd18289bb97b0e64304a971305b8f71 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 20:40:59 +0000 Subject: [PATCH 66/67] [patch] Follow checked key validation program --- ci/host-runtime-security.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/host-runtime-security.sh b/ci/host-runtime-security.sh index bd252d9..f133601 100644 --- a/ci/host-runtime-security.sh +++ b/ci/host-runtime-security.sh @@ -249,7 +249,8 @@ fi assert_contains "$rotate_script" 'prepare|status|audit|recover|authenticate|ready|commit|rollback|rollback-ready' assert_contains "$rotate_script" 'ROTATION_DISABLE_GRACE_SECONDS' -assert_contains "$rotate_script" 'https://oauth2.googleapis.com/token' +assert_contains "$repo_root/rootfs/etc/cloud-compose/jq/service-account-credentials-valid.jq" \ + 'https://oauth2.googleapis.com/token' assert_contains "$rotate_script" 'KEY_OPERATION_RESULT=absent' run_script="$repo_root/rootfs/home/cloud-compose/run.sh" From 9e4d05cb8addb056a6e8bcabcab69598f6d9d322 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sun, 9 Aug 2026 20:53:04 +0000 Subject: [PATCH 67/67] [patch] Preserve GCP bootstrap metadata headroom --- ci/filesystem-prep-contract.sh | 8 ++++++++ docs/runtime-contracts.md | 5 +++++ modules/gcp/main.tf | 24 +++++++++++++--------- modules/gcp/runtime_contracts.tftest.hcl | 11 +++++++--- templates/cloud-init.yml | 8 ++++---- templates/gcp-filesystem-boothook.sh.tftpl | 8 ++++---- 6 files changed, 43 insertions(+), 21 deletions(-) diff --git a/ci/filesystem-prep-contract.sh b/ci/filesystem-prep-contract.sh index 0c37ffc..d9006fe 100644 --- a/ci/filesystem-prep-contract.sh +++ b/ci/filesystem-prep-contract.sh @@ -601,6 +601,14 @@ if grep -Eq 'fsck[^\n]*\|\|[^\n]*mkfs|fsck[^\n]*mkfs' "$repo_root/templates/clou fi grep -Fq 'FILESYSTEM_PREP_SCRIPT_B64' "$gcp_filesystem_boothook" || \ fail "GCP boothook does not bootstrap the tested filesystem helper" +for compressed_boot_program in \ + 'base64 -d | gzip -d >"$filesystem_boot"' \ + 'base64 -d | gzip -d >"$filesystem_prep"' \ + 'base64 -d | gzip -d >"$filesystem_persist"' \ + 'base64 -d | gzip -d >"$filesystem_reconcile"'; do + grep -Fq "$compressed_boot_program" "$gcp_filesystem_boothook" || \ + fail "GCP boothook does not unpack a compressed checked program: $compressed_boot_program" +done grep -Fq 'bash "$filesystem_boot"' "$gcp_filesystem_boothook" || \ fail "GCP boothook does not invoke the checked filesystem program" grep -Fq 'Content-Type: text/cloud-boothook' "$gcp_cloud_init_mime" || \ diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 7917933..0464d5b 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -221,6 +221,11 @@ AWK program from the same Terraform source or already verified rootfs archive, requires an unlinked root-owned mode-0600 file, and invokes it as data with `awk -f` before the full stateless rootfs is available. +GCP compresses checked bootstrap programs before carrying them in user-data and +enforces a 240 KiB plan-time budget. This leaves explicit headroom below the +provider's 256 KiB metadata-item limit; larger custom inputs must use the +verified rootfs archive path or be reduced before a VM can be replaced. + ## Sitectl During init, the VM creates a `sitectl` context for every app. The default `up` diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf index 13fcfa2..64efa0e 100644 --- a/modules/gcp/main.tf +++ b/modules/gcp/main.tf @@ -270,8 +270,8 @@ EOT - path: "/home/cloud-compose/${name}" owner: "root:root" permissions: "0755" - encoding: b64 - content: ${filebase64("${local.rootFs}/home/cloud-compose/lifecycle-entrypoint.sh")} + encoding: gzip+base64 + content: ${base64gzip(file("${local.rootFs}/home/cloud-compose/lifecycle-entrypoint.sh"))} EOT ]) compose_projects_file = <<-EOT @@ -446,18 +446,18 @@ EOT use_overlay = length(var.volume_names) > 0 prod_disk_url = var.overlay_source_instance != "" ? format("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s-docker-volumes", var.project_id, var.zone, var.overlay_source_instance) : "" gcp_filesystem_boothook = templatefile("${path.module}/../../templates/gcp-filesystem-boothook.sh.tftpl", { - FILESYSTEM_PREP_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/prepare-filesystem.sh"), - FILESYSTEM_PERSIST_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/persist-filesystems.sh"), - FSTAB_RECONCILE_AWK_B64 = filebase64("${local.rootFs}/etc/cloud-compose/awk/reconcile-fstab.awk"), - GCP_FILESYSTEM_BOOT_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/libexec/gcp-filesystem-boot.sh"), + FILESYSTEM_PREP_SCRIPT_B64 = base64gzip(file("${local.rootFs}/home/cloud-compose/prepare-filesystem.sh")), + FILESYSTEM_PERSIST_SCRIPT_B64 = base64gzip(file("${local.rootFs}/home/cloud-compose/persist-filesystems.sh")), + FSTAB_RECONCILE_AWK_B64 = base64gzip(file("${local.rootFs}/etc/cloud-compose/awk/reconcile-fstab.awk")), + GCP_FILESYSTEM_BOOT_SCRIPT_B64 = base64gzip(file("${local.rootFs}/etc/cloud-compose/libexec/gcp-filesystem-boot.sh")), FRESH_FILESYSTEM_IDENTITY = "v1:gcp-disk-id:${google_compute_disk.data.disk_id}", USE_OVERLAY = local.use_overlay, }) cloud_config_yaml = templatefile("${path.module}/../../templates/cloud-init.yml", { - ROOTFS_ARCHIVE_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/libexec/rootfs-archive.sh"), - GCP_CLOUD_INIT_FINALIZE_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh"), - GCP_CLOUD_INIT_POST_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/libexec/gcp-cloud-init-post-bootstrap.sh"), - DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"), + ROOTFS_ARCHIVE_SCRIPT_B64 = base64gzip(file("${local.rootFs}/etc/cloud-compose/libexec/rootfs-archive.sh")), + GCP_CLOUD_INIT_FINALIZE_SCRIPT_B64 = base64gzip(file("${local.rootFs}/etc/cloud-compose/libexec/gcp-cloud-init-finalize.sh")), + GCP_CLOUD_INIT_POST_SCRIPT_B64 = base64gzip(file("${local.rootFs}/etc/cloud-compose/libexec/gcp-cloud-init-post-bootstrap.sh")), + DIAGNOSTICS_SCRIPT_B64 = base64gzip(file("${local.rootFs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh")), DIAGNOSTICS_SCRIPT_SHA256 = filesha256("${local.rootFs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"), INIT_COMMANDS_B64 = base64encode(local.initcmd_content), RUNCMD_B64 = base64encode(local.runcmd_content), @@ -875,6 +875,10 @@ resource "google_compute_instance" "cloud-compose" { } lifecycle { + precondition { + condition = length(data.cloudinit_config.ci.part[0].content) <= 245760 + error_message = "GCP user-data must stay within a 240 KiB review budget so the 256 KiB metadata-item limit retains bootstrap headroom. Select a verified rootfs archive or reduce unusually large runtime inputs." + } precondition { condition = var.project_number == "" || var.project_number == local.project_number error_message = "project_number does not match the number derived from project_id; omit the deprecated assertion or correct it." diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl index bc2a53d..e1a70a5 100644 --- a/modules/gcp/runtime_contracts.tftest.hcl +++ b/modules/gcp/runtime_contracts.tftest.hcl @@ -102,11 +102,11 @@ run "disables_privileged_services_by_default" { ) && strcontains( local.cloud_init_yaml, - filebase64("${path.module}/../../rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh"), + base64gzip(file("${path.module}/../../rootfs/etc/cloud-compose/libexec/gcp-filesystem-boot.sh")), ) && strcontains( local.cloud_init_yaml, - filebase64("${path.module}/../../rootfs/etc/cloud-compose/awk/reconcile-fstab.awk"), + base64gzip(file("${path.module}/../../rootfs/etc/cloud-compose/awk/reconcile-fstab.awk")), ) && strcontains( local.write_files_content, @@ -117,6 +117,11 @@ run "disables_privileged_services_by_default" { error_message = "GCP's early every-boot filesystem programs and root runtime environment must carry the same immutable data-disk identity and checked fstab reconciler." } + assert { + condition = length(data.cloudinit_config.ci.part[0].content) <= 245760 + error_message = "The default GCP user-data must preserve headroom below the provider's metadata-item limit." + } + assert { condition = ( google_project_iam_member.log.member == "serviceAccount:${local.vm_service_account_email}" && @@ -584,7 +589,7 @@ run "renders_verified_archive_before_downstream_overlay" { condition = ( strcontains( local.cloud_init_yaml, - filebase64("${path.module}/../../rootfs/etc/cloud-compose/libexec/rootfs-archive.sh"), + base64gzip(file("${path.module}/../../rootfs/etc/cloud-compose/libexec/rootfs-archive.sh")), ) && strcontains(local.cloud_init_yaml, local.rootfs_contract_sha256) && data.http.rootfs_contract[0].url == "https://example.invalid/cloud-compose-rootfs.contract.sha256" && diff --git a/templates/cloud-init.yml b/templates/cloud-init.yml index 202361f..f51bb08 100644 --- a/templates/cloud-init.yml +++ b/templates/cloud-init.yml @@ -32,22 +32,22 @@ write_files: - path: /var/lib/cloud-compose/bootstrap/rootfs-archive.sh owner: root:root permissions: "0700" - encoding: b64 + encoding: gzip+base64 content: ${ROOTFS_ARCHIVE_SCRIPT_B64} - path: /var/lib/cloud-compose/bootstrap/gcp-cloud-init-finalize.sh owner: root:root permissions: "0700" - encoding: b64 + encoding: gzip+base64 content: ${GCP_CLOUD_INIT_FINALIZE_SCRIPT_B64} - path: /var/lib/cloud-compose/bootstrap/gcp-cloud-init-post-bootstrap.sh owner: root:root permissions: "0700" - encoding: b64 + encoding: gzip+base64 content: ${GCP_CLOUD_INIT_POST_SCRIPT_B64} - path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh owner: root:root permissions: "0600" - encoding: b64 + encoding: gzip+base64 content: ${DIAGNOSTICS_SCRIPT_B64} - path: /var/lib/cloud-compose/bootstrap/init-commands.sh owner: root:root diff --git a/templates/gcp-filesystem-boothook.sh.tftpl b/templates/gcp-filesystem-boothook.sh.tftpl index cd35666..3641a7b 100644 --- a/templates/gcp-filesystem-boothook.sh.tftpl +++ b/templates/gcp-filesystem-boothook.sh.tftpl @@ -8,10 +8,10 @@ readonly filesystem_prep=/run/cloud-compose-prepare-filesystem readonly filesystem_persist=/run/cloud-compose-persist-filesystems readonly filesystem_reconcile=/run/cloud-compose-reconcile-fstab.awk -printf '%s' '${GCP_FILESYSTEM_BOOT_SCRIPT_B64}' | base64 -d >"$filesystem_boot" -printf '%s' '${FILESYSTEM_PREP_SCRIPT_B64}' | base64 -d >"$filesystem_prep" -printf '%s' '${FILESYSTEM_PERSIST_SCRIPT_B64}' | base64 -d >"$filesystem_persist" -printf '%s' '${FSTAB_RECONCILE_AWK_B64}' | base64 -d >"$filesystem_reconcile" +printf '%s' '${GCP_FILESYSTEM_BOOT_SCRIPT_B64}' | base64 -d | gzip -d >"$filesystem_boot" +printf '%s' '${FILESYSTEM_PREP_SCRIPT_B64}' | base64 -d | gzip -d >"$filesystem_prep" +printf '%s' '${FILESYSTEM_PERSIST_SCRIPT_B64}' | base64 -d | gzip -d >"$filesystem_persist" +printf '%s' '${FSTAB_RECONCILE_AWK_B64}' | base64 -d | gzip -d >"$filesystem_reconcile" chmod 0600 "$filesystem_boot" "$filesystem_prep" "$filesystem_persist" "$filesystem_reconcile" bash "$filesystem_boot" \