From e48247728ac7f4d08e52f1ae17cf82cfe2399ca6 Mon Sep 17 00:00:00 2001 From: Musa Misto <64855513+MusaMisto@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:18:49 +0300 Subject: [PATCH] ci: deploy releases/r10.0 to production, without the develop release Brings the deployment side of develop's pipeline to r10.0, and none of its application code, so production can run r10.0 until develop is released: - The workflow is develop's: the develop job runs only on develop, and the release job deploys to bitween on the production cluster (app.bitween.io on s9-prod-edge's https-wildcard-bitween-io listener) while BITWEEN_PROD_DEPLOY is true. It also publishes the chart to GHCR, which the deploy pulls from. Left out until the release: the CLI and SDK jobs and the SW.Bitween.Adapters package, whose code is only on develop. The old staging job's deploy to playground is gone with it. - The chart passes any other environmentVariables to the container, using develop's own lines. That's how production's AWS_*_CHECKSUM_* settings reach the pod. With default values the container gets the same 21 variables as before. --- .../workflows/bitween-api-cicd-gateway.yml | 150 +++++++++++++----- charts/default/templates/deployment.yaml | 9 ++ docs/deployment.md | 3 +- 3 files changed, 118 insertions(+), 44 deletions(-) diff --git a/.github/workflows/bitween-api-cicd-gateway.yml b/.github/workflows/bitween-api-cicd-gateway.yml index 421f4ceb..0463c5ec 100644 --- a/.github/workflows/bitween-api-cicd-gateway.yml +++ b/.github/workflows/bitween-api-cicd-gateway.yml @@ -1,20 +1,29 @@ name: Bitween API CI/CD (Gateway · both) run-name: Run-${{ github.ref_name }}-${{ github.run_number }} +# Two pipelines that publish nothing under the same name: +# releases/r10.0-develop the simplify9/bitween-develop image and the bitween-develop chart, +# deployed to bitween-dev on the dev cluster (app-dev.bitween.io). No NuGet +# packages, the CLI as a pre-release. +# releases/r10.0 the simplify9/bitween image and the bitween chart clients install +# (ChartMuseum + GHCR), the NuGet packages, the CLI and the SDKs, deployed +# to bitween on the production cluster (app.bitween.io). +# Run workflow by hand on either branch to rerun that branch's pipeline. on: push: branches: - - releases/** + - releases/r10.0 + - releases/r10.0-develop workflow_dispatch: -# One queue for staging AND release, not one per branch. Both jobs below take their patch from -# the same 10.0.x git tags and push the same docker.io/simplify9/bitween:10.0. image tag, -# and a run only claims its patch when it tags at the very end. Two runs overlapping (a staging -# push, then the staging -> r10.0 merge seconds later) would compute the same patch: the image -# tag is overwritten by whichever finishes last, while ChartMuseum (409 = success) and NuGet -# (--skip-duplicate) silently keep whichever came first. +# One queue per branch. Both pipelines still take their patch from the same 10.0.x git tags, so a +# develop run and a release run that overlap can compute the same patch, but they publish it under +# different names (bitween-develop / bitween, cli-v10.0.N-dev. / cli-v10.0.N, git tag +# 10.0.N-releases-r10-0-develop. / 10.0.N), so nothing is overwritten or silently kept. One +# queue for both would be worse: GitHub keeps a single pending run per group, so a develop push +# would cancel a release waiting behind another develop run. concurrency: - group: cicd-gateway-bitween-10.0 + group: cicd-gateway-bitween-${{ github.ref_name }} cancel-in-progress: false # contents: write -> tag job. packages: write -> publish AND pull the Helm chart @@ -27,8 +36,8 @@ permissions: jobs: build-publish-deploy: - # Staging: build, publish to GHCR only, deploy to playground (or any branch, manually). - if: ${{ (github.event_name == 'push' && github.ref_name == 'releases/r10.0-staging') || github.event_name == 'workflow_dispatch' }} + # Develop: build, publish bitween-develop, deploy to bitween-dev. + if: ${{ github.ref_name == 'releases/r10.0-develop' }} uses: simplify9/.github/.github/workflows/reusable-service-cicd.yml@main with: major-version: '10' @@ -39,62 +48,78 @@ jobs: docker-context: '.' docker-platforms: 'linux/amd64' - chart-name: 'bitween' # must match charts/default Chart.yaml name: + # The production chart, published as bitween-develop: rename-chart sets Chart.yaml's name + # at package time, so develop and production ship from one chart directory and cannot drift. + chart-name: 'bitween-develop' chart-path: './charts/default' + rename-chart: true + # ChartMuseum to install it by hand, GHCR for the deploy below. + chart-publish-method: 'both' + chart-repo-url: 'https://charts.sf9.io' - # 1) Publish the Helm chart to GHCR OCI only. ChartMuseum is what clients install from, - # so only the release job below publishes there; staging versions are plain x.y.z and - # would otherwise be the newest chart a client gets. - chart-publish-method: 'github-oci' - - # Docker image stays on Docker Hub + # The chart's image is /:, so the image is named after + # the chart. Its own repository also keeps develop off simplify9/bitween:latest. container-registry: 'docker.io' - image-name: 'simplify9/bitween' + image-name: 'simplify9/bitween-develop' - # Build + unit-test + push the SDK NuGet package (versioned with the - # pipeline semver, e.g. 8.1.x — was 8.0. in the old job). - nuget-projects: 'SW.Bitween.Sdk/SW.Bitween.Sdk.csproj' + # No nuget-projects: develop publishes no packages. The tests still run (build and test, + # no pack); as before, a failure does not hold back the deploy. run-tests: 'true' - # Integration tests run against real Postgres/RabbitMQ/MailHog via - # Testcontainers — self-contained, no extra CI services needed, since - # ubuntu-latest already has a Docker daemon available. test-projects: | SW.Bitween.UnitTests/SW.Bitween.UnitTests.csproj SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj - # 2) Deploy to playground via Gateway API + # Deploy to bitween-dev via Gateway API deploy: true - # 3) Pull the chart from GHCR OCI + # Pull the chart from GHCR OCI deploy-chart-source: 'github-oci' - deploy-namespace: 'playground' + deploy-namespace: 'bitween-dev' + # Release bitween with nameOverride=bitween (helm-set-values), so every resource name and + # selector is what the bitween chart renders in production. Gateway API gives a hostname to + # its oldest HTTPRoute: any other release serving app-dev.bitween.io on this cluster keeps + # the host until it is uninstalled. + deploy-release-name: 'bitween' routing-mode: 'gateway-api' gateway-hostnames: 'app-dev.bitween.io' # '/' serves the admin SPA out of this image's wwwroot, so the root has to route - # here too — previously only /api and /swagger did, leaving the root on the retired + # here too - previously only /api and /swagger did, leaving the root on the retired # legacy Bitween-UI deployment and making every deploy look like the UI never changed. gateway-paths: '/,/api,/swagger' - # app-dev.bitween.io is not *.sf9.io, so the deploy auto-onboards a - # dedicated listener + cert-manager Certificate on the parent gateway - # (public-gateway · s9-dev-edge · cilium · letsencrypt-production-gateway). - - # Non-routing chart values (routing itself is generated by gateway-api mode). + # Attach to the dev gateway's *.bitween.io wildcard listener (public-gateway · s9-dev-edge), + # certificate included, instead of onboarding a listener and certificate for the host. The + # deploy fails before helm runs if the listener is missing. An app-dev.bitween.io listener on + # that gateway would take the host from the wildcard and answer 404, since this route no + # longer attaches to it. + gateway-section-name: 'https-wildcard-bitween-io' + + # Non-routing chart values (routing itself is generated by gateway-api mode). Every line is + # handed to helm --set as it stands, so no comments inside. replicaCount=2: the lab's phase 6 + # kills one mid-traffic, and anything that runs on every replica at once (scheduled jobs, data + # source leases) shows up here first. + # AWS_*_CHECKSUM_*=when_required: an experiment for S3-compatible stores. Since AWSSDK.S3 3.7.410 + # uploads are aws-chunked with a trailing checksum, which such stores save into the object + # (the lab's S3 invoices arrive starting "C3;chunk-signature=..."). The lasting fix belongs + # in SW-CloudFiles; this shows on staging whether the SDK default is the cause. helm-set-values: | + nameOverride=bitween global.environment=Development + replicaCount=2 resources.requests.cpu=25m resources.requests.memory=704Mi resources.limits.memory=1408Mi + environmentVariables.Bitween__BusProvidersEnabled=true + environmentVariables.AWS_REQUEST_CHECKSUM_CALCULATION=when_required + environmentVariables.AWS_RESPONSE_CHECKSUM_VALIDATION=when_required secrets: # Docker image push credentials (Docker Hub) registry-username: ${{ secrets.DOCKERHUB_USERNAME }} registry-password: ${{ secrets.DOCKERHUB_TOKEN }} + chartmuseum-username: ${{ secrets.CM_USER }} + chartmuseum-password: ${{ secrets.CM_PASSWORD }} # GHCR chart push + pull and git tagging use the built-in token github-token: ${{ secrets.GITHUB_TOKEN }} - # NuGet API key for the SDK package push (publishes to nuget.org, the - # reusable's default feed — no nuget-source override needed). - nuget-api-key: ${{ secrets.SWNUGETKEY }} - # Gateway-API cluster kubeconfig (V2) kubeconfig-gateway: ${{ secrets.S9DEV_KUBECONFIG_V2 }} @@ -103,17 +128,17 @@ jobs: # App configuration passed to helm --set-string during deploy (masked) # # global.token.key signs every authentication token. The chart still carries a sample - # key as its default, and that value is published in this public repository — anyone + # key as its default, and that value is published in this public repository - anyone # holding it can mint a token for any identity, so the app refuses to start on it. # Set the TokenKey secret to a long random string unique to this environment # (openssl rand -base64 48). Changing it signs out everyone holding an older token. helm-set-secret-values: 'global.token.key=${{ secrets.TokenKey }},db=${{ secrets.dbConnection }},global.bus.rabbitUrl=${{ secrets.rabbitUrl }},global.cloudFiles.secretAccessKey=${{ secrets.SecretAccessKey }},global.cloudFiles.accessKeyId=${{ secrets.AccessKeyId }},global.cloudFiles.serviceUrl=${{ secrets.ServiceUrl }},global.cloudFiles.bucketName=${{ secrets.BucketName }},environmentVariables.Bitween__RabbitMqManagementUrl=${{ secrets.RabbitMqManagementUrl }},environmentVariables.Bitween__RabbitMqManagementPassword=${{ secrets.RabbitMqManagementPassword }},environmentVariables.Bitween__RabbitMqManagementUsername=${{ secrets.RabbitMqManagementUsername }}' publish-release: - # Release: build and publish to ChartMuseum for clients, never deploy. releases/r10.0 is the + # Release: build and publish for clients, deploy production. releases/r10.0 is the # repository's default branch, so the reusable workflow treats it as a release: a clean # 10.0. git tag, and the critical Dependabot gate runs first. - if: ${{ github.event_name == 'push' && github.ref_name == 'releases/r10.0' }} + if: ${{ github.ref_name == 'releases/r10.0' }} uses: simplify9/.github/.github/workflows/reusable-service-cicd.yml@main with: major-version: '10' @@ -126,7 +151,9 @@ jobs: chart-name: 'bitween' chart-path: './charts/default' - chart-publish-method: 'chartmuseum' + # ChartMuseum, where clients install from today, and GHCR OCI (oci://ghcr.io/simplify9/charts/bitween), + # the same versions as an immutable, digest-pinned mirror. + chart-publish-method: 'both' chart-repo-url: 'https://charts.sf9.io' container-registry: 'docker.io' @@ -134,11 +161,41 @@ jobs: nuget-projects: 'SW.Bitween.Sdk/SW.Bitween.Sdk.csproj' run-tests: 'true' + # Integration tests run against real Postgres/RabbitMQ/MailHog via + # Testcontainers - self-contained, no extra CI services needed, since + # ubuntu-latest already has a Docker daemon available. test-projects: | SW.Bitween.UnitTests/SW.Bitween.UnitTests.csproj SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj - # deploy is left at its default, false: the deploy job never runs, so no kubeconfig - # or app secrets are passed. + # Deploy to bitween on the production cluster. Off until the repository variable + # BITWEEN_PROD_DEPLOY is 'true': first the PROD_* secrets below must exist, + # S9PROD_KUBECONFIG_V2 must be shared with this repository, and app.bitween.io must + # point at the production gateway. Without them a deploy installs a release with no + # database, or fails waiting for its certificate. + deploy: ${{ vars.BITWEEN_PROD_DEPLOY == 'true' }} + deploy-chart-source: 'github-oci' + deploy-namespace: 'bitween' + deploy-environment: 'production' + routing-mode: 'gateway-api' + gateway-hostnames: 'app.bitween.io' + gateway-paths: '/,/api,/swagger' + # The production cluster's gateway. The default, s9-dev-edge, does not exist there, and + # the deploy would create a second public-gateway in it instead of failing. + gateway-parent-namespace: 's9-prod-edge' + # The production gateway's *.bitween.io wildcard listener, as on develop. + gateway-section-name: 'https-wildcard-bitween-io' + # Every line is handed to helm --set as it stands, so no comments inside. + # AWS_*_CHECKSUM_*=when_required: production stores files in DigitalOcean Spaces too, so it + # carries develop's checksum setting (see the note there) until the fix lands in SW-CloudFiles. + helm-set-values: | + global.environment=Production + replicaCount=2 + resources.requests.cpu=25m + resources.requests.memory=704Mi + resources.limits.memory=1408Mi + environmentVariables.Bitween__BusProvidersEnabled=true + environmentVariables.AWS_REQUEST_CHECKSUM_CALCULATION=when_required + environmentVariables.AWS_RESPONSE_CHECKSUM_VALIDATION=when_required secrets: registry-username: ${{ secrets.DOCKERHUB_USERNAME }} registry-password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -147,3 +204,10 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} nuget-api-key: ${{ secrets.SWNUGETKEY }} dependabot-alerts-token: ${{ secrets.DEPENDABOT_ALERTS_TOKEN }} + + # Production cluster kubeconfig (V2, Gateway API) + kubeconfig-gateway: ${{ secrets.S9PROD_KUBECONFIG_V2 }} + + # Production's own configuration, the same settings as the develop job's above. PROD_TOKENKEY + # is its own random string (openssl rand -base64 48), never the develop key. + helm-set-secret-values: 'global.token.key=${{ secrets.PROD_TOKENKEY }},db=${{ secrets.PROD_DBCONNECTION }},global.bus.rabbitUrl=${{ secrets.PROD_RABBITURL }},global.cloudFiles.secretAccessKey=${{ secrets.PROD_SECRETACCESSKEY }},global.cloudFiles.accessKeyId=${{ secrets.PROD_ACCESSKEYID }},global.cloudFiles.serviceUrl=${{ secrets.PROD_SERVICEURL }},global.cloudFiles.bucketName=${{ secrets.PROD_BUCKETNAME }},environmentVariables.Bitween__RabbitMqManagementUrl=${{ secrets.PROD_RABBITMQMANAGEMENTURL }},environmentVariables.Bitween__RabbitMqManagementPassword=${{ secrets.PROD_RABBITMQMANAGEMENTPASSWORD }},environmentVariables.Bitween__RabbitMqManagementUsername=${{ secrets.PROD_RABBITMQMANAGEMENTUSERNAME }}' diff --git a/charts/default/templates/deployment.yaml b/charts/default/templates/deployment.yaml index 97873073..0a5e2911 100644 --- a/charts/default/templates/deployment.yaml +++ b/charts/default/templates/deployment.yaml @@ -89,6 +89,15 @@ spec: - name: Bitween__RabbitMqManagementUsername value: {{ .Values.environmentVariables.Bitween__RabbitMqManagementUsername | quote }} + # Any other setting given under environmentVariables, as the docs say to: Bitween__BusProvidersEnabled + # to run data sources, say. The three above are handled on their own. + {{- range $name, $value := .Values.environmentVariables }} + {{- if not (has $name (list "Bitween__RabbitMqManagementUrl" "Bitween__RabbitMqManagementPassword" "Bitween__RabbitMqManagementUsername")) }} + - name: {{ $name }} + value: {{ $value | toString | quote }} + {{- end }} + {{- end }} + - name: Bitween__StorageProvider value: {{ .Values.storageProvider }} diff --git a/docs/deployment.md b/docs/deployment.md index 7ea047b7..2826fc5a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -113,7 +113,8 @@ Run several replicas behind a load balancer for throughput and availability, and | Pipeline | Trigger | What it does | |---|---|---| -| `.github/workflows/bitween-api-cicd-gateway.yml` | Push to `releases/r10.0`, or manual | Runs unit and integration tests, pushes the image to Docker Hub, publishes the chart to GHCR and `charts.sf9.io`, pushes `SimplyWorks.Bitween.Sdk` to NuGet, tags the repository, and deploys the playground environment | +| `.github/workflows/bitween-api-cicd-gateway.yml` | Push to `releases/r10.0-develop`, or manual on it | Runs unit and integration tests, pushes the `simplify9/bitween-develop` image to Docker Hub, publishes the production chart (`charts/default`) as `bitween-develop` to `charts.sf9.io` and GHCR, tags the repository, and deploys to the `bitween-dev` namespace on the dev cluster (`app-dev.bitween.io`, on the gateway's `*.bitween.io` wildcard listener). Publishes no NuGet packages. | +| `.github/workflows/bitween-api-cicd-gateway.yml` | Push to `releases/r10.0`, or manual on it | Runs unit and integration tests, pushes the `simplify9/bitween` image to Docker Hub, publishes the `bitween` chart clients install to `charts.sf9.io` and GHCR (`oci://ghcr.io/simplify9/charts/bitween`), pushes `SimplyWorks.Bitween.Sdk` and `SimplyWorks.Bitween.Adapters` to NuGet, tags the repository, and deploys to the `bitween` namespace on the production cluster (`app.bitween.io`, on the gateway's `*.bitween.io` wildcard listener). The deploy runs only while the `BITWEEN_PROD_DEPLOY` repository variable is `true`. | | `.github/workflows/critical-vuln-check.yml` | Pull requests to release and develop branches | Fails while critical Dependabot alerts are open | | `.github/workflows/dependabot-auto-merge.yml` | Dependabot pull requests | Auto-merges patch updates | | `azure-pipelines.yml` | `releases/*` | The earlier Azure DevOps pipeline, still in the repository |