From f4685e6d3df011e5fe85f308ce3afd462b09b068 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Mon, 13 Jul 2026 16:42:59 +0100
Subject: [PATCH 01/13] fix(hosting): deploy ClickHouse from the official image
 instead of Bitnami

The Bitnami free image catalog is EOL and its frozen bitnamilegacy
archive tops out at ClickHouse 25.7.5, below the 25.8 floor the
platform requires since v4.5.0. The Docker Compose stack and the Helm
chart now run the official clickhouse/clickhouse-server image at 26.2,
the same version the platform is developed and tested against. The Helm
chart deploys ClickHouse with a chart-owned StatefulSet instead of the
Bitnami subchart.

Existing deployments keep their data with no manual steps: a config
override keeps the on-disk layout compatible with volumes created by
the Bitnami-based setup, Compose reuses the same named volume, and the
Helm chart automatically adopts the data PVC left behind by the old
subchart.
---
 docs/self-hosting/docker.mdx                  |   2 +
 docs/self-hosting/kubernetes.mdx              |  14 +-
 hosting/docker/.env.example                   |   2 +-
 hosting/docker/clickhouse/data-paths.xml      |  17 ++
 hosting/docker/webapp/docker-compose.yml      |  18 +-
 hosting/k8s/helm/Chart.lock                   |   7 +-
 hosting/k8s/helm/Chart.yaml                   |   4 -
 hosting/k8s/helm/templates/_helpers.tpl       |   6 +-
 hosting/k8s/helm/templates/clickhouse.yaml    | 205 ++++++++++++++++++
 .../helm/templates/tests/test-clickhouse.yaml |   2 +-
 .../k8s/helm/values-production-example.yaml   |   4 +-
 hosting/k8s/helm/values.yaml                  |  68 ++++--
 12 files changed, 313 insertions(+), 36 deletions(-)
 create mode 100644 hosting/docker/clickhouse/data-paths.xml
 create mode 100644 hosting/k8s/helm/templates/clickhouse.yaml

diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx
index 6c8de292f17..f409b6ec414 100644
--- a/docs/self-hosting/docker.mdx
+++ b/docs/self-hosting/docker.mdx
@@ -362,6 +362,8 @@ TRIGGER_IMAGE_TAG=v4.5.0
 
 We patch the latest released version line only, so keep an eye on new releases to receive security fixes. See [Security & vulnerability reporting](/self-hosting/security).
 
+You can also lock the versions of the bundled services, for example with `CLICKHOUSE_IMAGE_TAG`. If you do, or if you bring your own ClickHouse via `CLICKHOUSE_URL`, note that Trigger.dev requires ClickHouse 25.8 or newer.
+
 <Note>
   Trigger.dev 4.5.0 is the last version we officially support for running v3 (SDK v3) tasks. If
   you still have v3 tasks, pin `TRIGGER_IMAGE_TAG` to exactly `v4.5.0` or [migrate to
diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx
index 33d06f3086c..cf605d1d06e 100644
--- a/docs/self-hosting/kubernetes.mdx
+++ b/docs/self-hosting/kubernetes.mdx
@@ -279,6 +279,16 @@ redis:
 
 #### ClickHouse
 
+<Note>Trigger.dev requires ClickHouse 25.8 or newer.</Note>
+
+<Note>
+  When upgrading from a chart version that bundled ClickHouse via the Bitnami subchart, the chart
+  automatically adopts the existing data volume, so no manual migration is needed. If you render
+  manifests without cluster access (for example with GitOps tools that use `helm template`), set
+  `clickhouse.persistence.existingClaim` to the old PVC name
+  (`data-<release>-clickhouse-shard0-0`) to keep your data.
+</Note>
+
 **Direct configuration:**
 
 ```yaml
@@ -286,7 +296,7 @@ clickhouse:
   deploy: false
   external:
     host: "my-clickhouse.example.com"
-    port: 8123
+    httpPort: 8123
     username: "my-username"
     password: "my-password"
 ```
@@ -298,7 +308,7 @@ clickhouse:
   deploy: false
   external:
     host: "my-clickhouse.example.com"
-    port: 8123
+    httpPort: 8123
     username: "my-username"
     existingSecret: "clickhouse-credentials"
     # existingSecretKey: "clickhouse-password" # default (optional)
diff --git a/hosting/docker/.env.example b/hosting/docker/.env.example
index 4c7cf11bc70..b1a797f137d 100644
--- a/hosting/docker/.env.example
+++ b/hosting/docker/.env.example
@@ -136,7 +136,7 @@ OBJECT_STORE_SECRET_ACCESS_KEY=
 # POSTGRES_IMAGE_TAG=14
 # REDIS_IMAGE_TAG=7
 # ELECTRIC_IMAGE_TAG=1.0.13
-# CLICKHOUSE_IMAGE_TAG=latest
+# CLICKHOUSE_IMAGE_TAG=26.2
 # REGISTRY_IMAGE_TAG=2
 # MINIO_IMAGE_TAG=latest
 # DOCKER_PROXY_IMAGE_TAG=latest
diff --git a/hosting/docker/clickhouse/data-paths.xml b/hosting/docker/clickhouse/data-paths.xml
new file mode 100644
index 00000000000..bfa9c5ce22a
--- /dev/null
+++ b/hosting/docker/clickhouse/data-paths.xml
@@ -0,0 +1,17 @@
+<!--
+    Keeps ClickHouse's on-disk layout compatible with data volumes created by
+    the previous Bitnami-based setup, which stored everything under a data/
+    subdirectory of the volume. Fresh installs get the same layout. tmp lives
+    outside data/ because old volumes contain a dangling tmp symlink there.
+-->
+<clickhouse>
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/data/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/data/format_schemas/</format_schema_path>
+    <user_directories>
+        <local_directory>
+            <path>/var/lib/clickhouse/data/access/</path>
+        </local_directory>
+    </user_directories>
+</clickhouse>
diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml
index 8d133cbeea5..1ecdd01dc49 100644
--- a/hosting/docker/webapp/docker-compose.yml
+++ b/hosting/docker/webapp/docker-compose.yml
@@ -157,18 +157,26 @@ services:
       start_period: 10s
 
   clickhouse:
-    image: bitnamilegacy/clickhouse:${CLICKHOUSE_IMAGE_TAG:-latest}
+    image: clickhouse/clickhouse-server:${CLICKHOUSE_IMAGE_TAG:-26.2}
     restart: ${RESTART_POLICY:-unless-stopped}
     logging: *logging-config
     ports:
       - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9123:8123
       - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9090:9000
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
     environment:
-      CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default}
-      CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set CLICKHOUSE_PASSWORD in .env - run ./generate-secrets.sh}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set CLICKHOUSE_PASSWORD in .env - run ./generate-secrets.sh}
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
     volumes:
-      - clickhouse:/bitnami/clickhouse
-      - ../clickhouse/override.xml:/bitnami/clickhouse/etc/config.d/override.xml:ro
+      # The same volume works across upgrades from the previous Bitnami-based
+      # setup: data-paths.xml keeps the on-disk layout compatible.
+      - clickhouse:/var/lib/clickhouse
+      - ../clickhouse/data-paths.xml:/etc/clickhouse-server/config.d/data-paths.xml:ro
+      - ../clickhouse/override.xml:/etc/clickhouse-server/config.d/override.xml:ro
     networks:
       - webapp
     healthcheck:
diff --git a/hosting/k8s/helm/Chart.lock b/hosting/k8s/helm/Chart.lock
index 5a7882cfa0e..338d0db4805 100644
--- a/hosting/k8s/helm/Chart.lock
+++ b/hosting/k8s/helm/Chart.lock
@@ -5,11 +5,8 @@ dependencies:
 - name: redis
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 21.2.6
-- name: clickhouse
-  repository: oci://registry-1.docker.io/bitnamicharts
-  version: 9.4.4
 - name: minio
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 17.0.9
-digest: sha256:e1b572ab8eca0cc376311398c27b1734d8a598095fccc81dd9c32b2c8b9c1149
-generated: "2026-05-05T10:31:58.493590751+01:00"
+digest: sha256:a735954c8b78fcf5b30689bdcdfed66b9be57135368ea696aff2b52ecd731474
+generated: "2026-07-13T16:36:15.800113+01:00"
diff --git a/hosting/k8s/helm/Chart.yaml b/hosting/k8s/helm/Chart.yaml
index 69538c1a239..69971704525 100644
--- a/hosting/k8s/helm/Chart.yaml
+++ b/hosting/k8s/helm/Chart.yaml
@@ -26,10 +26,6 @@ dependencies:
     version: "21.2.6"
     repository: "oci://registry-1.docker.io/bitnamicharts"
     condition: redis.deploy
-  - name: clickhouse
-    version: "9.4.4"
-    repository: "oci://registry-1.docker.io/bitnamicharts"
-    condition: clickhouse.deploy
   - name: minio
     version: "17.0.9"
     repository: "oci://registry-1.docker.io/bitnamicharts"
diff --git a/hosting/k8s/helm/templates/_helpers.tpl b/hosting/k8s/helm/templates/_helpers.tpl
index 0ecc17b9893..c1a7c5a3261 100644
--- a/hosting/k8s/helm/templates/_helpers.tpl
+++ b/hosting/k8s/helm/templates/_helpers.tpl
@@ -415,7 +415,7 @@ ClickHouse hostname
 {{- if .Values.clickhouse.host }}
 {{- .Values.clickhouse.host }}
 {{- else if .Values.clickhouse.deploy }}
-{{- printf "%s-clickhouse" .Release.Name }}
+{{- printf "%s-clickhouse" (include "trigger-v4.fullname" .) }}
 {{- end }}
 {{- end }}
 
@@ -439,7 +439,7 @@ hex-encoded password or percent-encode before storing in the Secret.
 {{- if .Values.clickhouse.deploy -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}}
 {{- $secure := ternary "true" "false" .Values.clickhouse.secure -}}
-{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123?secure={{ $secure }}
+{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}?secure={{ $secure }}
 {{- else if .Values.clickhouse.external.host -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}}
 {{- $secure := ternary "true" "false" .Values.clickhouse.external.secure -}}
@@ -460,7 +460,7 @@ applies to the replication URL.
 {{- define "trigger-v4.clickhouse.replication.url" -}}
 {{- if .Values.clickhouse.deploy -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}}
-{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123
+{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}
 {{- else if .Values.clickhouse.external.host -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}}
 {{- if .Values.clickhouse.external.existingSecret -}}
diff --git a/hosting/k8s/helm/templates/clickhouse.yaml b/hosting/k8s/helm/templates/clickhouse.yaml
new file mode 100644
index 00000000000..006b438e0b2
--- /dev/null
+++ b/hosting/k8s/helm/templates/clickhouse.yaml
@@ -0,0 +1,205 @@
+{{- if .Values.clickhouse.deploy }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "trigger-v4.fullname" . }}-clickhouse-config
+  labels:
+    {{- $component := "clickhouse" }}
+    {{- include "trigger-v4.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }}
+data:
+  {{- if not (hasKey .Values.clickhouse.configdFiles "data-paths.xml") }}
+  {{- /* Keeps the on-disk layout compatible with data volumes created by the
+         Bitnami subchart this chart used previously, which stored everything
+         under a data/ subdirectory of the volume. Fresh installs get the same
+         layout. tmp lives outside data/ because old volumes contain a
+         dangling tmp symlink there. */}}
+  data-paths.xml: |
+    <clickhouse>
+      <path>/var/lib/clickhouse/data/</path>
+      <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+      <user_files_path>/var/lib/clickhouse/data/user_files/</user_files_path>
+      <format_schema_path>/var/lib/clickhouse/data/format_schemas/</format_schema_path>
+      <user_directories>
+        <local_directory>
+          <path>/var/lib/clickhouse/data/access/</path>
+        </local_directory>
+      </user_directories>
+    </clickhouse>
+  {{- end }}
+  {{- range $filename, $content := .Values.clickhouse.configdFiles }}
+  {{ $filename }}: |
+    {{- $content | nindent 4 }}
+  {{- end }}
+---
+{{- /* Reuse an existing data PVC instead of creating one via
+       volumeClaimTemplates. Set explicitly through persistence.existingClaim,
+       or detected automatically: upgrades from chart versions that bundled
+       the Bitnami subchart leave their PVC behind under the old name, and
+       adopting it preserves all ClickHouse data with no manual migration.
+       (lookup returns nothing during template/dry-run rendering; set
+       persistence.existingClaim explicitly when pre-rendering manifests,
+       e.g. with GitOps tools.) */}}
+{{- $existingClaim := .Values.clickhouse.persistence.existingClaim }}
+{{- if and (not $existingClaim) .Values.clickhouse.persistence.enabled }}
+{{- $legacyName := printf "data-%s-clickhouse-shard0-0" .Release.Name }}
+{{- if lookup "v1" "PersistentVolumeClaim" .Release.Namespace $legacyName }}
+{{- $existingClaim = $legacyName }}
+{{- end }}
+{{- end }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "trigger-v4.fullname" . }}-clickhouse
+  labels:
+    {{- $component := "clickhouse" }}
+    {{- include "trigger-v4.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }}
+spec:
+  replicas: 1
+  serviceName: {{ include "trigger-v4.fullname" . }}-clickhouse
+  selector:
+    matchLabels:
+      {{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ .Values.clickhouse.configdFiles | toYaml | sha256sum }}
+        {{- with .Values.clickhouse.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 8 }}
+    spec:
+      {{- with .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.clickhouse.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: clickhouse
+          {{- with .Values.clickhouse.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          image: "{{ .Values.global.imageRegistry | default .Values.clickhouse.image.registry }}/{{ .Values.clickhouse.image.repository }}:{{ .Values.clickhouse.image.tag }}{{ with .Values.clickhouse.image.digest }}@{{ . }}{{ end }}"
+          imagePullPolicy: {{ .Values.clickhouse.image.pullPolicy }}
+          env:
+            - name: CLICKHOUSE_USER
+              value: {{ .Values.clickhouse.auth.username | quote }}
+            {{- /* Same chart-managed datastore secret the webapp reads for its
+                   connection URL, so the server credential and the app's URL
+                   always match. */}}
+            - name: CLICKHOUSE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.clickhouse.auth.existingSecret | default (include "trigger-v4.datastore.secretName" .) }}
+                  key: {{ .Values.clickhouse.auth.existingSecretKey | default "clickhouse-admin-password" }}
+            - name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
+              value: "1"
+          ports:
+            - name: http
+              containerPort: 8123
+              protocol: TCP
+            - name: native
+              containerPort: 9000
+              protocol: TCP
+          {{- if .Values.clickhouse.livenessProbe.enabled }}
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: {{ .Values.clickhouse.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.clickhouse.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.clickhouse.livenessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.clickhouse.livenessProbe.failureThreshold }}
+            successThreshold: {{ .Values.clickhouse.livenessProbe.successThreshold }}
+          {{- end }}
+          {{- if .Values.clickhouse.readinessProbe.enabled }}
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: {{ .Values.clickhouse.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.clickhouse.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.clickhouse.readinessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.clickhouse.readinessProbe.failureThreshold }}
+            successThreshold: {{ .Values.clickhouse.readinessProbe.successThreshold }}
+          {{- end }}
+          resources:
+            {{- toYaml .Values.clickhouse.resources | nindent 12 }}
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/clickhouse
+            - name: logs
+              mountPath: /var/log/clickhouse-server
+            {{- /* Mount each override file individually: shadowing the whole
+                   config.d directory would remove the image's built-in
+                   docker_related_config.xml, which makes the server listen on
+                   0.0.0.0 instead of localhost only. */}}
+            {{- if not (hasKey .Values.clickhouse.configdFiles "data-paths.xml") }}
+            - name: config
+              mountPath: /etc/clickhouse-server/config.d/data-paths.xml
+              subPath: data-paths.xml
+            {{- end }}
+            {{- range $filename, $_ := .Values.clickhouse.configdFiles }}
+            - name: config
+              mountPath: /etc/clickhouse-server/config.d/{{ $filename }}
+              subPath: {{ $filename }}
+            {{- end }}
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "trigger-v4.fullname" . }}-clickhouse-config
+        - name: logs
+          emptyDir: {}
+        {{- if not .Values.clickhouse.persistence.enabled }}
+        - name: data
+          emptyDir: {}
+        {{- else if $existingClaim }}
+        - name: data
+          persistentVolumeClaim:
+            claimName: {{ $existingClaim }}
+        {{- end }}
+  {{- if and .Values.clickhouse.persistence.enabled (not $existingClaim) }}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+        {{- if .Values.clickhouse.persistence.retain }}
+        annotations:
+          helm.sh/resource-policy: keep
+        {{- end }}
+      spec:
+        accessModes:
+          - {{ .Values.clickhouse.persistence.accessMode }}
+        resources:
+          requests:
+            storage: {{ .Values.clickhouse.persistence.size }}
+        {{- $storageClass := .Values.clickhouse.persistence.storageClass | default .Values.global.storageClass }}
+        {{- if $storageClass }}
+        storageClassName: {{ $storageClass }}
+        {{- end }}
+  {{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "trigger-v4.fullname" . }}-clickhouse
+  labels:
+    {{- $component := "clickhouse" }}
+    {{- include "trigger-v4.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }}
+spec:
+  type: {{ .Values.clickhouse.service.type }}
+  ports:
+    - port: {{ .Values.clickhouse.service.ports.http }}
+      targetPort: http
+      protocol: TCP
+      name: http
+    - port: {{ .Values.clickhouse.service.ports.native }}
+      targetPort: native
+      protocol: TCP
+      name: native
+  selector:
+    {{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }}
+{{- end }}
diff --git a/hosting/k8s/helm/templates/tests/test-clickhouse.yaml b/hosting/k8s/helm/templates/tests/test-clickhouse.yaml
index 9bde62c2ada..3e420d05c1b 100644
--- a/hosting/k8s/helm/templates/tests/test-clickhouse.yaml
+++ b/hosting/k8s/helm/templates/tests/test-clickhouse.yaml
@@ -16,6 +16,6 @@ spec:
       args:
         - |
           echo "Testing ClickHouse HTTP interface..."
-          curl -f --user "{{ .Values.clickhouse.auth.adminUser }}:{{ .Values.clickhouse.auth.adminPassword }}" "http://{{ include "trigger-v4.fullname" . }}-clickhouse:{{ .Values.clickhouse.service.ports.http }}/ping"
+          curl -f --user "{{ .Values.clickhouse.auth.username }}:{{ .Values.clickhouse.auth.password }}" "http://{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}/ping"
           echo "ClickHouse test completed successfully"
 {{- end }}
\ No newline at end of file
diff --git a/hosting/k8s/helm/values-production-example.yaml b/hosting/k8s/helm/values-production-example.yaml
index ee99de58a9b..3f99547fcf0 100644
--- a/hosting/k8s/helm/values-production-example.yaml
+++ b/hosting/k8s/helm/values-production-example.yaml
@@ -77,12 +77,12 @@ redis:
         memory: 512Mi
 
 # Production ClickHouse
+# The bundled ClickHouse serves plain HTTP inside the cluster. For TLS,
+# use an external ClickHouse (deploy: false) with secure: true (see below).
 clickhouse:
   auth:
     # Required — no built-in default. The webapp connection string uses clickhouse.auth.password.
     password: "your-strong-clickhouse-password"
-  # Set to true to enable TLS/secure connections in production
-  secure: true
   persistence:
     enabled: true
     size: 100Gi
diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml
index e76a921279c..39240f0192a 100644
--- a/hosting/k8s/helm/values.yaml
+++ b/hosting/k8s/helm/values.yaml
@@ -628,19 +628,24 @@ s2:
     existingSecretAccessTokenKey: "access-token"
 
 # ClickHouse configuration
-# Subchart: https://github.com/bitnami/charts/tree/main/bitnami/clickhouse
+# Deploys a single-node ClickHouse using the official image:
+# https://hub.docker.com/r/clickhouse/clickhouse-server
+# For clustered/replicated setups, use an external ClickHouse (deploy: false).
 clickhouse:
   deploy: true
 
   image:
-    # Use bitnami legacy repo
-    repository: bitnamilegacy/clickhouse
-    # image: docker.io/bitnamilegacy/clickhouse:25.7.5-debian-12-r0
+    registry: docker.io
+    repository: clickhouse/clickhouse-server
+    # Trigger.dev requires ClickHouse >= 25.8
+    tag: "26.2"
+    # Pinning by digest is strongly recommended for reproducible deployments
+    digest: ""
+    pullPolicy: IfNotPresent
 
   # TLS/Secure connection configuration
   secure: false # Set to true to use HTTPS and secure connections
 
-  # Bitnami ClickHouse chart configuration (when deploy: true)
   auth:
     username: "default"
     password: "" # Leave empty to auto-generate into the datastore secret, or set to pin.
@@ -648,24 +653,61 @@ clickhouse:
     existingSecret: "trigger-datastore"
     existingSecretKey: "clickhouse-admin-password"
 
-  # Single-node configuration (disable clustering for dev/test)
-  keeper:
-    enabled: false
+  podAnnotations: {}
 
-  shards: 1
-  replicaCount: 1
+  # The official image runs ClickHouse as uid 101. fsGroup makes the persistent
+  # volume writable by that user without running the container as root, and
+  # OnRootMismatch relabels volumes carried over from older chart versions
+  # (different uid) on first mount without rechecking every file on later mounts.
+  podSecurityContext:
+    fsGroup: 101
+    fsGroupChangePolicy: OnRootMismatch
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 101
+    runAsGroup: 101
+
+  service:
+    type: ClusterIP
+    ports:
+      http: 8123
+      native: 9000
 
   persistence:
     enabled: true
     size: 10Gi
+    accessMode: ReadWriteOnce
+    storageClass: ""
+    retain: false
+    # Name of an existing PVC to use for ClickHouse data instead of creating
+    # one. Normally left empty: upgrades from chart versions that bundled the
+    # Bitnami ClickHouse subchart adopt the old data PVC automatically. Set
+    # this explicitly when rendering manifests without cluster access (e.g.
+    # GitOps tools that use `helm template`), where auto-detection can't run:
+    # the old PVC is named data-<release>-clickhouse-shard0-0.
+    existingClaim: ""
 
   ## ClickHouse resource requests and limits
   ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
-  ## @param resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).
-  ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15
-  resourcesPreset: "xlarge"
+  ## ClickHouse can be very resource intensive. Setting explicit limits and
+  ## requests is strongly recommended for production (see values-production-example.yaml).
   resources: {}
 
+  livenessProbe:
+    enabled: true
+    initialDelaySeconds: 10
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 5
+    successThreshold: 1
+  readinessProbe:
+    enabled: true
+    initialDelaySeconds: 5
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 5
+    successThreshold: 1
+
   # External ClickHouse connection (when deploy: false)
   external:
     host: ""

From 68aed697890bcf0876ff241796ede3241d873a1a Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Tue, 14 Jul 2026 11:05:44 +0100
Subject: [PATCH 02/13] fix(hosting): store the ClickHouse password in a Secret
 and URL-encode inline credentials

The chart-deployed ClickHouse now reads CLICKHOUSE_PASSWORD from a
chart-owned Secret instead of a plaintext env value in the pod spec, and
the CLICKHOUSE_URL helpers percent-encode inline usernames and passwords
so special characters no longer produce an unparseable URL.
---
 hosting/k8s/helm/templates/_helpers.tpl    | 28 +++++++++++++---------
 hosting/k8s/helm/templates/clickhouse.yaml |  1 +
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/hosting/k8s/helm/templates/_helpers.tpl b/hosting/k8s/helm/templates/_helpers.tpl
index c1a7c5a3261..efeec79ceb8 100644
--- a/hosting/k8s/helm/templates/_helpers.tpl
+++ b/hosting/k8s/helm/templates/_helpers.tpl
@@ -422,9 +422,9 @@ ClickHouse hostname
 {{/*
 ClickHouse URL for application (with secure parameter)
 
-Note on the external+existingSecret branch: the password is expanded via
-Kubernetes' `$(VAR)` syntax, not shell `${VAR}`. Kubelet substitutes
-`$(CLICKHOUSE_PASSWORD)` at container-creation time from the
+Note on the deploy and external+existingSecret branches: the password is
+expanded via Kubernetes' `$(VAR)` syntax, not shell `${VAR}`. Kubelet
+substitutes `$(CLICKHOUSE_PASSWORD)` at container-creation time from the
 CLICKHOUSE_PASSWORD env var declared just before CLICKHOUSE_URL in
 webapp.yaml. Shell-style `${...}` does not work here because
 `docker/scripts/entrypoint.sh` assigns CLICKHOUSE_URL to GOOSE_DBSTRING
@@ -432,21 +432,27 @@ with a single-pass expansion (`export GOOSE_DBSTRING="$CLICKHOUSE_URL"`),
 so any inner `${...}` reaches goose verbatim and fails URL parsing.
 
 CLICKHOUSE_PASSWORD must contain only URL-userinfo-safe characters — the
-value is substituted verbatim, so `@ : / ? # [ ] %` break the URL. Use a
-hex-encoded password or percent-encode before storing in the Secret.
+value is substituted verbatim, so `@ : / ? # [ ] %` break the URL. The
+chart-generated datastore password is hex, which is safe; a pinned
+auth.password or external Secret value must be URL-safe too.
+
+Inline credentials (usernames and the external plain password) are
+percent-encoded via urlquery, so special characters are safe there —
+except spaces, which urlquery encodes as `+` and userinfo decoding keeps
+literal.
 */}}
 {{- define "trigger-v4.clickhouse.url" -}}
 {{- if .Values.clickhouse.deploy -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}}
 {{- $secure := ternary "true" "false" .Values.clickhouse.secure -}}
-{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}?secure={{ $secure }}
+{{ $protocol }}://{{ .Values.clickhouse.auth.username | urlquery }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}?secure={{ $secure }}
 {{- else if .Values.clickhouse.external.host -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}}
 {{- $secure := ternary "true" "false" .Values.clickhouse.external.secure -}}
 {{- if .Values.clickhouse.external.existingSecret -}}
-{{ $protocol }}://{{ .Values.clickhouse.external.username }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }}
+{{ $protocol }}://{{ .Values.clickhouse.external.username | urlquery }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }}
 {{- else -}}
-{{ $protocol }}://{{ .Values.clickhouse.external.username }}:{{ .Values.clickhouse.external.password }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }}
+{{ $protocol }}://{{ .Values.clickhouse.external.username | urlquery }}:{{ .Values.clickhouse.external.password | urlquery }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }}
 {{- end -}}
 {{- end -}}
 {{- end }}
@@ -460,13 +466,13 @@ applies to the replication URL.
 {{- define "trigger-v4.clickhouse.replication.url" -}}
 {{- if .Values.clickhouse.deploy -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}}
-{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}
+{{ $protocol }}://{{ .Values.clickhouse.auth.username | urlquery }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}
 {{- else if .Values.clickhouse.external.host -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}}
 {{- if .Values.clickhouse.external.existingSecret -}}
-{{ $protocol }}://{{ .Values.clickhouse.external.username }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}
+{{ $protocol }}://{{ .Values.clickhouse.external.username | urlquery }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}
 {{- else -}}
-{{ $protocol }}://{{ .Values.clickhouse.external.username }}:{{ .Values.clickhouse.external.password }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}
+{{ $protocol }}://{{ .Values.clickhouse.external.username | urlquery }}:{{ .Values.clickhouse.external.password | urlquery }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}
 {{- end -}}
 {{- end -}}
 {{- end }}
diff --git a/hosting/k8s/helm/templates/clickhouse.yaml b/hosting/k8s/helm/templates/clickhouse.yaml
index 006b438e0b2..70e9a793ae7 100644
--- a/hosting/k8s/helm/templates/clickhouse.yaml
+++ b/hosting/k8s/helm/templates/clickhouse.yaml
@@ -63,6 +63,7 @@ spec:
     metadata:
       annotations:
         checksum/config: {{ .Values.clickhouse.configdFiles | toYaml | sha256sum }}
+        checksum/secret: {{ .Values.clickhouse.auth.password | sha256sum }}
         {{- with .Values.clickhouse.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

From ad6e5a151e118600651d9b39f0aa90d50191c37c Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Tue, 14 Jul 2026 13:53:15 +0100
Subject: [PATCH 03/13] fix(hosting): percent-encode spaces correctly in
 ClickHouse URL credentials

urlquery encodes spaces as plus signs, which URL userinfo decoding keeps
literal. A shared urlencode helper rewrites them to %20 (a real plus
already encodes as %2B), so passwords containing spaces now work.
---
 hosting/k8s/helm/templates/_helpers.tpl | 26 ++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/hosting/k8s/helm/templates/_helpers.tpl b/hosting/k8s/helm/templates/_helpers.tpl
index efeec79ceb8..226c09b020f 100644
--- a/hosting/k8s/helm/templates/_helpers.tpl
+++ b/hosting/k8s/helm/templates/_helpers.tpl
@@ -408,6 +408,16 @@ http://{{ include "trigger-v4.fullname" . }}-s2:{{ .Values.s2.service.port }}/v1
 {{- end -}}
 {{- end }}
 
+{{/*
+Percent-encode a string for the userinfo part of a URL. urlquery encodes
+spaces as `+` (query semantics), which userinfo decoding keeps literal; a
+real `+` becomes `%2B`, so any `+` left in the output is a space and can be
+rewritten to `%20`.
+*/}}
+{{- define "trigger-v4.urlencode" -}}
+{{- . | urlquery | replace "+" "%20" -}}
+{{- end }}
+
 {{/*
 ClickHouse hostname
 */}}
@@ -437,22 +447,20 @@ chart-generated datastore password is hex, which is safe; a pinned
 auth.password or external Secret value must be URL-safe too.
 
 Inline credentials (usernames and the external plain password) are
-percent-encoded via urlquery, so special characters are safe there —
-except spaces, which urlquery encodes as `+` and userinfo decoding keeps
-literal.
+percent-encoded, so any special characters are safe there.
 */}}
 {{- define "trigger-v4.clickhouse.url" -}}
 {{- if .Values.clickhouse.deploy -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}}
 {{- $secure := ternary "true" "false" .Values.clickhouse.secure -}}
-{{ $protocol }}://{{ .Values.clickhouse.auth.username | urlquery }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}?secure={{ $secure }}
+{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}?secure={{ $secure }}
 {{- else if .Values.clickhouse.external.host -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}}
 {{- $secure := ternary "true" "false" .Values.clickhouse.external.secure -}}
 {{- if .Values.clickhouse.external.existingSecret -}}
-{{ $protocol }}://{{ .Values.clickhouse.external.username | urlquery }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }}
+{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.external.username }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }}
 {{- else -}}
-{{ $protocol }}://{{ .Values.clickhouse.external.username | urlquery }}:{{ .Values.clickhouse.external.password | urlquery }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }}
+{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.external.username }}:{{ include "trigger-v4.urlencode" .Values.clickhouse.external.password }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }}
 {{- end -}}
 {{- end -}}
 {{- end }}
@@ -466,13 +474,13 @@ applies to the replication URL.
 {{- define "trigger-v4.clickhouse.replication.url" -}}
 {{- if .Values.clickhouse.deploy -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}}
-{{ $protocol }}://{{ .Values.clickhouse.auth.username | urlquery }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}
+{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}
 {{- else if .Values.clickhouse.external.host -}}
 {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}}
 {{- if .Values.clickhouse.external.existingSecret -}}
-{{ $protocol }}://{{ .Values.clickhouse.external.username | urlquery }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}
+{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.external.username }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}
 {{- else -}}
-{{ $protocol }}://{{ .Values.clickhouse.external.username | urlquery }}:{{ .Values.clickhouse.external.password | urlquery }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}
+{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.external.username }}:{{ include "trigger-v4.urlencode" .Values.clickhouse.external.password }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}
 {{- end -}}
 {{- end -}}
 {{- end }}

From f9d6f213f86e9e1092f07b0c2973923e31d74886 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Tue, 14 Jul 2026 16:14:44 +0100
Subject: [PATCH 04/13] fix(hosting): include the default ClickHouse data-paths
 config in the restart checksum

The data-paths.xml default now lives in a shared helper used by both the
ConfigMap and the pod checksum annotation, so changing it in a future
chart version restarts the pod (subPath mounts do not update in place).
---
 hosting/k8s/helm/templates/_helpers.tpl    | 22 ++++++++++++++++++++++
 hosting/k8s/helm/templates/clickhouse.yaml | 19 ++-----------------
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/hosting/k8s/helm/templates/_helpers.tpl b/hosting/k8s/helm/templates/_helpers.tpl
index 226c09b020f..ebd3bf7f021 100644
--- a/hosting/k8s/helm/templates/_helpers.tpl
+++ b/hosting/k8s/helm/templates/_helpers.tpl
@@ -418,6 +418,28 @@ rewritten to `%20`.
 {{- . | urlquery | replace "+" "%20" -}}
 {{- end }}
 
+{{/*
+ClickHouse data-paths config. Keeps the on-disk layout compatible with data
+volumes created by the Bitnami subchart this chart used previously, which
+stored everything under a data/ subdirectory of the volume. Fresh installs
+get the same layout. tmp lives outside data/ because old volumes contain a
+dangling tmp symlink there. Users can override by defining their own
+data-paths.xml in clickhouse.configdFiles.
+*/}}
+{{- define "trigger-v4.clickhouse.dataPathsConfig" -}}
+<clickhouse>
+  <path>/var/lib/clickhouse/data/</path>
+  <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+  <user_files_path>/var/lib/clickhouse/data/user_files/</user_files_path>
+  <format_schema_path>/var/lib/clickhouse/data/format_schemas/</format_schema_path>
+  <user_directories>
+    <local_directory>
+      <path>/var/lib/clickhouse/data/access/</path>
+    </local_directory>
+  </user_directories>
+</clickhouse>
+{{- end }}
+
 {{/*
 ClickHouse hostname
 */}}
diff --git a/hosting/k8s/helm/templates/clickhouse.yaml b/hosting/k8s/helm/templates/clickhouse.yaml
index 70e9a793ae7..8c1de94c443 100644
--- a/hosting/k8s/helm/templates/clickhouse.yaml
+++ b/hosting/k8s/helm/templates/clickhouse.yaml
@@ -8,23 +8,8 @@ metadata:
     {{- include "trigger-v4.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }}
 data:
   {{- if not (hasKey .Values.clickhouse.configdFiles "data-paths.xml") }}
-  {{- /* Keeps the on-disk layout compatible with data volumes created by the
-         Bitnami subchart this chart used previously, which stored everything
-         under a data/ subdirectory of the volume. Fresh installs get the same
-         layout. tmp lives outside data/ because old volumes contain a
-         dangling tmp symlink there. */}}
   data-paths.xml: |
-    <clickhouse>
-      <path>/var/lib/clickhouse/data/</path>
-      <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
-      <user_files_path>/var/lib/clickhouse/data/user_files/</user_files_path>
-      <format_schema_path>/var/lib/clickhouse/data/format_schemas/</format_schema_path>
-      <user_directories>
-        <local_directory>
-          <path>/var/lib/clickhouse/data/access/</path>
-        </local_directory>
-      </user_directories>
-    </clickhouse>
+    {{- include "trigger-v4.clickhouse.dataPathsConfig" . | nindent 4 }}
   {{- end }}
   {{- range $filename, $content := .Values.clickhouse.configdFiles }}
   {{ $filename }}: |
@@ -62,7 +47,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: {{ .Values.clickhouse.configdFiles | toYaml | sha256sum }}
+        checksum/config: {{ printf "%s\n%s" (include "trigger-v4.clickhouse.dataPathsConfig" .) (.Values.clickhouse.configdFiles | toYaml) | sha256sum }}
         checksum/secret: {{ .Values.clickhouse.auth.password | sha256sum }}
         {{- with .Values.clickhouse.podAnnotations }}
         {{- toYaml . | nindent 8 }}

From a52bdcdac9045f4669779d0a9cf6d782759ce60e Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Jul 2026 14:40:06 +0100
Subject: [PATCH 05/13] fix(hosting): close upgrade gaps found in adversarial
 review of the ClickHouse chart

Keeps existing self-hosted deployments working without manual steps:

- Default resource requests/limits match the preset the Bitnami subchart
  applied, so the ClickHouse pod doesn't silently become BestEffort.
- nodeSelector, tolerations, and affinity pass through to the StatefulSet
  for deployments that schedule ClickHouse onto dedicated nodes.
- Optional volumePermissions init container fixes data-volume ownership
  on storage without fsGroup support (NFS, hostPath).
- The helm test reads the password from the datastore secret instead of
  the usually-empty values key, and the pinned-password restart checksum
  is omitted when the password is auto-generated.
- Docs cover updating a pinned CLICKHOUSE_IMAGE_TAG to an official tag
  and the GitOps existingClaim step in more detail.
---
 docs/self-hosting/docker.mdx                  |  7 +++++
 docs/self-hosting/kubernetes.mdx              |  9 +++++-
 hosting/k8s/helm/templates/clickhouse.yaml    | 30 +++++++++++++++++++
 .../helm/templates/tests/test-clickhouse.yaml | 12 ++++++--
 hosting/k8s/helm/values.yaml                  | 29 ++++++++++++++++--
 5 files changed, 81 insertions(+), 6 deletions(-)

diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx
index f409b6ec414..ed8aaa1bada 100644
--- a/docs/self-hosting/docker.mdx
+++ b/docs/self-hosting/docker.mdx
@@ -364,6 +364,13 @@ We patch the latest released version line only, so keep an eye on new releases t
 
 You can also lock the versions of the bundled services, for example with `CLICKHOUSE_IMAGE_TAG`. If you do, or if you bring your own ClickHouse via `CLICKHOUSE_URL`, note that Trigger.dev requires ClickHouse 25.8 or newer.
 
+<Note>
+  The bundled ClickHouse now uses the official `clickhouse/clickhouse-server` image. Your existing
+  data volume carries over automatically. If you previously pinned `CLICKHOUSE_IMAGE_TAG` to a
+  Bitnami tag (for example `25.7.5-debian-12-r0`), update it to an official image tag such as
+  `26.2` — Bitnami tags don't exist in the official repository.
+</Note>
+
 <Note>
   Trigger.dev 4.5.0 is the last version we officially support for running v3 (SDK v3) tasks. If
   you still have v3 tasks, pin `TRIGGER_IMAGE_TAG` to exactly `v4.5.0` or [migrate to
diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx
index cf605d1d06e..6b7dbf3e73d 100644
--- a/docs/self-hosting/kubernetes.mdx
+++ b/docs/self-hosting/kubernetes.mdx
@@ -286,7 +286,14 @@ redis:
   automatically adopts the existing data volume, so no manual migration is needed. If you render
   manifests without cluster access (for example with GitOps tools that use `helm template`), set
   `clickhouse.persistence.existingClaim` to the old PVC name
-  (`data-<release>-clickhouse-shard0-0`) to keep your data.
+  (`data-<release>-clickhouse-shard0-0`) to keep your data — auto-detection can't run there, and
+  skipping this starts ClickHouse on a fresh empty volume.
+</Note>
+
+<Note>
+  On storage that doesn't support `fsGroup` ownership changes (for example NFS or hostPath), set
+  `clickhouse.volumePermissions.enabled: true` so a one-time init container fixes the data
+  volume's ownership for the non-root ClickHouse server.
 </Note>
 
 **Direct configuration:**
diff --git a/hosting/k8s/helm/templates/clickhouse.yaml b/hosting/k8s/helm/templates/clickhouse.yaml
index 8c1de94c443..edc7e62a760 100644
--- a/hosting/k8s/helm/templates/clickhouse.yaml
+++ b/hosting/k8s/helm/templates/clickhouse.yaml
@@ -48,7 +48,12 @@ spec:
     metadata:
       annotations:
         checksum/config: {{ printf "%s\n%s" (include "trigger-v4.clickhouse.dataPathsConfig" .) (.Values.clickhouse.configdFiles | toYaml) | sha256sum }}
+        {{- /* Restart on pinned-password changes; when the password is
+               auto-generated it lives in the retained datastore secret and
+               never rotates on upgrade, so there is nothing to hash. */}}
+        {{- if .Values.clickhouse.auth.password }}
         checksum/secret: {{ .Values.clickhouse.auth.password | sha256sum }}
+        {{- end }}
         {{- with .Values.clickhouse.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -63,6 +68,19 @@ spec:
       securityContext:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if .Values.clickhouse.volumePermissions.enabled }}
+      initContainers:
+        - name: volume-permissions
+          image: "{{ .Values.global.imageRegistry | default .Values.clickhouse.volumePermissions.image.registry }}/{{ .Values.clickhouse.volumePermissions.image.repository }}:{{ .Values.clickhouse.volumePermissions.image.tag }}"
+          imagePullPolicy: {{ .Values.clickhouse.volumePermissions.image.pullPolicy }}
+          command: ["sh", "-c", "chown -R 101:101 /var/lib/clickhouse"]
+          securityContext:
+            runAsUser: 0
+            runAsNonRoot: false
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/clickhouse
+      {{- end }}
       containers:
         - name: clickhouse
           {{- with .Values.clickhouse.securityContext }}
@@ -134,6 +152,18 @@ spec:
               mountPath: /etc/clickhouse-server/config.d/{{ $filename }}
               subPath: {{ $filename }}
             {{- end }}
+      {{- with .Values.clickhouse.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.clickhouse.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.clickhouse.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       volumes:
         - name: config
           configMap:
diff --git a/hosting/k8s/helm/templates/tests/test-clickhouse.yaml b/hosting/k8s/helm/templates/tests/test-clickhouse.yaml
index 3e420d05c1b..bb71e381b71 100644
--- a/hosting/k8s/helm/templates/tests/test-clickhouse.yaml
+++ b/hosting/k8s/helm/templates/tests/test-clickhouse.yaml
@@ -12,10 +12,18 @@ spec:
   containers:
     - name: test-clickhouse
       image: curlimages/curl:8.14.1
+      env:
+        - name: CLICKHOUSE_USER
+          value: {{ .Values.clickhouse.auth.username | quote }}
+        - name: CLICKHOUSE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.clickhouse.auth.existingSecret | default (include "trigger-v4.datastore.secretName" .) }}
+              key: {{ .Values.clickhouse.auth.existingSecretKey | default "clickhouse-admin-password" }}
       command: ['sh', '-c']
       args:
         - |
           echo "Testing ClickHouse HTTP interface..."
-          curl -f --user "{{ .Values.clickhouse.auth.username }}:{{ .Values.clickhouse.auth.password }}" "http://{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}/ping"
+          curl -f --user "$CLICKHOUSE_USER:$CLICKHOUSE_PASSWORD" "http://{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}/ping"
           echo "ClickHouse test completed successfully"
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml
index 39240f0192a..239c9968683 100644
--- a/hosting/k8s/helm/values.yaml
+++ b/hosting/k8s/helm/values.yaml
@@ -667,6 +667,22 @@ clickhouse:
     runAsUser: 101
     runAsGroup: 101
 
+  # One-time root init container that chowns the data volume to the ClickHouse
+  # uid. Only needed on storage that doesn't support fsGroup ownership changes
+  # (e.g. NFS, hostPath); on such storage a data volume carried over from the
+  # Bitnami-based chart is otherwise unreadable by the non-root server.
+  volumePermissions:
+    enabled: false
+    image:
+      registry: docker.io
+      repository: busybox
+      tag: "1.35"
+      pullPolicy: IfNotPresent
+
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+
   service:
     type: ClusterIP
     ports:
@@ -689,9 +705,16 @@ clickhouse:
 
   ## ClickHouse resource requests and limits
   ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
-  ## ClickHouse can be very resource intensive. Setting explicit limits and
-  ## requests is strongly recommended for production (see values-production-example.yaml).
-  resources: {}
+  ## ClickHouse can be very resource intensive. The defaults below match the
+  ## resource preset the chart previously applied; size them to your workload
+  ## for production (see values-production-example.yaml).
+  resources:
+    requests:
+      cpu: 1000m
+      memory: 3Gi
+    limits:
+      cpu: 3000m
+      memory: 6Gi
 
   livenessProbe:
     enabled: true

From c798dc1ee98da11dc8d10cf1e2897c2ec3ab8f1b Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Jul 2026 17:19:44 +0100
Subject: [PATCH 06/13] docs(self-hosting): cover ClickHouse upgrade edge cases

Adds the recovery path for GitOps renders that missed existingClaim,
the pinned Bitnami image note for the Helm chart, the single-node
clustering note, and the one-way rollback caveat for Docker Compose.
---
 docs/self-hosting/docker.mdx     |  4 +++-
 docs/self-hosting/kubernetes.mdx | 11 ++++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx
index ed8aaa1bada..065b4ad55da 100644
--- a/docs/self-hosting/docker.mdx
+++ b/docs/self-hosting/docker.mdx
@@ -368,7 +368,9 @@ You can also lock the versions of the bundled services, for example with `CLICKH
   The bundled ClickHouse now uses the official `clickhouse/clickhouse-server` image. Your existing
   data volume carries over automatically. If you previously pinned `CLICKHOUSE_IMAGE_TAG` to a
   Bitnami tag (for example `25.7.5-debian-12-r0`), update it to an official image tag such as
-  `26.2` — Bitnami tags don't exist in the official repository.
+  `26.2` — Bitnami tags don't exist in the official repository. Note the switch is one-way: the
+  official image takes ownership of the data files, so rolling back to the Bitnami image requires
+  manually restoring their previous owner (`chown -R 1001:1001` on the volume).
 </Note>
 
 <Note>
diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx
index 6b7dbf3e73d..e5105ae57b6 100644
--- a/docs/self-hosting/kubernetes.mdx
+++ b/docs/self-hosting/kubernetes.mdx
@@ -287,7 +287,16 @@ redis:
   manifests without cluster access (for example with GitOps tools that use `helm template`), set
   `clickhouse.persistence.existingClaim` to the old PVC name
   (`data-<release>-clickhouse-shard0-0`) to keep your data — auto-detection can't run there, and
-  skipping this starts ClickHouse on a fresh empty volume.
+  skipping this starts ClickHouse on a fresh empty volume. If that happened, your old data is
+  still on the old PVC: delete the ClickHouse StatefulSet with `--cascade=orphan` (its volume
+  configuration is immutable), set `existingClaim`, and sync again.
+</Note>
+
+<Note>
+  If you pinned `clickhouse.image` to a Bitnami repository or tag in your values, update it to
+  the official `clickhouse/clickhouse-server` image — Bitnami tags don't exist there. The
+  bundled ClickHouse is single-node: the old Bitnami subchart keys (`shards`, `replicaCount`,
+  `keeper`) are no longer supported, so use an external ClickHouse for clustered setups.
 </Note>
 
 <Note>

From 1db78429d49a641e941a53c863452cab5ba1debd Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Jul 2026 17:51:27 +0100
Subject: [PATCH 07/13] fix(hosting): sync webapp to a custom ClickHouse secret
 and add a startup probe

The webapp now resolves CLICKHOUSE_PASSWORD through the same
auth.existingSecret/existingSecretKey values the bundled server uses,
so pointing the chart at a custom credentials secret keeps both sides
on the same password. A startup probe gives ClickHouse up to ten
minutes to load metadata on first boot with a large adopted data
volume before liveness checks begin.
---
 hosting/k8s/helm/templates/clickhouse.yaml | 11 +++++++++++
 hosting/k8s/helm/templates/webapp.yaml     |  6 ++++--
 hosting/k8s/helm/values.yaml               |  9 +++++++++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/hosting/k8s/helm/templates/clickhouse.yaml b/hosting/k8s/helm/templates/clickhouse.yaml
index edc7e62a760..fb42565dee9 100644
--- a/hosting/k8s/helm/templates/clickhouse.yaml
+++ b/hosting/k8s/helm/templates/clickhouse.yaml
@@ -131,6 +131,17 @@ spec:
             failureThreshold: {{ .Values.clickhouse.readinessProbe.failureThreshold }}
             successThreshold: {{ .Values.clickhouse.readinessProbe.successThreshold }}
           {{- end }}
+          {{- if .Values.clickhouse.startupProbe.enabled }}
+          startupProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: {{ .Values.clickhouse.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.clickhouse.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.clickhouse.startupProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.clickhouse.startupProbe.failureThreshold }}
+            successThreshold: {{ .Values.clickhouse.startupProbe.successThreshold }}
+          {{- end }}
           resources:
             {{- toYaml .Values.clickhouse.resources | nindent 12 }}
           volumeMounts:
diff --git a/hosting/k8s/helm/templates/webapp.yaml b/hosting/k8s/helm/templates/webapp.yaml
index 49acee98f4a..d666237daaf 100644
--- a/hosting/k8s/helm/templates/webapp.yaml
+++ b/hosting/k8s/helm/templates/webapp.yaml
@@ -419,11 +419,13 @@ spec:
                   name: {{ include "trigger-v4.clickhouse.external.secretName" . }}
                   key: {{ include "trigger-v4.clickhouse.external.passwordKey" . }}
             {{- else if .Values.clickhouse.deploy }}
+            {{- /* Same secret reference the bundled ClickHouse server uses, so a
+                   custom auth.existingSecret keeps the app and server in sync. */}}
             - name: CLICKHOUSE_PASSWORD
               valueFrom:
                 secretKeyRef:
-                  name: {{ include "trigger-v4.datastore.secretName" . }}
-                  key: clickhouse-admin-password
+                  name: {{ .Values.clickhouse.auth.existingSecret | default (include "trigger-v4.datastore.secretName" .) }}
+                  key: {{ .Values.clickhouse.auth.existingSecretKey | default "clickhouse-admin-password" }}
             {{- end }}
             - name: CLICKHOUSE_URL
               value: {{ include "trigger-v4.clickhouse.url" . | quote }}
diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml
index 239c9968683..e6ea51864f1 100644
--- a/hosting/k8s/helm/values.yaml
+++ b/hosting/k8s/helm/values.yaml
@@ -730,6 +730,15 @@ clickhouse:
     timeoutSeconds: 5
     failureThreshold: 5
     successThreshold: 1
+  # Generous startup window: first boot on a large adopted data volume can
+  # spend a while loading metadata before the HTTP listener answers.
+  startupProbe:
+    enabled: true
+    initialDelaySeconds: 5
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 60
+    successThreshold: 1
 
   # External ClickHouse connection (when deploy: false)
   external:

From 17b82384e9d43cd48c28b893cbb5b17e453b30a5 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Sat, 1 Aug 2026 10:35:45 +0100
Subject: [PATCH 08/13] fix(hosting): fail the upgrade when
 secrets.existingSecret is missing keys

When secrets.existingSecret is set the chart generates nothing and reads
every application and control-plane key from that Secret. A chart version
that starts consuming a new key therefore breaks the webapp rollout with a
CreateContainerConfigError partway through, leaving the release half-applied.

The pre-install/pre-upgrade validation now looks the Secret up and fails with
the full list of missing keys, so a running release is left untouched. The
lookup returns nothing under helm template and client-side dry-run, where the
check is skipped rather than guessing.
---
 .../templates/validate-external-config.yaml   | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/hosting/k8s/helm/templates/validate-external-config.yaml b/hosting/k8s/helm/templates/validate-external-config.yaml
index 939e699a423..7c5b25087aa 100644
--- a/hosting/k8s/helm/templates/validate-external-config.yaml
+++ b/hosting/k8s/helm/templates/validate-external-config.yaml
@@ -54,7 +54,36 @@ Application and control-plane secrets are auto-generated by templates/secrets.ya
 when left unset (retained across upgrades via lookup), so they need no fail-closed
 guard here. The webapp still rejects previously published values at startup, even
 when supplied via secrets.existingSecret.
+
+With secrets.existingSecret the chart generates nothing, so every key the workloads
+reference has to already be present in that Secret. Report missing keys up front -
+otherwise a chart version that starts consuming a new key only surfaces it as a
+CreateContainerConfigError partway through the rollout. The check is skipped when
+`lookup` returns nothing (helm template, client-side dry-run, or a Secret created
+later in the same apply): it can only report on what it can read.
 */}}
+{{- if .Values.secrets.existingSecret }}
+{{- $required := list "SESSION_SECRET" "MAGIC_LINK_SECRET" "ENCRYPTION_KEY" "PROVIDER_SECRET" "COORDINATOR_SECRET" "MANAGED_WORKER_SECRET" }}
+{{- if and .Values.s3.deploy (not .Values.s3.auth.existingSecret) }}
+{{- $required = concat $required (list "s3-auth-access-key-id" "s3-auth-secret-access-key") }}
+{{- end }}
+{{- if and (not .Values.s3.deploy) (not .Values.s3.external.existingSecret) .Values.s3.external.accessKeyId }}
+{{- $required = concat $required (list "s3-access-key-id" "s3-secret-access-key") }}
+{{- end }}
+{{- $found := lookup "v1" "Secret" .Release.Namespace .Values.secrets.existingSecret }}
+{{- if $found }}
+{{- $data := (get $found "data") | default dict }}
+{{- $missing := list }}
+{{- range $key := $required }}
+{{- if not (hasKey $data $key) }}
+{{- $missing = append $missing $key }}
+{{- end }}
+{{- end }}
+{{- if $missing }}
+{{- fail (printf "Secret %q (secrets.existingSecret) is missing required keys: %s. Add them before upgrading - while secrets.existingSecret is set the chart generates nothing and inline secrets.* values are ignored. See https://trigger.dev/docs/self-hosting/kubernetes#upgrading" .Values.secrets.existingSecret (join ", " $missing)) }}
+{{- end }}
+{{- end }}
+{{- end }}
 
 {{/*
 This template produces no output but will fail the deployment if validation fails

From fbf811c73575b7e09a471d3d20d69ce8a57d8f30 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Sat, 1 Aug 2026 10:35:50 +0100
Subject: [PATCH 09/13] docs(self-hosting): add a Kubernetes upgrade section

Collects the chart changes that need action before an upgrade: the two
secret keys the webapp started reading in 4.5.6, the move of the bundled
datastore credentials into the chart-managed Secret (with the old-to-new
mapping for consumers outside the chart), and a pointer to the ClickHouse
image notes.

Also corrects the existingSecret key list, which named the S3 environment
variables rather than the keys the chart actually reads.
---
 docs/self-hosting/kubernetes.mdx | 49 ++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx
index e5105ae57b6..3a6303c3941 100644
--- a/docs/self-hosting/kubernetes.mdx
+++ b/docs/self-hosting/kubernetes.mdx
@@ -80,6 +80,50 @@ kubectl port-forward svc/trigger-webapp 3040:3030 -n trigger
 kubectl logs -n trigger deployment/trigger-webapp | grep -A1 "magic link"
 ```
 
+## Upgrading
+
+`helm upgrade` keeps generated secrets, PVCs, and datastore volumes, so most upgrades need no preparation. These chart versions are the exceptions.
+
+### 4.5.6: two new required secret keys
+
+The webapp now reads `PROVIDER_SECRET` and `COORDINATOR_SECRET`. When you supply `secrets.existingSecret` the chart generates nothing and reads every key from your Secret, so both keys must exist there before you upgrade. Inline `secrets.*` values are ignored while `existingSecret` is set.
+
+Add them with two fresh 32-character hex values:
+
+```bash
+kubectl patch secret my-trigger-secrets -n trigger --type merge -p \
+  "{\"stringData\":{\"PROVIDER_SECRET\":\"$(openssl rand -hex 16)\",\"COORDINATOR_SECRET\":\"$(openssl rand -hex 16)\"}}"
+```
+
+The chart checks your Secret in a pre-upgrade hook and aborts with the list of missing keys, leaving the running release untouched. Without that check a missing key surfaces as a `CreateContainerConfigError` partway through the webapp rollout.
+
+<Note>
+  GitOps tools that render with `helm template` have no cluster access, so the check is
+  skipped and a missing key still reaches the rollout. Verify the keys yourself before syncing.
+</Note>
+
+### 4.5.6: ClickHouse credentials moved
+
+The bundled ClickHouse password moved out of the subchart-generated Secret and into the chart-managed datastore Secret:
+
+|        | Up to 4.5.5            | 4.5.6 and later             |
+| ------ | ---------------------- | --------------------------- |
+| Secret | `<release>-clickhouse` | `trigger-datastore`         |
+| Key    | `admin-password`       | `clickhouse-admin-password` |
+
+The webapp and the ClickHouse server both read the new location, so the upgrade itself needs no action. Repoint anything outside the chart that reads the old Secret — a maintenance CronJob, a Grafana datasource, an external secret sync:
+
+```bash
+kubectl get secret trigger-datastore -n trigger \
+  -o jsonpath='{.data.clickhouse-admin-password}' | base64 -d
+```
+
+The same move applies to the bundled PostgreSQL (`postgres-password`) and MinIO (`minio-root-user`, `minio-root-password`).
+
+### Bitnami ClickHouse to the official image
+
+The bundled ClickHouse runs the official `clickhouse/clickhouse-server` image instead of the Bitnami subchart. The chart adopts your existing data volume automatically, with three caveats covered under [ClickHouse](#clickhouse) in external services: rendering without cluster access, pinned Bitnami image tags, and storage that doesn't support `fsGroup`.
+
 ## Configuration
 
 Most values map directly to the environment variables documented in the [webapp](/self-hosting/env/webapp) and [supervisor](/self-hosting/env/supervisor) environment variable overview.
@@ -138,8 +182,9 @@ secrets:
 # - PROVIDER_SECRET
 # - COORDINATOR_SECRET
 # - MANAGED_WORKER_SECRET
-# - OBJECT_STORE_ACCESS_KEY_ID
-# - OBJECT_STORE_SECRET_ACCESS_KEY
+# Plus s3-auth-access-key-id and s3-auth-secret-access-key if you deploy the
+# bundled MinIO with s3.auth.existingSecret cleared. The chart lists any keys
+# it can't find and fails the install before touching a running release.
 secrets:
   enabled: false
   existingSecret: "your-existing-secret"

From f95ad01b82ee09825ff8f966e666c2e2dc66ed03 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Sat, 1 Aug 2026 12:57:19 +0100
Subject: [PATCH 10/13] fix(helm): auto-run the clickhouse chown init when
 adopting a bitnami volume

Upgrading the bundled ClickHouse from the Bitnami subchart to the official image adopts the old data volume, whose files are owned by the previous uid. The official server runs non-root (uid 101), so on storage where fsGroup can't relabel the volume (hostPath, NFS, local-path - common for self-hosters) it crashes with 'cd: /var/lib/clickhouse/data/: Permission denied'. Run the one-time chown init automatically whenever a volume is adopted (existingClaim set, explicitly or auto-detected); fresh installs create their own data and skip it, and volumePermissions.enabled still forces it. Verified live in kind: a bitnami->official upgrade on local-path storage now comes up with data and auth intact, with no manual flag.
---
 docs/self-hosting/kubernetes.mdx           | 8 +++++---
 hosting/k8s/helm/templates/clickhouse.yaml | 7 ++++++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx
index 3a6303c3941..e86557f5893 100644
--- a/docs/self-hosting/kubernetes.mdx
+++ b/docs/self-hosting/kubernetes.mdx
@@ -345,9 +345,11 @@ redis:
 </Note>
 
 <Note>
-  On storage that doesn't support `fsGroup` ownership changes (for example NFS or hostPath), set
-  `clickhouse.volumePermissions.enabled: true` so a one-time init container fixes the data
-  volume's ownership for the non-root ClickHouse server.
+  When a volume is adopted from the Bitnami-based chart, a one-time init container fixes its
+  ownership for the non-root ClickHouse server. This runs automatically on `helm upgrade` (and
+  when you set `clickhouse.persistence.existingClaim`), so storage that doesn't support `fsGroup`
+  ownership changes (NFS, hostPath, local-path) works without manual steps. Set
+  `clickhouse.volumePermissions.enabled: true` to force it in other cases.
 </Note>
 
 **Direct configuration:**
diff --git a/hosting/k8s/helm/templates/clickhouse.yaml b/hosting/k8s/helm/templates/clickhouse.yaml
index fb42565dee9..3a15a0a0ede 100644
--- a/hosting/k8s/helm/templates/clickhouse.yaml
+++ b/hosting/k8s/helm/templates/clickhouse.yaml
@@ -68,7 +68,12 @@ spec:
       securityContext:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if .Values.clickhouse.volumePermissions.enabled }}
+      {{- /* Adopting a volume from the Bitnami-based chart (auto-detected or an
+             explicit existingClaim) means the data is owned by the old uid. On
+             storage where fsGroup can't relabel it (hostPath, NFS, local-path),
+             the non-root server can't read it, so run the chown init container
+             automatically. Fresh installs create their own data and skip it. */}}
+      {{- if or .Values.clickhouse.volumePermissions.enabled $existingClaim }}
       initContainers:
         - name: volume-permissions
           image: "{{ .Values.global.imageRegistry | default .Values.clickhouse.volumePermissions.image.registry }}/{{ .Values.clickhouse.volumePermissions.image.repository }}:{{ .Values.clickhouse.volumePermissions.image.tag }}"

From 03477ab5a5eab34f948d19d337320fc34112d719 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Sat, 1 Aug 2026 13:20:57 +0100
Subject: [PATCH 11/13] fix(hosting): derive the docker clickhouse url username
 from CLICKHOUSE_USER

The bundled ClickHouse creates only the user named by CLICKHOUSE_USER (the official image removes the built-in default user when it differs), but the webapp CLICKHOUSE_URL and RUN_REPLICATION_CLICKHOUSE_URL hardcoded 'default', so a custom CLICKHOUSE_USER left the app unable to authenticate while the healthcheck (which already honours the var) stayed green. Derive the username in both URLs from CLICKHOUSE_USER. Also document, for the Helm chart, that inline external ClickHouse credentials are now percent-encoded (store the raw value), and that existingClaim should be pinned once a bitnami volume is adopted so client-side renders stay deterministic.
---
 docs/self-hosting/kubernetes.mdx         | 12 +++++++++++-
 hosting/docker/webapp/docker-compose.yml |  4 ++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx
index e86557f5893..634f91d208a 100644
--- a/docs/self-hosting/kubernetes.mdx
+++ b/docs/self-hosting/kubernetes.mdx
@@ -334,7 +334,10 @@ redis:
   (`data-<release>-clickhouse-shard0-0`) to keep your data — auto-detection can't run there, and
   skipping this starts ClickHouse on a fresh empty volume. If that happened, your old data is
   still on the old PVC: delete the ClickHouse StatefulSet with `--cascade=orphan` (its volume
-  configuration is immutable), set `existingClaim`, and sync again.
+  configuration is immutable), set `existingClaim`, and sync again. If you render without cluster
+  access, pin `clickhouse.persistence.existingClaim` for good once the volume is adopted: a later
+  render where the lookup can't see the PVC would otherwise try to re-add `volumeClaimTemplates`,
+  which the API server rejects on an existing StatefulSet.
 </Note>
 
 <Note>
@@ -364,6 +367,13 @@ clickhouse:
     password: "my-password"
 ```
 
+<Note>
+  An inline external `username`/`password` is percent-encoded into the connection URL for you, so
+  store the **raw** value - special characters like `@ : / %` are handled automatically. If you
+  previously percent-encoded the password by hand to work around this, switch back to the raw value.
+  (Credentials from `existingSecret` are injected at runtime and are unaffected.)
+</Note>
+
 **Using existing secrets (recommended):**
 
 ```yaml
diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml
index 1ecdd01dc49..a8bc1167b77 100644
--- a/hosting/docker/webapp/docker-compose.yml
+++ b/hosting/docker/webapp/docker-compose.yml
@@ -80,11 +80,11 @@ services:
       TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: bootstrap
       TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token
       # ClickHouse configuration
-      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123?secure=false}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://${CLICKHOUSE_USER:-default}:${CLICKHOUSE_PASSWORD}@clickhouse:8123?secure=false}
       CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info}
       # Run replication
       RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1}
-      RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123}
+      RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://${CLICKHOUSE_USER:-default}:${CLICKHOUSE_PASSWORD}@clickhouse:8123}
       RUN_REPLICATION_LOG_LEVEL: ${RUN_REPLICATION_LOG_LEVEL:-info}
       # Limits
       # TASK_PAYLOAD_OFFLOAD_THRESHOLD: 524288 # 512KB

From 68ffbf8b7d7c860de5c2fdaf8cae3bab7ebe5c73 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Sat, 1 Aug 2026 13:49:16 +0100
Subject: [PATCH 12/13] fix(helm): chown the clickhouse volume to the
 configured run-as user, not a fixed uid

The volume-permissions init container hardcoded chown -R 101:101, ignoring clickhouse.securityContext.runAsUser/runAsGroup. Anyone who overrides the run-as user (e.g. for a restricted cluster policy) got a data directory owned by an account the server no longer runs as, so it failed to start on the adopted volume. Derive the uid/gid from securityContext (defaulting to 101). Also document the transient ReadWriteOnce multi-attach warning during the upgrade window so operators don't abort mid-migration.
---
 docs/self-hosting/kubernetes.mdx           | 6 ++++++
 hosting/k8s/helm/templates/clickhouse.yaml | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx
index 634f91d208a..1939ff5dfb8 100644
--- a/docs/self-hosting/kubernetes.mdx
+++ b/docs/self-hosting/kubernetes.mdx
@@ -124,6 +124,12 @@ The same move applies to the bundled PostgreSQL (`postgres-password`) and MinIO
 
 The bundled ClickHouse runs the official `clickhouse/clickhouse-server` image instead of the Bitnami subchart. The chart adopts your existing data volume automatically, with three caveats covered under [ClickHouse](#clickhouse) in external services: rendering without cluster access, pinned Bitnami image tags, and storage that doesn't support `fsGroup`.
 
+<Note>
+  The data volume is `ReadWriteOnce`, so during the upgrade the new ClickHouse pod may sit in
+  `ContainerCreating` with a multi-attach warning for a minute while the old pod still holds the
+  volume. This clears itself once the old pod is removed later in the same upgrade - don't abort.
+</Note>
+
 ## Configuration
 
 Most values map directly to the environment variables documented in the [webapp](/self-hosting/env/webapp) and [supervisor](/self-hosting/env/supervisor) environment variable overview.
diff --git a/hosting/k8s/helm/templates/clickhouse.yaml b/hosting/k8s/helm/templates/clickhouse.yaml
index 3a15a0a0ede..3d7060a7521 100644
--- a/hosting/k8s/helm/templates/clickhouse.yaml
+++ b/hosting/k8s/helm/templates/clickhouse.yaml
@@ -78,7 +78,7 @@ spec:
         - name: volume-permissions
           image: "{{ .Values.global.imageRegistry | default .Values.clickhouse.volumePermissions.image.registry }}/{{ .Values.clickhouse.volumePermissions.image.repository }}:{{ .Values.clickhouse.volumePermissions.image.tag }}"
           imagePullPolicy: {{ .Values.clickhouse.volumePermissions.image.pullPolicy }}
-          command: ["sh", "-c", "chown -R 101:101 /var/lib/clickhouse"]
+          command: ["sh", "-c", "chown -R {{ .Values.clickhouse.securityContext.runAsUser | default 101 }}:{{ .Values.clickhouse.securityContext.runAsGroup | default 101 }} /var/lib/clickhouse"]
           securityContext:
             runAsUser: 0
             runAsNonRoot: false

From 8e283754a66f88b5d6230c3939ad7f982af30cc3 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Sat, 1 Aug 2026 13:59:35 +0100
Subject: [PATCH 13/13] fix(helm): make the clickhouse chown init idempotent so
 it doesn't re-walk on every restart

Because the adopted-volume condition stays true for the life of the release, the ownership init container runs on every pod restart, and an unconditional recursive chown re-walks the whole database each time - minutes of delay on large or network-backed data. Skip the chown when the volume root is already owned by the run-as user (mirroring fsGroupChangePolicy: OnRootMismatch), so only the first mount after adoption pays the cost.
---
 hosting/k8s/helm/templates/clickhouse.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/hosting/k8s/helm/templates/clickhouse.yaml b/hosting/k8s/helm/templates/clickhouse.yaml
index 3d7060a7521..c4baca8d4da 100644
--- a/hosting/k8s/helm/templates/clickhouse.yaml
+++ b/hosting/k8s/helm/templates/clickhouse.yaml
@@ -78,7 +78,12 @@ spec:
         - name: volume-permissions
           image: "{{ .Values.global.imageRegistry | default .Values.clickhouse.volumePermissions.image.registry }}/{{ .Values.clickhouse.volumePermissions.image.repository }}:{{ .Values.clickhouse.volumePermissions.image.tag }}"
           imagePullPolicy: {{ .Values.clickhouse.volumePermissions.image.pullPolicy }}
-          command: ["sh", "-c", "chown -R {{ .Values.clickhouse.securityContext.runAsUser | default 101 }}:{{ .Values.clickhouse.securityContext.runAsGroup | default 101 }} /var/lib/clickhouse"]
+          {{- $chownUser := .Values.clickhouse.securityContext.runAsUser | default 101 }}
+          {{- $chownGroup := .Values.clickhouse.securityContext.runAsGroup | default 101 }}
+          {{- /* Idempotent: skip the recursive chown when the volume root is already
+                 owned by the run-as user, so it doesn't re-walk the whole database on
+                 every restart (only the first mount after adoption pays the cost). */}}
+          command: ["sh", "-c", "test \"$(stat -c %u /var/lib/clickhouse)\" = \"{{ $chownUser }}\" || chown -R {{ $chownUser }}:{{ $chownGroup }} /var/lib/clickhouse"]
           securityContext:
             runAsUser: 0
             runAsNonRoot: false