From 64839a42637f479641a108f9078f0795f3fedaed Mon Sep 17 00:00:00 2001
From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com>
Date: Tue, 8 Sep 2026 22:24:05 +0530
Subject: [PATCH 1/4] feat(*): continuos deployment script updates

---
 .github/workflows/create-release.yml     |  95 +++++++++++++++++---
 .github/workflows/deploy-staging-ecs.yml |   2 +-
 .github/workflows/deploy-staging.yml     | 105 +++++++++++++++++++++++
 backend/Dockerfile                       |   3 +
 backend/app/core/config.py               |   1 +
 backend/app/main.py                      |   1 +
 6 files changed, 194 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
index d094405ad..8df8afa82 100644
--- a/.github/workflows/create-release.yml
+++ b/.github/workflows/create-release.yml
@@ -62,6 +62,7 @@ jobs:
           TAG: ${{ github.ref_name }}
         run: |
           docker build \
+            --build-arg GIT_SHA=${{ github.sha }} \
             -t $REGISTRY/$REPOSITORY:latest \
             -t $REGISTRY/$REPOSITORY:$TAG \
             ./backend
@@ -103,16 +104,86 @@ jobs:
           fi
           echo "Migration completed successfully"
 
-      - name: Deploy to ECS
+      - name: Deploy to ECS and verify rollout
+        # Bound the wait; the circuit breaker itself trips well before this.
+        timeout-minutes: 15
+        env:
+          CLUSTER: ${{ vars.AWS_RESOURCE_PREFIX }}-cluster
+          POLL_INTERVAL: "15"
         run: |
-          aws ecs update-service \
-            --cluster ${{ vars.AWS_RESOURCE_PREFIX }}-cluster \
-            --service ${{ vars.AWS_RESOURCE_PREFIX }}-service \
-            --task-definition ${{ vars.AWS_RESOURCE_PREFIX }}-task \
-            --force-new-deployment
-
-          aws ecs update-service \
-            --cluster ${{ vars.AWS_RESOURCE_PREFIX }}-cluster \
-            --service ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task \
-            --task-definition ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task \
-            --force-new-deployment
+          # A plain update-service returns before the new tasks are healthy, and
+          # the old task can keep answering 200 while the new one crash-loops.
+          # So roll each service to its family's latest revision, then poll the
+          # PRIMARY deployment's rolloutState. With the deployment circuit
+          # breaker enabled on the service (one-time prep), a bad rollout flips
+          # to FAILED and auto-rolls-back — which we surface here as a failure.
+          deploy_and_wait() {
+            SERVICE="$1"
+            FAMILY="$2"
+            echo "[$SERVICE] forcing new deployment on family $FAMILY"
+            aws ecs update-service \
+              --cluster "$CLUSTER" \
+              --service "$SERVICE" \
+              --task-definition "$FAMILY" \
+              --force-new-deployment >/dev/null
+
+            while true; do
+              STATE=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \
+                --query "services[0].deployments[?status=='PRIMARY'].rolloutState | [0]" \
+                --output text)
+              case "$STATE" in
+                COMPLETED)
+                  echo "[$SERVICE] rollout COMPLETED"
+                  return 0 ;;
+                FAILED)
+                  echo "::error::[$SERVICE] rollout FAILED — new tasks never became healthy (rolled back by circuit breaker)"
+                  return 1 ;;
+                *)
+                  echo "[$SERVICE] rollout $STATE — waiting ${POLL_INTERVAL}s"
+                  sleep "$POLL_INTERVAL" ;;
+              esac
+            done
+          }
+
+          deploy_and_wait "${{ vars.AWS_RESOURCE_PREFIX }}-service" "${{ vars.AWS_RESOURCE_PREFIX }}-task"
+          deploy_and_wait "${{ vars.AWS_RESOURCE_PREFIX }}-celery-task" "${{ vars.AWS_RESOURCE_PREFIX }}-celery-task"
+
+  # Green "healthy" on a clean release, red "failed" otherwise (including a
+  # release aborted because CI never passed on the tagged commit).
+  notify:
+    needs: [verify-ci, build]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Discord
+        env:
+          DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          NAME: kaapi-production
+          RELEASE: ${{ github.ref_name }}
+          # true only when the build+deploy job succeeded.
+          OK: ${{ needs.build.result == 'success' }}
+        run: |
+          [ -z "$DISCORD_WEBHOOK_URL" ] && { echo "No webhook configured, skipping"; exit 0; }
+          if [ "$OK" = "true" ]; then
+            TITLE="🟢 $NAME deployment healthy"; COLOR=3066993   # green
+          else
+            TITLE="🔴 $NAME deployment failed";  COLOR=15158332  # red
+          fi
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+          payload=$(jq -n \
+            --arg title "$TITLE" \
+            --argjson color "$COLOR" \
+            --arg release "$RELEASE" \
+            --arg sha "$SHA_SHORT" \
+            --arg url "$RUN_URL" \
+            '{embeds: [{
+              title: $title, url: $url, color: $color,
+              fields: [
+                {name: "Release", value: $release, inline: true},
+                {name: "SHA", value: $sha, inline: true}
+              ],
+              timestamp: (now | todate)
+            }]}')
+          curl -sf -H "Content-Type: application/json" -X POST -d "$payload" "$DISCORD_WEBHOOK_URL" \
+            || echo "Discord notification failed to send"
diff --git a/.github/workflows/deploy-staging-ecs.yml b/.github/workflows/deploy-staging-ecs.yml
index fffcbb714..3ebb43bb5 100644
--- a/.github/workflows/deploy-staging-ecs.yml
+++ b/.github/workflows/deploy-staging-ecs.yml
@@ -36,7 +36,7 @@ jobs:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           REPOSITORY: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-repo
         run: |
-          docker build -t $REGISTRY/$REPOSITORY:latest ./backend
+          docker build --build-arg GIT_SHA=${{ github.sha }} -t $REGISTRY/$REPOSITORY:latest ./backend
           docker push $REGISTRY/$REPOSITORY:latest
 
       - name: Run database migrations
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index 39c788eef..ef5acaea7 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -85,3 +85,108 @@ jobs:
             --instance-id "$INSTANCE_ID" \
             --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
             --output json
+
+  ecs-rehearsal:
+    needs: deploy
+    runs-on: ubuntu-latest
+    environment: AWS_ENV_VARS
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v7
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
+          aws-region: ap-south-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build and push staging image
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-repo
+        run: |
+          docker build \
+            --build-arg GIT_SHA=${{ github.sha }} \
+            -t $REGISTRY/$REPOSITORY:latest \
+            ./backend
+          docker push $REGISTRY/$REPOSITORY:latest
+
+      - name: Scale staging ECS up and verify rollout
+        timeout-minutes: 15
+        env:
+          CLUSTER: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-cluster
+          SERVICE: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-service
+          POLL_INTERVAL: "15"
+        run: |
+          aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \
+            --desired-count 1 --force-new-deployment >/dev/null
+          echo "[$SERVICE] scaled to 1; waiting for rollout"
+          while true; do
+            STATE=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \
+              --query "services[0].deployments[?status=='PRIMARY'].rolloutState | [0]" \
+              --output text)
+            case "$STATE" in
+              COMPLETED) echo "[$SERVICE] rehearsal rollout COMPLETED"; break ;;
+              FAILED)
+                echo "::error::[$SERVICE] rehearsal FAILED — the production ECS deploy path is broken"
+                exit 1 ;;
+              *) echo "[$SERVICE] rollout $STATE — waiting ${POLL_INTERVAL}s"; sleep "$POLL_INTERVAL" ;;
+            esac
+          done
+
+      - name: Scale staging ECS back to 0
+        if: always()
+        env:
+          CLUSTER: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-cluster
+          SERVICE: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-service
+        run: |
+          aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \
+            --desired-count 0 >/dev/null
+          echo "[$SERVICE] scaled back to 0"
+
+  # Green "healthy" on a clean deploy + rehearsal, red "failed" otherwise.
+  # Skipped (not sent) when the deploy itself was skipped on a red CI.
+  notify:
+    needs: [deploy, ecs-rehearsal]
+    if: ${{ always() && needs.deploy.result != 'skipped' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Discord
+        env:
+          DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          NAME: kaapi-staging
+          RELEASE: "#${{ github.run_number }}"
+          # true only when every deploy job succeeded.
+          OK: ${{ needs.deploy.result == 'success' && needs.ecs-rehearsal.result == 'success' }}
+        run: |
+          [ -z "$DISCORD_WEBHOOK_URL" ] && { echo "No webhook configured, skipping"; exit 0; }
+          if [ "$OK" = "true" ]; then
+            TITLE="🟢 $NAME deployment healthy"; COLOR=3066993   # green
+          else
+            TITLE="🔴 $NAME deployment failed";  COLOR=15158332  # red
+          fi
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+          payload=$(jq -n \
+            --arg title "$TITLE" \
+            --argjson color "$COLOR" \
+            --arg release "$RELEASE" \
+            --arg sha "$SHA_SHORT" \
+            --arg url "$RUN_URL" \
+            '{embeds: [{
+              title: $title, url: $url, color: $color,
+              fields: [
+                {name: "Release", value: $release, inline: true},
+                {name: "SHA", value: $sha, inline: true}
+              ],
+              timestamp: (now | todate)
+            }]}')
+          curl -sf -H "Content-Type: application/json" -X POST -d "$payload" "$DISCORD_WEBHOOK_URL" \
+            || echo "Discord notification failed to send"
diff --git a/backend/Dockerfile b/backend/Dockerfile
index f34b22b36..7bc1453d4 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -44,6 +44,9 @@ COPY scripts /app/scripts
 COPY app /app/app
 COPY alembic.ini /app/alembic.ini
 
+ARG GIT_SHA=unknown
+ENV GIT_SHA=$GIT_SHA
+
 # Expose port 80
 EXPOSE 80
 
diff --git a/backend/app/core/config.py b/backend/app/core/config.py
index cc105fbec..cb1f3badf 100644
--- a/backend/app/core/config.py
+++ b/backend/app/core/config.py
@@ -46,6 +46,7 @@ class Settings(BaseSettings):
 
     PROJECT_NAME: str
     API_VERSION: str = "0.5.0"
+    GIT_SHA: str = "unknown"
     SENTRY_DSN: HttpUrl | None = None
     DISCORD_STATS_WEBHOOK_URL: HttpUrl | None = None
     POSTGRES_SERVER: str
diff --git a/backend/app/main.py b/backend/app/main.py
index 5d2a09cf0..0c1c3f66b 100644
--- a/backend/app/main.py
+++ b/backend/app/main.py
@@ -105,4 +105,5 @@ def custom_openapi():
 async def health() -> dict[str, str | float]:
     return {
         "status": "ok",
+        "sha": settings.GIT_SHA,
     }

From 60185fe6519ab4f35608a06f2f5660d2aef925ab Mon Sep 17 00:00:00 2001
From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com>
Date: Tue, 8 Sep 2026 23:38:55 +0530
Subject: [PATCH 2/4] fix(*): get the role id from the github secret

---
 .github/workflows/create-release.yml          |  12 +-
 .github/workflows/deploy-staging-ecs.yml      |   2 +-
 .../glific-evals-configs-api-coverage.md      | 107 ++++++++++++++++++
 3 files changed, 110 insertions(+), 11 deletions(-)
 create mode 100644 docs/guides/glific-evals-configs-api-coverage.md

diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
index 8df8afa82..23ddb4495 100644
--- a/.github/workflows/create-release.yml
+++ b/.github/workflows/create-release.yml
@@ -46,9 +46,9 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v6 # More information on this action can be found below in the 'AWS Credentials' section
+        uses: aws-actions/configure-aws-credentials@v6
         with:
-          role-to-assume: arn:aws:iam::024209611402:role/github-action-role
+          role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
           aws-region: ap-south-1
 
       - name: Login to Amazon ECR
@@ -111,12 +111,6 @@ jobs:
           CLUSTER: ${{ vars.AWS_RESOURCE_PREFIX }}-cluster
           POLL_INTERVAL: "15"
         run: |
-          # A plain update-service returns before the new tasks are healthy, and
-          # the old task can keep answering 200 while the new one crash-loops.
-          # So roll each service to its family's latest revision, then poll the
-          # PRIMARY deployment's rolloutState. With the deployment circuit
-          # breaker enabled on the service (one-time prep), a bad rollout flips
-          # to FAILED and auto-rolls-back — which we surface here as a failure.
           deploy_and_wait() {
             SERVICE="$1"
             FAMILY="$2"
@@ -148,8 +142,6 @@ jobs:
           deploy_and_wait "${{ vars.AWS_RESOURCE_PREFIX }}-service" "${{ vars.AWS_RESOURCE_PREFIX }}-task"
           deploy_and_wait "${{ vars.AWS_RESOURCE_PREFIX }}-celery-task" "${{ vars.AWS_RESOURCE_PREFIX }}-celery-task"
 
-  # Green "healthy" on a clean release, red "failed" otherwise (including a
-  # release aborted because CI never passed on the tagged commit).
   notify:
     needs: [verify-ci, build]
     if: always()
diff --git a/.github/workflows/deploy-staging-ecs.yml b/.github/workflows/deploy-staging-ecs.yml
index 3ebb43bb5..6aacd23af 100644
--- a/.github/workflows/deploy-staging-ecs.yml
+++ b/.github/workflows/deploy-staging-ecs.yml
@@ -23,7 +23,7 @@ jobs:
         # More information on this action can be found below in the 'AWS Credentials' section
         uses: aws-actions/configure-aws-credentials@v6
         with:
-          role-to-assume: arn:aws:iam::024209611402:role/github-action-role
+          role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
           aws-region: ap-south-1
 
       - name: Login to Amazon ECR
diff --git a/docs/guides/glific-evals-configs-api-coverage.md b/docs/guides/glific-evals-configs-api-coverage.md
new file mode 100644
index 000000000..89ffcaa3b
--- /dev/null
+++ b/docs/guides/glific-evals-configs-api-coverage.md
@@ -0,0 +1,107 @@
+# Glific Evals & Configs UI → Kaapi API Coverage
+
+This maps the **Evals** and **Configs** flows in the Glific `AI Assistants`
+prototype (`glific-evals-v13.html`) to the corresponding Kaapi backend APIs, and
+marks whether each API already exists.
+
+**How the prototype's concepts map to Kaapi:**
+
+| Prototype concept | Kaapi concept |
+| --- | --- |
+| An "Assistant" | A **Config** (`/api/v1/configs`) |
+| A saved "Version" (prompt + model + settings) | A **Config version** (`/configs/{id}/versions`) |
+| A "Golden Q&A set" | An **evaluation dataset** (`/evaluations/datasets`) |
+| A "Run" / evaluation | An **evaluation run** (`/api/v2/evaluations`) |
+
+---
+
+## Configs flow
+
+| API | Available |
+| --- | --- |
+| Get Configs (list assistants) | **Yes** |
+| Get Config (open an assistant) | **Yes** |
+| Create Config (create assistant) | **Yes** |
+| Update Config (rename / edit) | **Yes** |
+| Delete Config (delete assistant) | **Yes** |
+| Create Config Version (Save Version) | **Yes** |
+| Get Config Versions (version dropdown) | **Yes** |
+| Get Config Version (load a version) | **Yes** |
+| Duplicate Config (Duplicate assistant) | **No** |
+| Publish / Set-Live Config Version (Publish & go live) | **No** |
+
+### Endpoints & notes
+
+| API | Available | Kaapi endpoint / notes |
+| --- | --- | --- |
+| Get Configs | ✅ Yes | `GET /api/v1/configs` |
+| Get Config | ✅ Yes | `GET /api/v1/configs/{config_id}` |
+| Create Config | ✅ Yes | `POST /api/v1/configs` — also creates version 1 in the same call |
+| Update Config | ✅ Yes | `PATCH /api/v1/configs/{config_id}` |
+| Delete Config | ✅ Yes | `DELETE /api/v1/configs/{config_id}` |
+| Create Config Version | ✅ Yes | `POST /api/v1/configs/{config_id}/versions` — matches "Save Version" (each save = a new version) |
+| Get Config Versions | ✅ Yes | `GET /api/v1/configs/{config_id}/versions` |
+| Get Config Version | ✅ Yes | `GET /api/v1/configs/{config_id}/versions/{version_number}` |
+| Duplicate Config | ❌ No | No clone/copy endpoint. The UI "Duplicate" would need a new API (or client-side create-config from a fetched blob). |
+| Publish / Set-Live Config Version | ❌ No | **No concept of a live/published/active version in Kaapi.** The `ConfigVersion` model has no `is_live`/`published`/`active` field, and there is no promote/go-live endpoint. The prototype's "Publish & go live", the LIVE badge, and "was live" states have no backing API. |
+
+---
+
+## Evals flow
+
+| API | Available |
+| --- | --- |
+| Upload Dataset (add Golden Q&A set) | **Yes** |
+| List Datasets (Manage sets) | **Yes** |
+| Get Dataset (view a set) | **Yes** (partial) |
+| Delete Dataset (delete a set) | **Yes** |
+| Export Dataset (export set CSV) | **No** |
+| Run Eval (Run evaluation) | **Yes** |
+| Get Eval (in-progress / completed status) | **Yes** |
+| Get Eval Results (metrics + per-question) | **Yes** |
+| List Evals (History) | **Yes** |
+| Export Eval Results (Export CSV) | **No** |
+| Improve Prompt (What to change next) | **Yes** |
+| Run-time / online evaluation (live conversation scoring) | **No** |
+
+### Endpoints & notes
+
+| API | Available | Kaapi endpoint / notes |
+| --- | --- | --- |
+| Upload Dataset | ✅ Yes | `POST /api/v2/evaluations/datasets` (v1 also exists). CSV columns `question`, `answer`, optional `category` — matches the prototype's CSV. |
+| List Datasets | ✅ Yes | `GET /api/v1/evaluations/datasets` (v1 only; no v2 variant) |
+| Get Dataset | ⚠️ Partial | `GET /api/v1/evaluations/datasets/{dataset_id}` returns the dataset record, but there is **no per-item/questions listing route**. The prototype's "View set" question table would read rows from the stored CSV (`signed_url`), not a questions API. |
+| Delete Dataset | ✅ Yes | `DELETE /api/v1/evaluations/datasets/{dataset_id}` |
+| Export Dataset | ❌ No | The prototype exports the set as CSV client-side; there's no API for it (the source CSV is already retrievable via the dataset's `signed_url`). |
+| Run Eval | ✅ Yes | `POST /api/v2/evaluations` — body `dataset_id`, `experiment_name`, `config_id`, `config_version`. **Mismatch:** the prototype's per-run duplication (1× / 5×) does not map to the run endpoint — in Kaapi `duplication_factor` is set at **dataset upload** time (`1–5`), not per run. |
+| Get Eval (status) | ✅ Yes | `GET /api/v1/evaluations/{evaluation_id}` — `status` goes `processing → completed`/`failed`, backing the "in progress" / "completed" job banner. |
+| Get Eval Results | ✅ Yes | Same `GET /api/v1/evaluations/{evaluation_id}` — run-level `score` + per-row judge scores/reasoning in the `score_trace_url` trace. Covers the overall gauge + question-level table. |
+| List Evals (History) | ✅ Yes | `GET /api/v1/evaluations` (`limit`/`offset`). The prototype's version/set filters and sorting would be applied client-side. |
+| Export Eval Results (CSV) | ❌ No | No CSV/file export route. `GET /api/v1/evaluations/{id}` has an `export_format` param but it only accepts `row`/`grouped` and just restructures the JSON — it does not produce a CSV. The prototype's "⤓ Export CSV" has no API. |
+| Improve Prompt | ✅ Yes | `POST /api/v2/evaluations/{evaluation_id}/improve-prompt` (v1 also exists). Backs the "What to change next" / suggested-prompt-change panel. Async — delivers to an HTTPS `callback_url`. |
+| Run-time / online evaluation | ❌ No | **No API.** The entire "Run-time Evaluations" tab (continuous scoring of real conversations, rolling trend, flagged log) has no backing endpoint — Kaapi only does the on-demand Golden Q&A run. |
+
+---
+
+## Adjacent UI surfaces (outside Evals & Configs)
+
+Called out for completeness — these prototype tabs aren't part of the Evals/Configs
+flow but affect a full build:
+
+| API | Available | Notes |
+| --- | --- | --- |
+| Try It Out (run one prompt against a saved config) | ❌ No | No single-prompt sandbox route takes a `config_id`. The only config-by-reference execution is the full STS chain `POST /api/v1/llm/chain/sts`, not a text playground. `POST /responses` exists but doesn't reference a config. |
+| Knowledge Base (list / add / remove files, vector store) | — | KB management lives outside the config/evaluations routes; not evaluated here. The config blob only references `knowledge_base_ids`. |
+
+---
+
+## Summary of gaps
+
+Everything the Evals & Configs flow needs **exists today except**:
+
+1. **Publish / go-live for a config version** — no live/published/active concept in Kaapi at all (the biggest gap; the prototype's whole version-lifecycle UI depends on it).
+2. **Duplicate config** — no clone endpoint.
+3. **Run-time / online evaluation** — no live-traffic scoring; only on-demand runs.
+4. **CSV export** of eval results and of a dataset — no export API.
+5. **Per-run duplication factor** — Kaapi sets it at dataset-upload time, not per run.
+6. **Dataset questions listing** — `GET dataset` returns the record, not an items API (rows come from the stored CSV).

From 3c7a560d6daa8641fb1210466b0ace2cbb9f43d1 Mon Sep 17 00:00:00 2001
From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com>
Date: Wed, 9 Sep 2026 00:02:17 +0530
Subject: [PATCH 3/4] fix(*): get the role id from the github secret

---
 .github/workflows/deploy-staging-ecs.yml      |   1 -
 .github/workflows/deploy-staging.yml          |   2 -
 .../glific-evals-configs-api-coverage.md      | 107 ------------------
 3 files changed, 110 deletions(-)
 delete mode 100644 docs/guides/glific-evals-configs-api-coverage.md

diff --git a/.github/workflows/deploy-staging-ecs.yml b/.github/workflows/deploy-staging-ecs.yml
index 6aacd23af..12dbeb76d 100644
--- a/.github/workflows/deploy-staging-ecs.yml
+++ b/.github/workflows/deploy-staging-ecs.yml
@@ -20,7 +20,6 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Configure AWS credentials
-        # More information on this action can be found below in the 'AWS Credentials' section
         uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index ef5acaea7..462a459ab 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -151,8 +151,6 @@ jobs:
             --desired-count 0 >/dev/null
           echo "[$SERVICE] scaled back to 0"
 
-  # Green "healthy" on a clean deploy + rehearsal, red "failed" otherwise.
-  # Skipped (not sent) when the deploy itself was skipped on a red CI.
   notify:
     needs: [deploy, ecs-rehearsal]
     if: ${{ always() && needs.deploy.result != 'skipped' }}
diff --git a/docs/guides/glific-evals-configs-api-coverage.md b/docs/guides/glific-evals-configs-api-coverage.md
deleted file mode 100644
index 89ffcaa3b..000000000
--- a/docs/guides/glific-evals-configs-api-coverage.md
+++ /dev/null
@@ -1,107 +0,0 @@
-# Glific Evals & Configs UI → Kaapi API Coverage
-
-This maps the **Evals** and **Configs** flows in the Glific `AI Assistants`
-prototype (`glific-evals-v13.html`) to the corresponding Kaapi backend APIs, and
-marks whether each API already exists.
-
-**How the prototype's concepts map to Kaapi:**
-
-| Prototype concept | Kaapi concept |
-| --- | --- |
-| An "Assistant" | A **Config** (`/api/v1/configs`) |
-| A saved "Version" (prompt + model + settings) | A **Config version** (`/configs/{id}/versions`) |
-| A "Golden Q&A set" | An **evaluation dataset** (`/evaluations/datasets`) |
-| A "Run" / evaluation | An **evaluation run** (`/api/v2/evaluations`) |
-
----
-
-## Configs flow
-
-| API | Available |
-| --- | --- |
-| Get Configs (list assistants) | **Yes** |
-| Get Config (open an assistant) | **Yes** |
-| Create Config (create assistant) | **Yes** |
-| Update Config (rename / edit) | **Yes** |
-| Delete Config (delete assistant) | **Yes** |
-| Create Config Version (Save Version) | **Yes** |
-| Get Config Versions (version dropdown) | **Yes** |
-| Get Config Version (load a version) | **Yes** |
-| Duplicate Config (Duplicate assistant) | **No** |
-| Publish / Set-Live Config Version (Publish & go live) | **No** |
-
-### Endpoints & notes
-
-| API | Available | Kaapi endpoint / notes |
-| --- | --- | --- |
-| Get Configs | ✅ Yes | `GET /api/v1/configs` |
-| Get Config | ✅ Yes | `GET /api/v1/configs/{config_id}` |
-| Create Config | ✅ Yes | `POST /api/v1/configs` — also creates version 1 in the same call |
-| Update Config | ✅ Yes | `PATCH /api/v1/configs/{config_id}` |
-| Delete Config | ✅ Yes | `DELETE /api/v1/configs/{config_id}` |
-| Create Config Version | ✅ Yes | `POST /api/v1/configs/{config_id}/versions` — matches "Save Version" (each save = a new version) |
-| Get Config Versions | ✅ Yes | `GET /api/v1/configs/{config_id}/versions` |
-| Get Config Version | ✅ Yes | `GET /api/v1/configs/{config_id}/versions/{version_number}` |
-| Duplicate Config | ❌ No | No clone/copy endpoint. The UI "Duplicate" would need a new API (or client-side create-config from a fetched blob). |
-| Publish / Set-Live Config Version | ❌ No | **No concept of a live/published/active version in Kaapi.** The `ConfigVersion` model has no `is_live`/`published`/`active` field, and there is no promote/go-live endpoint. The prototype's "Publish & go live", the LIVE badge, and "was live" states have no backing API. |
-
----
-
-## Evals flow
-
-| API | Available |
-| --- | --- |
-| Upload Dataset (add Golden Q&A set) | **Yes** |
-| List Datasets (Manage sets) | **Yes** |
-| Get Dataset (view a set) | **Yes** (partial) |
-| Delete Dataset (delete a set) | **Yes** |
-| Export Dataset (export set CSV) | **No** |
-| Run Eval (Run evaluation) | **Yes** |
-| Get Eval (in-progress / completed status) | **Yes** |
-| Get Eval Results (metrics + per-question) | **Yes** |
-| List Evals (History) | **Yes** |
-| Export Eval Results (Export CSV) | **No** |
-| Improve Prompt (What to change next) | **Yes** |
-| Run-time / online evaluation (live conversation scoring) | **No** |
-
-### Endpoints & notes
-
-| API | Available | Kaapi endpoint / notes |
-| --- | --- | --- |
-| Upload Dataset | ✅ Yes | `POST /api/v2/evaluations/datasets` (v1 also exists). CSV columns `question`, `answer`, optional `category` — matches the prototype's CSV. |
-| List Datasets | ✅ Yes | `GET /api/v1/evaluations/datasets` (v1 only; no v2 variant) |
-| Get Dataset | ⚠️ Partial | `GET /api/v1/evaluations/datasets/{dataset_id}` returns the dataset record, but there is **no per-item/questions listing route**. The prototype's "View set" question table would read rows from the stored CSV (`signed_url`), not a questions API. |
-| Delete Dataset | ✅ Yes | `DELETE /api/v1/evaluations/datasets/{dataset_id}` |
-| Export Dataset | ❌ No | The prototype exports the set as CSV client-side; there's no API for it (the source CSV is already retrievable via the dataset's `signed_url`). |
-| Run Eval | ✅ Yes | `POST /api/v2/evaluations` — body `dataset_id`, `experiment_name`, `config_id`, `config_version`. **Mismatch:** the prototype's per-run duplication (1× / 5×) does not map to the run endpoint — in Kaapi `duplication_factor` is set at **dataset upload** time (`1–5`), not per run. |
-| Get Eval (status) | ✅ Yes | `GET /api/v1/evaluations/{evaluation_id}` — `status` goes `processing → completed`/`failed`, backing the "in progress" / "completed" job banner. |
-| Get Eval Results | ✅ Yes | Same `GET /api/v1/evaluations/{evaluation_id}` — run-level `score` + per-row judge scores/reasoning in the `score_trace_url` trace. Covers the overall gauge + question-level table. |
-| List Evals (History) | ✅ Yes | `GET /api/v1/evaluations` (`limit`/`offset`). The prototype's version/set filters and sorting would be applied client-side. |
-| Export Eval Results (CSV) | ❌ No | No CSV/file export route. `GET /api/v1/evaluations/{id}` has an `export_format` param but it only accepts `row`/`grouped` and just restructures the JSON — it does not produce a CSV. The prototype's "⤓ Export CSV" has no API. |
-| Improve Prompt | ✅ Yes | `POST /api/v2/evaluations/{evaluation_id}/improve-prompt` (v1 also exists). Backs the "What to change next" / suggested-prompt-change panel. Async — delivers to an HTTPS `callback_url`. |
-| Run-time / online evaluation | ❌ No | **No API.** The entire "Run-time Evaluations" tab (continuous scoring of real conversations, rolling trend, flagged log) has no backing endpoint — Kaapi only does the on-demand Golden Q&A run. |
-
----
-
-## Adjacent UI surfaces (outside Evals & Configs)
-
-Called out for completeness — these prototype tabs aren't part of the Evals/Configs
-flow but affect a full build:
-
-| API | Available | Notes |
-| --- | --- | --- |
-| Try It Out (run one prompt against a saved config) | ❌ No | No single-prompt sandbox route takes a `config_id`. The only config-by-reference execution is the full STS chain `POST /api/v1/llm/chain/sts`, not a text playground. `POST /responses` exists but doesn't reference a config. |
-| Knowledge Base (list / add / remove files, vector store) | — | KB management lives outside the config/evaluations routes; not evaluated here. The config blob only references `knowledge_base_ids`. |
-
----
-
-## Summary of gaps
-
-Everything the Evals & Configs flow needs **exists today except**:
-
-1. **Publish / go-live for a config version** — no live/published/active concept in Kaapi at all (the biggest gap; the prototype's whole version-lifecycle UI depends on it).
-2. **Duplicate config** — no clone endpoint.
-3. **Run-time / online evaluation** — no live-traffic scoring; only on-demand runs.
-4. **CSV export** of eval results and of a dataset — no export API.
-5. **Per-run duplication factor** — Kaapi sets it at dataset-upload time, not per run.
-6. **Dataset questions listing** — `GET dataset` returns the record, not an items API (rows come from the stored CSV).

From c4f492d22036c9352f8a232a67269b5dcc8d1d0f Mon Sep 17 00:00:00 2001
From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com>
Date: Thu, 10 Sep 2026 12:42:09 +0530
Subject: [PATCH 4/4] fix(*): few updates on the deployment staging script

---
 .github/workflows/deploy-staging.yml | 50 ++++++++++++++++------------
 1 file changed, 29 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index 462a459ab..7d7cafd2b 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -121,35 +121,43 @@ jobs:
       - name: Scale staging ECS up and verify rollout
         timeout-minutes: 15
         env:
-          CLUSTER: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-cluster
-          SERVICE: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-service
+          CLUSTER: kaapi-staging
+          SERVICES: "kaapi-staging-backend-celery kaapi-staging-celery-worker"
           POLL_INTERVAL: "15"
         run: |
-          aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \
-            --desired-count 1 --force-new-deployment >/dev/null
-          echo "[$SERVICE] scaled to 1; waiting for rollout"
-          while true; do
-            STATE=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \
-              --query "services[0].deployments[?status=='PRIMARY'].rolloutState | [0]" \
-              --output text)
-            case "$STATE" in
-              COMPLETED) echo "[$SERVICE] rehearsal rollout COMPLETED"; break ;;
-              FAILED)
-                echo "::error::[$SERVICE] rehearsal FAILED — the production ECS deploy path is broken"
-                exit 1 ;;
-              *) echo "[$SERVICE] rollout $STATE — waiting ${POLL_INTERVAL}s"; sleep "$POLL_INTERVAL" ;;
-            esac
+          for SERVICE in $SERVICES; do
+            aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \
+              --desired-count 1 --force-new-deployment >/dev/null
+            echo "[$SERVICE] scaled to 1"
+          done
+
+          for SERVICE in $SERVICES; do
+            echo "[$SERVICE] waiting for rollout"
+            while true; do
+              STATE=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \
+                --query "services[0].deployments[?status=='PRIMARY'].rolloutState | [0]" \
+                --output text)
+              case "$STATE" in
+                COMPLETED) echo "[$SERVICE] rehearsal rollout COMPLETED"; break ;;
+                FAILED)
+                  echo "::error::[$SERVICE] rehearsal FAILED — the production ECS deploy path is broken"
+                  exit 1 ;;
+                *) echo "[$SERVICE] rollout $STATE — waiting ${POLL_INTERVAL}s"; sleep "$POLL_INTERVAL" ;;
+              esac
+            done
           done
 
       - name: Scale staging ECS back to 0
         if: always()
         env:
-          CLUSTER: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-cluster
-          SERVICE: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-service
+          CLUSTER: kaapi-staging
+          SERVICES: "kaapi-staging-backend-celery kaapi-staging-celery-worker"
         run: |
-          aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \
-            --desired-count 0 >/dev/null
-          echo "[$SERVICE] scaled back to 0"
+          for SERVICE in $SERVICES; do
+            aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \
+              --desired-count 0 >/dev/null
+            echo "[$SERVICE] scaled back to 0"
+          done
 
   notify:
     needs: [deploy, ecs-rehearsal]