feat(continuous-deployment): Update deployment scripts

.github/workflows/create-release.yml

@@ -46,9 +46,9 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v6 # More information on this action can be found below in the 'AWS Credentials' section
+        uses: aws-actions/configure-aws-credentials@v6
         with:
-          role-to-assume: arn:aws:iam::024209611402:role/github-action-role
+          role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
           aws-region: ap-south-1
 
       - name: Login to Amazon ECR
@@ -62,6 +62,7 @@ jobs:
           TAG: ${{ github.ref_name }}
         run: |
           docker build \
+            --build-arg GIT_SHA=${{ github.sha }} \
             -t $REGISTRY/$REPOSITORY:latest \
             -t $REGISTRY/$REPOSITORY:$TAG \
             ./backend
@@ -103,16 +104,78 @@ jobs:
           fi
           echo "Migration completed successfully"
 
-      - name: Deploy to ECS
+      - name: Deploy to ECS and verify rollout
+        # Bound the wait; the circuit breaker itself trips well before this.
+        timeout-minutes: 15
+        env:
+          CLUSTER: ${{ vars.AWS_RESOURCE_PREFIX }}-cluster
+          POLL_INTERVAL: "15"
         run: |
-          aws ecs update-service \
-            --cluster ${{ vars.AWS_RESOURCE_PREFIX }}-cluster \
-            --service ${{ vars.AWS_RESOURCE_PREFIX }}-service \
-            --task-definition ${{ vars.AWS_RESOURCE_PREFIX }}-task \
-            --force-new-deployment
-
-          aws ecs update-service \
-            --cluster ${{ vars.AWS_RESOURCE_PREFIX }}-cluster \
-            --service ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task \
-            --task-definition ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task \
-            --force-new-deployment
+          deploy_and_wait() {
+            SERVICE="$1"
+            FAMILY="$2"
+            echo "[$SERVICE] forcing new deployment on family $FAMILY"
+            aws ecs update-service \
+              --cluster "$CLUSTER" \
+              --service "$SERVICE" \
+              --task-definition "$FAMILY" \
+              --force-new-deployment >/dev/null
+
+            while true; do
+              STATE=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \
+                --query "services[0].deployments[?status=='PRIMARY'].rolloutState | [0]" \
+                --output text)
+              case "$STATE" in
+                COMPLETED)
+                  echo "[$SERVICE] rollout COMPLETED"
+                  return 0 ;;
+                FAILED)
+                  echo "::error::[$SERVICE] rollout FAILED — new tasks never became healthy (rolled back by circuit breaker)"
+                  return 1 ;;
+                *)
+                  echo "[$SERVICE] rollout $STATE — waiting ${POLL_INTERVAL}s"
+                  sleep "$POLL_INTERVAL" ;;
+              esac
+            done
+          }
+
+          deploy_and_wait "${{ vars.AWS_RESOURCE_PREFIX }}-service" "${{ vars.AWS_RESOURCE_PREFIX }}-task"
+          deploy_and_wait "${{ vars.AWS_RESOURCE_PREFIX }}-celery-task" "${{ vars.AWS_RESOURCE_PREFIX }}-celery-task"
+
+  notify:
+    needs: [verify-ci, build]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Discord
+        env:
+          DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          NAME: kaapi-production
+          RELEASE: ${{ github.ref_name }}
+          # true only when the build+deploy job succeeded.
+          OK: ${{ needs.build.result == 'success' }}
+        run: |
+          [ -z "$DISCORD_WEBHOOK_URL" ] && { echo "No webhook configured, skipping"; exit 0; }
+          if [ "$OK" = "true" ]; then
+            TITLE="🟢 $NAME deployment healthy"; COLOR=3066993   # green
+          else
+            TITLE="🔴 $NAME deployment failed";  COLOR=15158332  # red
+          fi
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+          payload=$(jq -n \
+            --arg title "$TITLE" \
+            --argjson color "$COLOR" \
+            --arg release "$RELEASE" \
+            --arg sha "$SHA_SHORT" \
+            --arg url "$RUN_URL" \
+            '{embeds: [{
+              title: $title, url: $url, color: $color,
+              fields: [
+                {name: "Release", value: $release, inline: true},
+                {name: "SHA", value: $sha, inline: true}
+              ],
+              timestamp: (now | todate)
+            }]}')
+          curl -sf -H "Content-Type: application/json" -X POST -d "$payload" "$DISCORD_WEBHOOK_URL" \
+            || echo "Discord notification failed to send"

.github/workflows/deploy-staging-ecs.yml

@@ -20,10 +20,9 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Configure AWS credentials
-        # More information on this action can be found below in the 'AWS Credentials' section
         uses: aws-actions/configure-aws-credentials@v6
         with:
-          role-to-assume: arn:aws:iam::024209611402:role/github-action-role
+          role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
           aws-region: ap-south-1
 
       - name: Login to Amazon ECR
@@ -36,7 +35,7 @@ jobs:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           REPOSITORY: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-repo
         run: |
-          docker build -t $REGISTRY/$REPOSITORY:latest ./backend
+          docker build --build-arg GIT_SHA=${{ github.sha }} -t $REGISTRY/$REPOSITORY:latest ./backend
           docker push $REGISTRY/$REPOSITORY:latest
 
       - name: Run database migrations

.github/workflows/deploy-staging.yml

@@ -85,3 +85,114 @@ jobs:
             --instance-id "$INSTANCE_ID" \
             --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
             --output json
+
+  ecs-rehearsal:
+    needs: deploy
+    runs-on: ubuntu-latest
+    environment: AWS_ENV_VARS
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v7
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
+          aws-region: ap-south-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build and push staging image
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-repo
+        run: |
+          docker build \
+            --build-arg GIT_SHA=${{ github.sha }} \
+            -t $REGISTRY/$REPOSITORY:latest \
+            ./backend
+          docker push $REGISTRY/$REPOSITORY:latest
+
+      - name: Scale staging ECS up and verify rollout
+        timeout-minutes: 15
+        env:
+          CLUSTER: kaapi-staging
+          SERVICES: "kaapi-staging-backend-celery kaapi-staging-celery-worker"
+          POLL_INTERVAL: "15"
+        run: |
+          for SERVICE in $SERVICES; do
+            aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \
+              --desired-count 1 --force-new-deployment >/dev/null
+            echo "[$SERVICE] scaled to 1"
+          done
+
+          for SERVICE in $SERVICES; do
+            echo "[$SERVICE] waiting for rollout"
+            while true; do
+              STATE=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \
+                --query "services[0].deployments[?status=='PRIMARY'].rolloutState | [0]" \
+                --output text)
+              case "$STATE" in
+                COMPLETED) echo "[$SERVICE] rehearsal rollout COMPLETED"; break ;;
+                FAILED)
+                  echo "::error::[$SERVICE] rehearsal FAILED — the production ECS deploy path is broken"
+                  exit 1 ;;
+                *) echo "[$SERVICE] rollout $STATE — waiting ${POLL_INTERVAL}s"; sleep "$POLL_INTERVAL" ;;
+              esac
+            done
+          done
+
+      - name: Scale staging ECS back to 0
+        if: always()
+        env:
+          CLUSTER: kaapi-staging
+          SERVICES: "kaapi-staging-backend-celery kaapi-staging-celery-worker"
+        run: |
+          for SERVICE in $SERVICES; do
+            aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \
+              --desired-count 0 >/dev/null
+            echo "[$SERVICE] scaled back to 0"
+          done
+
+  notify:
+    needs: [deploy, ecs-rehearsal]
+    if: ${{ always() && needs.deploy.result != 'skipped' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Discord
+        env:
+          DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          NAME: kaapi-staging
+          RELEASE: "#${{ github.run_number }}"
+          # true only when every deploy job succeeded.
+          OK: ${{ needs.deploy.result == 'success' && needs.ecs-rehearsal.result == 'success' }}
+        run: |
+          [ -z "$DISCORD_WEBHOOK_URL" ] && { echo "No webhook configured, skipping"; exit 0; }
+          if [ "$OK" = "true" ]; then
+            TITLE="🟢 $NAME deployment healthy"; COLOR=3066993   # green
+          else
+            TITLE="🔴 $NAME deployment failed";  COLOR=15158332  # red
+          fi
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+          payload=$(jq -n \
+            --arg title "$TITLE" \
+            --argjson color "$COLOR" \
+            --arg release "$RELEASE" \
+            --arg sha "$SHA_SHORT" \
+            --arg url "$RUN_URL" \
+            '{embeds: [{
+              title: $title, url: $url, color: $color,
+              fields: [
+                {name: "Release", value: $release, inline: true},
+                {name: "SHA", value: $sha, inline: true}
+              ],
+              timestamp: (now | todate)
+            }]}')
+          curl -sf -H "Content-Type: application/json" -X POST -d "$payload" "$DISCORD_WEBHOOK_URL" \
+            || echo "Discord notification failed to send"

backend/Dockerfile

@@ -44,6 +44,9 @@ COPY scripts /app/scripts
 COPY app /app/app
 COPY alembic.ini /app/alembic.ini
 
+ARG GIT_SHA=unknown
+ENV GIT_SHA=$GIT_SHA
+
 # Expose port 80
 EXPOSE 80
 

backend/app/core/config.py

@@ -46,6 +46,7 @@ class Settings(BaseSettings):
 
     PROJECT_NAME: str
     API_VERSION: str = "0.5.0"
+    GIT_SHA: str = "unknown"
     SENTRY_DSN: HttpUrl | None = None
     DISCORD_STATS_WEBHOOK_URL: HttpUrl | None = None
     POSTGRES_SERVER: str

backend/app/main.py

@@ -105,4 +105,5 @@ def custom_openapi():
 async def health() -> dict[str, str | float]:
     return {
         "status": "ok",
+        "sha": settings.GIT_SHA,
     }