diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index d094405ad..23ddb4495 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -46,9 +46,9 @@ jobs: uses: actions/checkout@v7 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v6 # More information on this action can be found below in the 'AWS Credentials' section + uses: aws-actions/configure-aws-credentials@v6 with: - role-to-assume: arn:aws:iam::024209611402:role/github-action-role + role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} aws-region: ap-south-1 - name: Login to Amazon ECR @@ -62,6 +62,7 @@ jobs: TAG: ${{ github.ref_name }} run: | docker build \ + --build-arg GIT_SHA=${{ github.sha }} \ -t $REGISTRY/$REPOSITORY:latest \ -t $REGISTRY/$REPOSITORY:$TAG \ ./backend @@ -103,16 +104,78 @@ jobs: fi echo "Migration completed successfully" - - name: Deploy to ECS + - name: Deploy to ECS and verify rollout + # Bound the wait; the circuit breaker itself trips well before this. + timeout-minutes: 15 + env: + CLUSTER: ${{ vars.AWS_RESOURCE_PREFIX }}-cluster + POLL_INTERVAL: "15" run: | - aws ecs update-service \ - --cluster ${{ vars.AWS_RESOURCE_PREFIX }}-cluster \ - --service ${{ vars.AWS_RESOURCE_PREFIX }}-service \ - --task-definition ${{ vars.AWS_RESOURCE_PREFIX }}-task \ - --force-new-deployment - - aws ecs update-service \ - --cluster ${{ vars.AWS_RESOURCE_PREFIX }}-cluster \ - --service ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task \ - --task-definition ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task \ - --force-new-deployment + deploy_and_wait() { + SERVICE="$1" + FAMILY="$2" + echo "[$SERVICE] forcing new deployment on family $FAMILY" + aws ecs update-service \ + --cluster "$CLUSTER" \ + --service "$SERVICE" \ + --task-definition "$FAMILY" \ + --force-new-deployment >/dev/null + + while true; do + STATE=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \ + --query "services[0].deployments[?status=='PRIMARY'].rolloutState | [0]" \ + --output text) + case "$STATE" in + COMPLETED) + echo "[$SERVICE] rollout COMPLETED" + return 0 ;; + FAILED) + echo "::error::[$SERVICE] rollout FAILED — new tasks never became healthy (rolled back by circuit breaker)" + return 1 ;; + *) + echo "[$SERVICE] rollout $STATE — waiting ${POLL_INTERVAL}s" + sleep "$POLL_INTERVAL" ;; + esac + done + } + + deploy_and_wait "${{ vars.AWS_RESOURCE_PREFIX }}-service" "${{ vars.AWS_RESOURCE_PREFIX }}-task" + deploy_and_wait "${{ vars.AWS_RESOURCE_PREFIX }}-celery-task" "${{ vars.AWS_RESOURCE_PREFIX }}-celery-task" + + notify: + needs: [verify-ci, build] + if: always() + runs-on: ubuntu-latest + steps: + - name: Notify Discord + env: + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + NAME: kaapi-production + RELEASE: ${{ github.ref_name }} + # true only when the build+deploy job succeeded. + OK: ${{ needs.build.result == 'success' }} + run: | + [ -z "$DISCORD_WEBHOOK_URL" ] && { echo "No webhook configured, skipping"; exit 0; } + if [ "$OK" = "true" ]; then + TITLE="🟢 $NAME deployment healthy"; COLOR=3066993 # green + else + TITLE="🔴 $NAME deployment failed"; COLOR=15158332 # red + fi + SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7) + payload=$(jq -n \ + --arg title "$TITLE" \ + --argjson color "$COLOR" \ + --arg release "$RELEASE" \ + --arg sha "$SHA_SHORT" \ + --arg url "$RUN_URL" \ + '{embeds: [{ + title: $title, url: $url, color: $color, + fields: [ + {name: "Release", value: $release, inline: true}, + {name: "SHA", value: $sha, inline: true} + ], + timestamp: (now | todate) + }]}') + curl -sf -H "Content-Type: application/json" -X POST -d "$payload" "$DISCORD_WEBHOOK_URL" \ + || echo "Discord notification failed to send" diff --git a/.github/workflows/deploy-staging-ecs.yml b/.github/workflows/deploy-staging-ecs.yml index fffcbb714..12dbeb76d 100644 --- a/.github/workflows/deploy-staging-ecs.yml +++ b/.github/workflows/deploy-staging-ecs.yml @@ -20,10 +20,9 @@ jobs: uses: actions/checkout@v7 - name: Configure AWS credentials - # More information on this action can be found below in the 'AWS Credentials' section uses: aws-actions/configure-aws-credentials@v6 with: - role-to-assume: arn:aws:iam::024209611402:role/github-action-role + role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} aws-region: ap-south-1 - name: Login to Amazon ECR @@ -36,7 +35,7 @@ jobs: REGISTRY: ${{ steps.login-ecr.outputs.registry }} REPOSITORY: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-repo run: | - docker build -t $REGISTRY/$REPOSITORY:latest ./backend + docker build --build-arg GIT_SHA=${{ github.sha }} -t $REGISTRY/$REPOSITORY:latest ./backend docker push $REGISTRY/$REPOSITORY:latest - name: Run database migrations diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 39c788eef..7d7cafd2b 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -85,3 +85,114 @@ jobs: --instance-id "$INSTANCE_ID" \ --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ --output json + + ecs-rehearsal: + needs: deploy + runs-on: ubuntu-latest + environment: AWS_ENV_VARS + permissions: + id-token: write + contents: read + steps: + - name: Checkout the repo + uses: actions/checkout@v7 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} + aws-region: ap-south-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build and push staging image + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-repo + run: | + docker build \ + --build-arg GIT_SHA=${{ github.sha }} \ + -t $REGISTRY/$REPOSITORY:latest \ + ./backend + docker push $REGISTRY/$REPOSITORY:latest + + - name: Scale staging ECS up and verify rollout + timeout-minutes: 15 + env: + CLUSTER: kaapi-staging + SERVICES: "kaapi-staging-backend-celery kaapi-staging-celery-worker" + POLL_INTERVAL: "15" + run: | + for SERVICE in $SERVICES; do + aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \ + --desired-count 1 --force-new-deployment >/dev/null + echo "[$SERVICE] scaled to 1" + done + + for SERVICE in $SERVICES; do + echo "[$SERVICE] waiting for rollout" + while true; do + STATE=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \ + --query "services[0].deployments[?status=='PRIMARY'].rolloutState | [0]" \ + --output text) + case "$STATE" in + COMPLETED) echo "[$SERVICE] rehearsal rollout COMPLETED"; break ;; + FAILED) + echo "::error::[$SERVICE] rehearsal FAILED — the production ECS deploy path is broken" + exit 1 ;; + *) echo "[$SERVICE] rollout $STATE — waiting ${POLL_INTERVAL}s"; sleep "$POLL_INTERVAL" ;; + esac + done + done + + - name: Scale staging ECS back to 0 + if: always() + env: + CLUSTER: kaapi-staging + SERVICES: "kaapi-staging-backend-celery kaapi-staging-celery-worker" + run: | + for SERVICE in $SERVICES; do + aws ecs update-service --cluster "$CLUSTER" --service "$SERVICE" \ + --desired-count 0 >/dev/null + echo "[$SERVICE] scaled back to 0" + done + + notify: + needs: [deploy, ecs-rehearsal] + if: ${{ always() && needs.deploy.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Notify Discord + env: + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + NAME: kaapi-staging + RELEASE: "#${{ github.run_number }}" + # true only when every deploy job succeeded. + OK: ${{ needs.deploy.result == 'success' && needs.ecs-rehearsal.result == 'success' }} + run: | + [ -z "$DISCORD_WEBHOOK_URL" ] && { echo "No webhook configured, skipping"; exit 0; } + if [ "$OK" = "true" ]; then + TITLE="🟢 $NAME deployment healthy"; COLOR=3066993 # green + else + TITLE="🔴 $NAME deployment failed"; COLOR=15158332 # red + fi + SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7) + payload=$(jq -n \ + --arg title "$TITLE" \ + --argjson color "$COLOR" \ + --arg release "$RELEASE" \ + --arg sha "$SHA_SHORT" \ + --arg url "$RUN_URL" \ + '{embeds: [{ + title: $title, url: $url, color: $color, + fields: [ + {name: "Release", value: $release, inline: true}, + {name: "SHA", value: $sha, inline: true} + ], + timestamp: (now | todate) + }]}') + curl -sf -H "Content-Type: application/json" -X POST -d "$payload" "$DISCORD_WEBHOOK_URL" \ + || echo "Discord notification failed to send" diff --git a/backend/Dockerfile b/backend/Dockerfile index f34b22b36..7bc1453d4 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -44,6 +44,9 @@ COPY scripts /app/scripts COPY app /app/app COPY alembic.ini /app/alembic.ini +ARG GIT_SHA=unknown +ENV GIT_SHA=$GIT_SHA + # Expose port 80 EXPOSE 80 diff --git a/backend/app/core/config.py b/backend/app/core/config.py index cc105fbec..cb1f3badf 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -46,6 +46,7 @@ class Settings(BaseSettings): PROJECT_NAME: str API_VERSION: str = "0.5.0" + GIT_SHA: str = "unknown" SENTRY_DSN: HttpUrl | None = None DISCORD_STATS_WEBHOOK_URL: HttpUrl | None = None POSTGRES_SERVER: str diff --git a/backend/app/main.py b/backend/app/main.py index 5d2a09cf0..0c1c3f66b 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -105,4 +105,5 @@ def custom_openapi(): async def health() -> dict[str, str | float]: return { "status": "ok", + "sha": settings.GIT_SHA, }