diff --git a/.github/workflows/helper-apply.yaml b/.github/workflows/helper-apply.yaml new file mode 100644 index 0000000..6ca1459 --- /dev/null +++ b/.github/workflows/helper-apply.yaml @@ -0,0 +1,232 @@ +name: helper-apply + +on: + workflow_call: + inputs: + terraform-directory: + required: true + type: string + description: 'Path to terraform directory (e.g., terraform/base)' + environment: + required: true + type: string + description: 'GitHub environment for approval gate (e.g., production)' + download-plan: + required: false + type: boolean + default: true + description: 'Download and apply saved plan artifact from PR' + terraform-version: + required: false + type: string + default: '~1.13' + description: 'Terraform version to use' + pr-number: + required: false + type: string + default: '' + description: 'PR number to download plan from (auto-detected if not provided)' + outputs: + apply-exitcode: + description: 'Terraform apply exit code' + value: ${{ jobs.apply.outputs.exitcode }} + +jobs: + apply: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + outputs: + exitcode: ${{ steps.apply.outputs.exitcode }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ inputs.terraform-version }} + terraform_wrapper: true + + - name: Find PR Number + id: find-pr + if: inputs.download-plan && inputs.pr-number == '' + uses: actions/github-script@v7 + with: + script: | + // Find the PR that was just merged + const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: context.sha + }); + + const mergedPR = prs.find(pr => pr.merged_at !== null); + + if (mergedPR) { + console.log(`Found merged PR #${mergedPR.number}`); + core.setOutput('pr-number', mergedPR.number.toString()); + return mergedPR.number; + } else { + console.log('No merged PR found for this commit'); + core.setOutput('pr-number', ''); + return ''; + } + + - name: Download Plan Artifact + if: inputs.download-plan + id: download-plan + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: tfplan-${{ inputs.pr-number || steps.find-pr.outputs.pr-number || github.sha }} + path: ${{ inputs.terraform-directory }} + + - name: Terraform Init + working-directory: ${{ inputs.terraform-directory }} + env: + BACK_BLAZE_KEY_ID: ${{ secrets.ORG_BACK_BLAZE_KEY_ID }} + BACK_BLAZE_APPLICATION_KEY: ${{ secrets.ORG_BACK_BLAZE_APPLICATION_KEY }} + run: | + terraform init \ + -backend-config="access_key=${BACK_BLAZE_KEY_ID}" \ + -backend-config="secret_key=${BACK_BLAZE_APPLICATION_KEY}" + + - name: Backup State + id: backup-state + working-directory: ${{ inputs.terraform-directory }} + run: | + timestamp=$(date +%s) + terraform state pull > state-backup-${timestamp}.json + + if [ -s state-backup-${timestamp}.json ]; then + echo "āœ… State backed up successfully" + echo "backup_file=state-backup-${timestamp}.json" >> $GITHUB_OUTPUT + else + echo "āš ļø State backup is empty (this may be first apply)" + fi + + - name: Upload State Backup + if: steps.backup-state.outputs.backup_file != '' + uses: actions/upload-artifact@v4 + with: + name: terraform-state-backup-${{ github.sha }} + path: ${{ inputs.terraform-directory }}/${{ steps.backup-state.outputs.backup_file }} + retention-days: 90 + + - name: Terraform Apply (with saved plan) + id: apply-with-plan + if: steps.download-plan.outcome == 'success' && hashFiles(format('{0}/tfplan', inputs.terraform-directory)) != '' + working-directory: ${{ inputs.terraform-directory }} + run: | + echo "šŸ“¦ Applying saved plan from PR..." + terraform apply -no-color tfplan | tee apply-output.txt + exitcode=${PIPESTATUS[0]} + echo "exitcode=$exitcode" >> $GITHUB_OUTPUT + + if [ $exitcode -ne 0 ]; then + echo "::error::Terraform apply failed with exit code $exitcode" + exit 1 + fi + + echo "āœ… Apply completed successfully" + + - name: Terraform Apply (fresh plan) + id: apply-fresh + if: steps.apply-with-plan.outcome == 'skipped' + working-directory: ${{ inputs.terraform-directory }} + env: + TF_VAR_hetzner_token: ${{ secrets.ORG_HETZNER_TOKEN }} + TF_VAR_admin_email: ${{ vars.ADMIN_EMAIL || 'admin@ainsley.dev' }} + TF_VAR_environment: production + TF_VAR_project_name: ainsley-dev-platform + run: | + echo "āš ļø No saved plan found, generating fresh plan and applying..." + terraform plan -out=tfplan + terraform apply -no-color tfplan | tee apply-output.txt + exitcode=${PIPESTATUS[0]} + echo "exitcode=$exitcode" >> $GITHUB_OUTPUT + + if [ $exitcode -ne 0 ]; then + echo "::error::Terraform apply failed with exit code $exitcode" + exit 1 + fi + + echo "āœ… Apply completed successfully" + + - name: Set Combined Output + id: apply + run: | + if [ "${{ steps.apply-with-plan.outcome }}" == "success" ]; then + echo "exitcode=${{ steps.apply-with-plan.outputs.exitcode }}" >> $GITHUB_OUTPUT + elif [ "${{ steps.apply-fresh.outcome }}" == "success" ]; then + echo "exitcode=${{ steps.apply-fresh.outputs.exitcode }}" >> $GITHUB_OUTPUT + else + echo "exitcode=1" >> $GITHUB_OUTPUT + fi + + - name: Post Apply Results + if: always() && (steps.apply-with-plan.outcome != 'skipped' || steps.apply-fresh.outcome != 'skipped') + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const applyOutputPath = '${{ inputs.terraform-directory }}/apply-output.txt'; + + let body = '## šŸš€ Terraform Apply Results\n\n'; + + const success = '${{ steps.apply.outputs.exitcode }}' === '0'; + + if (success) { + body += 'āœ… **Infrastructure changes applied successfully**\n\n'; + } else { + body += 'āŒ **Apply failed - please check logs**\n\n'; + } + + // Add apply output if available + if (fs.existsSync(applyOutputPath)) { + const applyOutput = fs.readFileSync(applyOutputPath, 'utf8'); + body += '
\nšŸ“„ View Apply Output\n\n'; + body += '```\n' + applyOutput + '\n```\n'; + body += '
\n'; + } + + body += `\nšŸ”— [View Workflow Run](${context.payload.repository.html_url}/actions/runs/${context.runId})`; + + // Try to find and comment on the merged PR + const prNumber = '${{ inputs.pr-number || steps.find-pr.outputs.pr-number }}'; + + if (prNumber) { + try { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: parseInt(prNumber), + body: body + }); + console.log(`Posted results to PR #${prNumber}`); + } catch (error) { + console.log(`Could not post to PR #${prNumber}: ${error.message}`); + } + } else { + console.log('No PR found to comment on'); + } + + // Also create a commit comment + try { + await github.rest.repos.createCommitComment({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: context.sha, + body: body + }); + console.log('Posted results as commit comment'); + } catch (error) { + console.log(`Could not create commit comment: ${error.message}`); + } + + - name: Cleanup Plan Files + if: always() + working-directory: ${{ inputs.terraform-directory }} + run: | + rm -f tfplan tfplan.json apply-output.txt state-backup-*.json diff --git a/.github/workflows/helper-plan.yaml b/.github/workflows/helper-plan.yaml new file mode 100644 index 0000000..7a08846 --- /dev/null +++ b/.github/workflows/helper-plan.yaml @@ -0,0 +1,186 @@ +name: helper-plan + +on: + workflow_call: + inputs: + terraform-directory: + required: true + type: string + description: 'Path to terraform directory (e.g., terraform/base)' + upload-plan: + required: false + type: boolean + default: true + description: 'Upload plan artifact for later apply' + terraform-version: + required: false + type: string + default: '~1.13' + description: 'Terraform version to use' + outputs: + plan-exitcode: + description: 'Terraform plan exit code (0=no changes, 2=has changes)' + value: ${{ jobs.plan.outputs.exitcode }} + has-changes: + description: 'Whether plan has changes (true/false)' + value: ${{ jobs.plan.outputs.has_changes }} + has-destroys: + description: 'Whether plan contains resource deletions (true/false)' + value: ${{ jobs.plan.outputs.has_destroys }} + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + exitcode: ${{ steps.plan.outputs.exitcode }} + has_changes: ${{ steps.plan.outputs.exitcode == '2' }} + has_destroys: ${{ steps.check-destroys.outputs.has_destroys }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ inputs.terraform-version }} + terraform_wrapper: true + + - name: Terraform Init + working-directory: ${{ inputs.terraform-directory }} + env: + BACK_BLAZE_KEY_ID: ${{ secrets.ORG_BACK_BLAZE_KEY_ID }} + BACK_BLAZE_APPLICATION_KEY: ${{ secrets.ORG_BACK_BLAZE_APPLICATION_KEY }} + run: | + terraform init \ + -backend-config="access_key=${BACK_BLAZE_KEY_ID}" \ + -backend-config="secret_key=${BACK_BLAZE_APPLICATION_KEY}" + + - name: Terraform Plan + id: plan + working-directory: ${{ inputs.terraform-directory }} + env: + TF_VAR_hetzner_token: ${{ secrets.ORG_HETZNER_TOKEN }} + TF_VAR_admin_email: ${{ vars.ADMIN_EMAIL || 'admin@ainsley.dev' }} + TF_VAR_environment: production + TF_VAR_project_name: ainsley-dev-platform + run: | + set +e + terraform plan -detailed-exitcode -out=tfplan -no-color | tee plan-output.txt + exitcode=$? + echo "exitcode=$exitcode" >> $GITHUB_OUTPUT + + # Exit code: 0 (no changes), 1 (error), 2 (has changes) + if [ $exitcode -eq 1 ]; then + echo "::error::Terraform plan failed" + exit 1 + fi + + exit 0 + + - name: Generate Plan Summary + id: plan-summary + if: always() && steps.plan.outcome != 'skipped' + working-directory: ${{ inputs.terraform-directory }} + run: | + if [ -f plan-output.txt ]; then + # Extract resource changes (no heading - added by PR comment step) + if grep -q "No changes" plan-output.txt; then + echo "āœ… **No infrastructure changes detected**" > plan-summary.md + else + echo "šŸ“‹ **Changes detected:**" > plan-summary.md + echo "" >> plan-summary.md + + # Extract Plan: line + grep "Plan:" plan-output.txt | head -1 >> plan-summary.md || echo "Changes will be applied" >> plan-summary.md + echo "" >> plan-summary.md + fi + + echo "
" >> plan-summary.md + echo "šŸ“„ View Full Plan" >> plan-summary.md + echo "" >> plan-summary.md + echo '```hcl' >> plan-summary.md + cat plan-output.txt >> plan-summary.md + echo '```' >> plan-summary.md + echo "
" >> plan-summary.md + else + echo "āš ļø Plan output not found" > plan-summary.md + fi + + - name: Check for Destructive Changes + id: check-destroys + if: steps.plan.outputs.exitcode == '2' + working-directory: ${{ inputs.terraform-directory }} + run: | + # Convert plan to JSON and check for deletions + terraform show -json tfplan > tfplan.json + + has_destroys="false" + if jq -e '.resource_changes[]? | select(.change.actions[] == "delete")' tfplan.json > /dev/null 2>&1; then + has_destroys="true" + echo "::warning::āš ļø This plan contains DESTRUCTIVE changes (resource deletions)" + fi + + echo "has_destroys=$has_destroys" >> $GITHUB_OUTPUT + + - name: Post Plan to PR + if: github.event_name == 'pull_request' && steps.plan.outcome == 'success' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const planSummary = fs.readFileSync('${{ inputs.terraform-directory }}/plan-summary.md', 'utf8'); + + let header = '## Terraform Plan Results\n\n'; + + // Add warning for destroys + const hasDestroys = '${{ steps.check-destroys.outputs.has_destroys }}' === 'true'; + if (hasDestroys) { + header += '> āš ļø **WARNING: This plan contains DESTRUCTIVE changes (resource deletions)**\n\n'; + } + + // Add note about apply process + const footer = '\n\n---\nšŸ“ *To apply these changes: Merge this PR. Terraform will apply automatically after approval on the `production` environment.*'; + + const body = header + planSummary + footer; + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('šŸ—ļø Terraform Plan Results') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } + + - name: Upload Plan Artifact + if: inputs.upload-plan && steps.plan.outputs.exitcode == '2' + uses: actions/upload-artifact@v4 + with: + name: tfplan-${{ github.event.pull_request.number || github.sha }} + path: | + ${{ inputs.terraform-directory }}/tfplan + ${{ inputs.terraform-directory }}/tfplan.json + retention-days: 30 + if-no-files-found: warn diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f0ea49b..25dc12c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -182,6 +182,15 @@ jobs: docker compose -f "$compose_file" config > /dev/null done + terraform-plan: + name: Terraform Plan + uses: ./.github/workflows/helper-plan.yaml + with: + terraform-directory: terraform/base + upload-plan: true + terraform-version: '~1.13' + secrets: inherit + pr-validation-summary: name: PR Validation Summary runs-on: ubuntu-latest @@ -189,6 +198,7 @@ jobs: - terraform-validation - terraform-lint - terraform-security + - terraform-plan - ansible-validation - yaml-validation - docker-validation @@ -202,6 +212,7 @@ jobs: # This job will fail if any required job failed if [ "${{ needs.terraform-validation.result }}" != "success" ] || \ [ "${{ needs.terraform-lint.result }}" != "success" ] || \ + [ "${{ needs.terraform-plan.result }}" != "success" ] || \ [ "${{ needs.ansible-validation.result }}" != "success" ] || \ [ "${{ needs.yaml-validation.result }}" != "success" ] || \ [ "${{ needs.docker-validation.result }}" != "success" ]; then diff --git a/.github/workflows/terraform-apply.yaml b/.github/workflows/terraform-apply.yaml new file mode 100644 index 0000000..cd8e18b --- /dev/null +++ b/.github/workflows/terraform-apply.yaml @@ -0,0 +1,104 @@ +name: Terraform Apply + +on: + push: + branches: + - main + paths: + - 'terraform/**' + - '**.tf' + workflow_dispatch: + inputs: + terraform-directory: + description: 'Terraform directory to apply (e.g., terraform/base)' + required: false + default: 'terraform/base' + skip-plan-download: + description: 'Skip downloading PR plan artifact (generate fresh plan)' + required: false + type: boolean + default: false + +jobs: + detect-changes: + name: Detect Terraform Changes + runs-on: ubuntu-latest + outputs: + has-terraform-changes: ${{ steps.check.outputs.has_changes }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Check for Terraform changes + id: check + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + echo "Manual workflow dispatch - will proceed with apply" + exit 0 + fi + + # Check if terraform files changed in this push + if git diff --name-only HEAD^..HEAD | grep -E '^terraform/|\.tf$'; then + echo "has_changes=true" >> $GITHUB_OUTPUT + echo "Terraform changes detected" + else + echo "has_changes=false" >> $GITHUB_OUTPUT + echo "No terraform changes detected" + fi + + apply: + name: Apply Infrastructure Changes + needs: detect-changes + if: needs.detect-changes.outputs.has-terraform-changes == 'true' + uses: ./.github/workflows/helper-apply.yaml + with: + terraform-directory: ${{ inputs.terraform-directory || 'terraform/base' }} + environment: production + download-plan: ${{ !inputs.skip-plan-download }} + terraform-version: '~1.13' + secrets: inherit + + notify-completion: + name: Notify Completion + runs-on: ubuntu-latest + needs: [detect-changes, apply] + if: always() && needs.detect-changes.outputs.has-terraform-changes == 'true' + steps: + - name: Summary + run: | + echo "## šŸ—ļø Terraform Apply Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ needs.apply.result }}" == "success" ]; then + echo "āœ… **Infrastructure changes applied successfully**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Your infrastructure has been updated on the \`production\` environment." >> $GITHUB_STEP_SUMMARY + elif [ "${{ needs.apply.result }}" == "failure" ]; then + echo "āŒ **Apply failed**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Please check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details." >> $GITHUB_STEP_SUMMARY + elif [ "${{ needs.apply.result }}" == "cancelled" ]; then + echo "āš ļø **Apply was cancelled**" >> $GITHUB_STEP_SUMMARY + else + echo "āš ļø **Apply was skipped or had an unknown status**" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "šŸ”— [View commit](${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }})" >> $GITHUB_STEP_SUMMARY + + notify-skip: + name: Notify No Changes + runs-on: ubuntu-latest + needs: detect-changes + if: needs.detect-changes.outputs.has-terraform-changes == 'false' + steps: + - name: Summary + run: | + echo "## šŸ—ļø Terraform Apply Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "ā„¹ļø **No infrastructure changes detected**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "This push did not modify any Terraform files. Skipping apply." >> $GITHUB_STEP_SUMMARY diff --git a/README.md b/README.md index 8d5ed69..9b07692 100644 --- a/README.md +++ b/README.md @@ -71,9 +71,83 @@ make init # -backend-config="secret_key=${BACK_BLAZE_APPLICATION_KEY}" ``` -## Usage +## CI/CD Automation -### Deploy Uptime Kuma +This repository uses automated Terraform workflows for infrastructure changes: + +### Workflow Overview + +1. **Pull Request** - Terraform plan runs automatically + - Shows what infrastructure changes will be made + - Posts plan output as PR comment + - Detects destructive changes (deletions) and warns + - Validates Terraform syntax and security (tfsec, Checkov) + - Saves plan artifact for exact apply on merge + +2. **Merge to Main** - Terraform apply runs with approval gate + - Downloads saved plan from PR (ensures plan-apply match) + - Requires approval on `production` environment before applying + - Backs up state before making changes + - Posts results back to merged PR and commit + - Only runs if Terraform files changed + +### Reusable Workflows + +The repository includes reusable workflow helpers: + +- **`.github/workflows/helper-plan.yaml`** - Reusable plan workflow + - Runs terraform plan + - Posts formatted results to PR + - Uploads plan artifact + - Detects destructive changes + +- **`.github/workflows/helper-apply.yaml`** - Reusable apply workflow + - Downloads plan artifact from PR + - Backs up state before apply + - Applies infrastructure changes + - Requires GitHub environment approval + +### Secrets Configuration + +Workflows use GitHub organization secrets (already configured): + +- `ORG_HETZNER_TOKEN` - Hetzner Cloud API token +- `ORG_BACK_BLAZE_KEY_ID` - Backblaze B2 access key for state backend +- `ORG_BACK_BLAZE_APPLICATION_KEY` - Backblaze B2 secret key + +### GitHub Environment Setup + +The `production` environment must be configured with: + +1. Go to **Settings → Environments → New environment** +2. Name: `production` +3. Add protection rules: + - āœ… Required reviewers (select yourself) + - āœ… Deployment branches: `main` only + - Optional: Wait timer (0-30 minutes) + +### Making Infrastructure Changes + +#### Automated (Recommended) + +```bash +# 1. Create a feature branch +git checkout -b feat/add-new-service + +# 2. Make your Terraform changes +vim terraform/services/new-service/main.tf + +# 3. Push and create PR +git push origin feat/add-new-service +gh pr create --title "Add new service" + +# 4. Review plan output in PR comment +# 5. Get PR approval, then merge +# 6. Approve the deploy in GitHub Actions (production environment) +# 7. Changes applied automatically! +``` + +#### Manual (Emergency/Testing) ```bash # Plan deployment @@ -86,6 +160,15 @@ make apply make output ``` +### Emergency Procedures + +If the automated workflow fails: + +1. **Check the workflow logs**: Actions tab → Failed workflow → View logs +2. **Manual apply via workflow_dispatch**: Actions → Terraform Apply → Run workflow +3. **Local manual apply**: Use `make apply` with local credentials +4. **State recovery**: Download backup from workflow artifacts (retained 90 days) + ## Ansible Roles This project references Ansible roles from the [WebKit](https://github.com/ainsleydev/webkit) diff --git a/terraform.tfvars.example b/terraform.tfvars.example index df1c545..898ebf7 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -1,6 +1,13 @@ # Terraform Variables for Ainsley Dev Platform # Copy this file to terraform.tfvars and update with your actual values # Note: terraform.tfvars is gitignored for security +# +# šŸ¤– CI/CD Note: +# GitHub Actions workflows use organization secrets instead of this file: +# - ORG_HETZNER_TOKEN (Hetzner API token) +# - ORG_BACK_BLAZE_KEY_ID (Backend state storage) +# - ORG_BACK_BLAZE_APPLICATION_KEY (Backend state storage) +# This file is only needed for local Terraform operations. # Hetzner Cloud API Token # Get from: https://console.hetzner.cloud/ -> Project -> Security -> API Tokens diff --git a/terraform/services/uptime-kuma/main.tf b/terraform/services/uptime-kuma/main.tf index cc4f2cb..8659f57 100644 --- a/terraform/services/uptime-kuma/main.tf +++ b/terraform/services/uptime-kuma/main.tf @@ -36,6 +36,8 @@ module "server" { # Hetzner Volume (Webkit) # # Uses persistent data to save the Uptime Kuma Data. +# Note: lifecycle blocks cannot be used on modules, only resources. +# Data protection is provided by state backups and approval gates in CI/CD. module "volume" { source = "github.com/ainsleydev/webkit//platform/terraform/providers/hetzner/volume?ref=main" @@ -55,7 +57,6 @@ resource "local_file" "ansible_inventory" { filename = "${path.root}/../../ansible/inventory-uptime-kuma.ini" content = <<-EOT # Auto-generated by Terraform - DO NOT EDIT MANUALLY - # Last updated: ${timestamp()} [uptime_kuma] ${var.domain} ansible_host=${module.server.ip_address} ansible_user=${module.server.server_user}