feat: overhaul Docker workflow and architecture with dynamic LLMs, vision API, and reusable tooling

.github/workflows/build-and-push-docker-image.yaml

@@ -1,43 +1,93 @@
-name: Crews-Control Docker Image
+name: Build and Push Crews-Control Docker Image
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - 'main'
 
 jobs:
-  crews-control-docker-image:
-    runs-on: [self-hosted-common-strong]
-    environment: crews-control
+  build-and-push:
+    runs-on: ubuntu-latest
+    environment: crews-control 
+    permissions:
+      contents: read
+      packages: write
+
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Free up disk space on the runner
+        uses: jlumbroso/free-disk-space@main
         with:
-          path: 'crews-control-action'
-          ref: 'main'
+          # This is the key. We don't touch the tool-cache where Python lives.
+          tool-cache: false
+          # These are large directories that are safe to remove for this project.
+          android: true
+          dotnet: true
+          haskell: true
+          # See the action's documentation for other options
 
-      - name: Get the commit SHA of Crews-Control
-        run: echo "CREWS_CONTROL_SHA=$(git -C crews-control-action rev-parse HEAD)" >> $GITHUB_ENV
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12.3'
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # aws-actions/configure-aws-credentials@v1
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
         with:
-          aws-access-key-id: ${{ secrets.ARTIFACTS_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.ARTIFACTS_AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ secrets.ECR_AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@261a7de32bda11ba01f4d75c4ed6caf3739e54be # aws-actions/amazon-ecr-login@v1
-
-      - name: Build, tag and push the image to ECR
-        id: build-push
-        working-directory: 'crews-control-action'
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: ${{ secrets.CREWS_CONTROL_ACTION_IMAGE_NAME }}
+          toolchain: stable
+
+      - name: Compile requirements with a stable and compatible toolset
         run: |
-          echo "Building the image"
+          # Pin all core packaging tools to a known-good, compatible set
+          python -m pip install --upgrade "pip==24.0" "setuptools" "wheel" "pip-tools==7.4.1"
+
+          echo "---"
+          echo "DEBUG: Checking installed tool versions"
+          pip --version
+          pip-compile --version
+          echo "---"
+
+          # Run the compilation, which will now use a stable environment
+          echo "Compiling requirements.in..."
+          pip-compile --generate-hashes --verbose --no-strip-extras requirements.in
 
-          docker build --no-cache \
-            -t "$ECR_REGISTRY/$ECR_REPOSITORY:${CREWS_CONTROL_SHA}" \
-            .
-          echo "Pushing image to ECR"
-          docker push "$ECR_REGISTRY/$ECR_REPOSITORY:${CREWS_CONTROL_SHA}"
+          echo "Compiling requirements-dev.in..."
+          pip-compile --generate-hashes --verbose --no-strip-extras requirements-dev.in
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ghcr.io/${{ github.repository }}
+          tags: |
+            type=sha,prefix=
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/run-crews-control-project.yaml

@@ -0,0 +1,122 @@
+name: Run Crews-Control Project
+
+on:
+  workflow_dispatch:
+    inputs:
+      project_name:
+        description: 'The name of the project to run (e.g., pr-security-review).'
+        required: true
+      project_source:
+        description: 'Source of the project files (execution.yaml, context/, etc.).'
+        type: choice
+        options:
+          - repository # Use files from this Git repo (for testing changes)
+          - image      # Use files already inside the Docker image
+        default: 'repository'
+      image_tag:
+        description: 'The Docker image tag to pull from Docker Hub.'
+        required: true
+        default: 'latest'
+      docker_image:
+        description: 'The full name of the Docker image.'
+        required: true
+        default: 'ghcr.io/avri-schneider/crews-control'
+      run_params:
+        description: 'Optional: Command-line parameters for the run (e.g., key1=value1 key2="value 2").'
+        required: false
+
+jobs:
+  run-crews-control-project:
+    runs-on: ubuntu-latest
+    environment: crews-control
+    permissions:
+      contents: read
+      packages: read
+
+    steps:
+      - name: Checkout repository files (if running from repository)
+        if: ${{ inputs.project_source == 'repository' }}
+        uses: actions/checkout@v4
+
+      - name: Create .env file from secrets
+        run: |
+          # This step securely creates the .env file needed by the Docker container
+          echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> .env
+          echo "JIRA_API_TOKEN=${{ secrets.JIRA_API_TOKEN }}" >> .env
+          echo "JIRA_USERNAME=${{ vars.JIRA_USERNAME }}" >> .env
+          echo "JIRA_INSTANCE_URL=${{ vars.JIRA_INSTANCE_URL }}" >> .env
+          echo "JIRA_CREATE_ISSUE_PROJECT_KEY=${{ vars.JIRA_CREATE_ISSUE_PROJECT_KEY }}" >> .env
+          echo "JIRA_CREATE_ISSUE_TYPE=${{ vars.JIRA_CREATE_ISSUE_TYPE }}" >> .env
+          echo "JIRA_LINK_ALLOWED_PAIRS=${{ vars.JIRA_LINK_ALLOWED_PAIRS }}" >> .env
+          echo "JIRA_ATTACH_ALLOWED_PREFIXES=${{ vars.JIRA_ATTACH_ALLOWED_PREFIXES }}" >> .env
+          echo "JIRA_REASSIGN_ALLOWED_PREFIXES=${{ vars.JIRA_REASSIGN_ALLOWED_PREFIXES }}" >> .env
+          echo "JIRA_SETPRIORITY_ALLOWED_PREFIXES=${{ vars.JIRA_SETPRIORITY_ALLOWED_PREFIXES }}" >> .env
+          echo "AZURE_API_KEY=${{ vars.AZURE_API_KEY }}" >> .env
+          echo "AZURE_API_BASE=${{ vars.AZURE_API_BASE }}" >> .env
+          echo "AZURE_API_VERSION=${{ vars.AZURE_API_VERSION }}" >> .env
+          echo "AZURE_OPENAI_VISION_DEPLOYMENT=${{ vars.AZURE_OPENAI_VISION_DEPLOYMENT }}" >> .env
+          echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .env
+          echo "OPENAI_API_VERSION=${{ vars.OPENAI_API_VERSION }}" >> .env
+          echo "OPENAI_MODEL_NAME=${{ vars.OPENAI_MODEL_NAME }}" >> .env
+          echo "OPENAI_EMBEDDING_MODEL_NAME=${{ vars.OPENAI_EMBEDDING_MODEL_NAME }}" >> .env
+          echo "OPENAI_VISION_MODEL=${{ vars.OPENAI_VISION_MODEL }}" >> .env
+          echo "LLM_NAME=${{ vars.LLM_NAME }}" >> .env
+          echo "EMBEDDER_NAME=${{ vars.EMBEDDER_NAME }}" >> .env
+          echo "CONFLUENCE_ENDPOINT=${{ vars.CONFLUENCE_ENDPOINT }}" >> .env
+          echo "CONFLUENCE_API_USER=${{ vars.CONFLUENCE_API_USER }}" >> .env
+          echo "CONFLUENCE_API_TOKEN=${{ secrets.CONFLUENCE_API_TOKEN }}" >> .env
+          echo "CONFLUENCE_SPACE=${{ vars.CONFLUENCE_SPACE }}" >> .env
+          # Add any other secrets from your .env.example here
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # - name: Log in to Docker Hub
+      #   uses: docker/login-action@v3
+      #   with:
+      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
+      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Pull Docker image
+        run: docker pull ${{ inputs.docker_image }}:${{ inputs.image_tag }}
+
+      - name: Run Project from Repository Source
+        if: ${{ inputs.project_source == 'repository' }}
+        run: |
+          docker run --rm \
+            -e HOST_USER_ID=$(id -u) \
+            -e HOST_GROUP_ID=$(id -g) \
+            --env-file .env \
+            -v ${{ github.workspace }}/projects/${{ inputs.project_name }}:/app/projects/${{ inputs.project_name }} \
+            ${{ inputs.docker_image }}:${{ inputs.image_tag }} \
+            --project-name ${{ inputs.project_name }} \
+            --params ${{ inputs.run_params }}
+
+      - name: Run Project from Image Source
+        if: ${{ inputs.project_source == 'image' }}
+        run: |
+          # Create a directory on the host to capture the output artifacts
+          mkdir -p ${{ github.workspace }}/outputs/${{ inputs.project_name }}
+
+          docker run --rm \
+            -e HOST_USER_ID=$(id -u) \
+            -e HOST_GROUP_ID=$(id -g) \
+            --env-file .env \
+            -v ${{ github.workspace }}/outputs/${{ inputs.project_name }}:/app/projects/${{ inputs.project_name }}/output \
+            ${{ inputs.docker_image }}:${{ inputs.image_tag }} \
+            --project-name ${{ inputs.project_name }} \
+            --params ${{ inputs.run_params }}
+
+      - name: Upload Project Artifacts
+        if: always() # Always run this step to capture logs and outputs even if the run fails
+        uses: actions/upload-artifact@v4
+        with:
+          name: output-${{ inputs.project_name }}-${{ github.run_id }}
+          path: |
+            ${{ github.workspace }}/projects/${{ inputs.project_name }}/output/
+            ${{ github.workspace }}/outputs/${{ inputs.project_name }}/
+          if-no-files-found: ignore

Dockerfile

@@ -1,65 +1,42 @@
-# Stage 1: Build and compile everything in a full Python image
+# Stage 1: Build stage for installing dependencies
 FROM python:3.12.3 AS build
 
-# Set the working directory in the container
 WORKDIR /app
 
-# Copy only the requirements file, to cache the installed packages layer
 COPY requirements.txt /app/
 
-# Install system dependencies for building packages (if any needed)
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    libssl-dev \
-    libffi-dev \
-    python3-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# Upgrade pip and install dependencies with retries
 RUN pip install --upgrade pip setuptools \
-    && pip install --require-hashes --no-cache-dir -r requirements.txt --verbose
-# Suggested retry mechanism for pip install (commented out)
-# RUN pip install --upgrade pip setuptools && \
-#     pip install --require-hashes --no-cache-dir -r requirements.txt || \
-#     pip install --require-hashes --no-cache-dir -r requirements.txt
-
-
-# Invalidate cache from here onwards when needed
-ARG CACHEBUSTER=1
+    && pip install --require-hashes --no-cache-dir -r requirements.txt --verbose \
+    && rm -rf /root/.cache/pip
 
-# Create a non-root user 'appuser' and switch to it
-RUN groupadd appuser && \
-    useradd -m -g appuser appuser
-
-USER appuser
-
-# Copy the current directory contents into the container at /app
 COPY . /app
 
-# Stage 2: Create a slim image for running the application
+
+# Stage 2: Final slim image for production
 FROM python:3.12.3-slim
 
-# Copy user and group data
-COPY --from=build /etc/passwd /etc/passwd
-COPY --from=build /etc/group /etc/group
+# Install gosu, a lightweight tool for switching users, then clean up.
+RUN apt-get update && apt-get install -y --no-install-recommends gosu \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-# Copy installed Python packages from build stage
-COPY --from=build /usr/local/lib/python3.12 /usr/local/lib/python3.12
+WORKDIR /app
 
-# Ensure scripts in /usr/local/bin are available
+# Copy installed dependencies and application code
+COPY --from=build /usr/local/lib/python3.12 /usr/local/lib/python3.12
 COPY --from=build /usr/local/bin /usr/local/bin
-
-# Copy application code and other necessary files from build stage
 COPY --from=build /app /app
 
-# Ensure the appuser owns the necessary directories
-RUN mkdir -p /home/appuser && \
-    chown -R appuser:appuser /home/appuser && \
+# Create a generic appuser with a standard home directory
+RUN useradd --create-home --shell /bin/bash appuser && \
     chown -R appuser:appuser /app
 
-# Set the working directory and user
-WORKDIR /app
-USER appuser
+# Copy the entrypoint script and make it executable
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+# Combine the entrypoint script and the main command.
+ENTRYPOINT ["entrypoint.sh", "python", "-u", "crews_control.py"]
 
-ENTRYPOINT [ "python", "-u", "crews_control.py" ]
-CMD []
+# The default command is now empty, as the main command is in the ENTRYPOINT.
+CMD []