OpenTofuPlayground/.gitea/workflows/deploy-proxmox.yml

name: Deploy / Update Proxmox Dev VMs

on:
  push:
    branches:
      - main
    paths:
      - "proxmox/apps/*.tfvars"

env:
  TOFU_VERSION: "1.9.0"
  TOFU_WORKING_DIR: "proxmox"

jobs:
  detect-changes:
    name: Detect changed Proxmox tfvars
    runs-on: ubuntu-latest
    outputs:
      added_modified: ${{ steps.diff.outputs.added_modified }}
      deleted: ${{ steps.diff.outputs.deleted }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Compute diff
        id: diff
        run: |
          ADDED_MODIFIED=$(git diff --name-only --diff-filter=ACM HEAD~1 HEAD -- 'proxmox/apps/*.tfvars' | jq -R -s -c 'split("\n") | map(select(length > 0))')
          DELETED=$(git diff --name-only --diff-filter=D HEAD~1 HEAD -- 'proxmox/apps/*.tfvars' | jq -R -s -c 'split("\n") | map(select(length > 0))')
          echo "added_modified=$ADDED_MODIFIED" >> "$GITHUB_OUTPUT"
          echo "deleted=$DELETED"               >> "$GITHUB_OUTPUT"

  # ─── Provision / Update VM ────────────────────────────────────────────────
  provision:
    name: Provision ${{ matrix.tfvars }}
    needs: detect-changes
    if: ${{ needs.detect-changes.outputs.added_modified != '[]' }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        tfvars: ${{ fromJson(needs.detect-changes.outputs.added_modified) }}

    steps:
      - uses: actions/checkout@v4

      - name: Install OpenTofu
        run: |
          curl -fsSL https://get.opentofu.org/install-opentofu.sh | sh -s -- --install-method standalone --opentofu-version ${{ env.TOFU_VERSION }}

      - name: Install gopass and openssh-client
        run: |
          sudo apt-get install -y -qq openssh-client
          curl -fsSL https://github.com/gopasspw/gopass/releases/latest/download/gopass-linux-amd64.tar.gz | tar xz
          sudo mv gopass /usr/local/bin/

      - name: Configure gopass
        env:
          GOPASS_GPG_KEY:    ${{ secrets.GOPASS_GPG_KEY }}
          GOPASS_STORE_REPO: ${{ secrets.GOPASS_STORE_REPO }}
        run: |
          echo "$GOPASS_GPG_KEY" | gpg --batch --import
          gopass clone "$GOPASS_STORE_REPO"

      - name: Resolve developer name from tfvars filename
        id: dev
        run: |
          DEV=$(basename "${{ matrix.tfvars }}" .tfvars)
          echo "name=$DEV" >> "$GITHUB_OUTPUT"

      - name: tofu init
        working-directory: ${{ env.TOFU_WORKING_DIR }}
        env:
          AWS_ACCESS_KEY_ID:     ${{ secrets.SEAWEED_ACCESS_KEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }}
        run: |
          tofu init \
            -backend-config="bucket=tofu-state" \
            -backend-config="key=apps-proxmox/${{ steps.dev.outputs.name }}.tfstate" \
            -backend-config="endpoint=${{ secrets.SEAWEED_S3_ENDPOINT }}" \
            -backend-config="region=us-east-1" \
            -backend-config="force_path_style=true"

      - name: Select or create workspace
        working-directory: ${{ env.TOFU_WORKING_DIR }}
        run: |
          tofu workspace select "${{ steps.dev.outputs.name }}" \
            || tofu workspace new "${{ steps.dev.outputs.name }}"

      - name: tofu apply (provision VM)
        working-directory: ${{ env.TOFU_WORKING_DIR }}
        env:
          AWS_ACCESS_KEY_ID:     ${{ secrets.SEAWEED_ACCESS_KEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }}
        run: |
          tofu apply -auto-approve \
            -var-file="../${{ matrix.tfvars }}" \
            -var="proxmox_endpoint=${{ secrets.PROXMOX_ENDPOINT }}" \
            -var="proxmox_api_token=${{ secrets.PROXMOX_API_TOKEN }}" \
            -var="proxmox_tls_insecure=${{ secrets.PROXMOX_TLS_INSECURE || 'false' }}"

      - name: Read VM outputs
        id: vm
        working-directory: ${{ env.TOFU_WORKING_DIR }}
        env:
          AWS_ACCESS_KEY_ID:     ${{ secrets.SEAWEED_ACCESS_KEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }}
        run: |
          VM_IP=$(tofu output -raw vm_ip)
          VM_ROLE=$(tofu output -raw vm_role)
          KEY_GEN=$(tofu output -raw key_was_generated)
          CI_USER=$(tofu output -raw cloud_init_user)
          echo "ip=$VM_IP"           >> "$GITHUB_OUTPUT"
          echo "role=$VM_ROLE"       >> "$GITHUB_OUTPUT"
          echo "key_generated=$KEY_GEN" >> "$GITHUB_OUTPUT"
          echo "ci_user=$CI_USER"    >> "$GITHUB_OUTPUT"
          echo "VM IP: $VM_IP  Role: $VM_ROLE  Key generated: $KEY_GEN"

      # Store auto-generated SSH key in gopass so subsequent app deploy pipelines
      # can retrieve it via infra/ssh-keys/<devname> (same convention as Docker stack).
      - name: Store generated SSH key in gopass
        if: steps.vm.outputs.key_generated == 'true'
        working-directory: ${{ env.TOFU_WORKING_DIR }}
        env:
          AWS_ACCESS_KEY_ID:     ${{ secrets.SEAWEED_ACCESS_KEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }}
        run: |
          SSH_KEY=$(tofu output -raw ssh_private_key)
          echo "::add-mask::$SSH_KEY"
          echo "$SSH_KEY" | gopass insert -f "infra/ssh-keys/${{ steps.dev.outputs.name }}"
          echo "SSH private key stored in gopass at infra/ssh-keys/${{ steps.dev.outputs.name }}"

      # For k3s VMs: SSH in, wait for k3s to be active, fetch the kubeconfig,
      # replace the loopback address with the actual VM IP, and store in gopass.
      - name: Fetch and store k3s kubeconfig
        if: steps.vm.outputs.role == 'k3s'
        run: |
          DEV="${{ steps.dev.outputs.name }}"
          VM_IP="${{ steps.vm.outputs.ip }}"
          CI_USER="${{ steps.vm.outputs.ci_user }}"

          gopass show -o "infra/ssh-keys/$DEV" > /tmp/deploy_key
          chmod 600 /tmp/deploy_key

          echo "Waiting for k3s to become active on $VM_IP..."
          for i in $(seq 1 30); do
            if ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
                   -i /tmp/deploy_key "$CI_USER@$VM_IP" \
                   "systemctl is-active --quiet k3s" 2>/dev/null; then
              echo "k3s is active after $i attempt(s)."
              break
            fi
            if [ "$i" -eq 30 ]; then
              echo "ERROR: k3s did not become active within 5 minutes."
              exit 1
            fi
            echo "  Attempt $i/30 — retrying in 10s..."
            sleep 10
          done

          ssh -o StrictHostKeyChecking=no -i /tmp/deploy_key "$CI_USER@$VM_IP" \
            "cat /etc/rancher/k3s/k3s.yaml" \
            | sed "s|127.0.0.1|$VM_IP|g" \
            | gopass insert -f "infra/kubeconfigs/$DEV"

          rm -f /tmp/deploy_key
          echo "kubeconfig stored in gopass at infra/kubeconfigs/$DEV"

      # ─── Chained deploy: apply linked QA app tfvars if present ─────────────
      # Convention: apps/<devname>-qa.tfvars for Docker, k8s/apps/<devname>-qa.tfvars for k3s.
      # The -qa suffix makes QA environments discoverable without extra config.

      - name: Check for linked Docker QA app tfvars
        id: linked_docker
        if: steps.vm.outputs.role == 'docker'
        run: |
          LINKED="apps/${{ steps.dev.outputs.name }}-qa.tfvars"
          if [ -f "$LINKED" ]; then
            echo "found=true"   >> "$GITHUB_OUTPUT"
            echo "path=$LINKED" >> "$GITHUB_OUTPUT"
            echo "Linked Docker QA app found: $LINKED"
          else
            echo "found=false" >> "$GITHUB_OUTPUT"
            echo "No linked Docker QA app at $LINKED — skipping chained deploy."
          fi

      - name: Check for linked K8s QA app tfvars
        id: linked_k8s
        if: steps.vm.outputs.role == 'k3s'
        run: |
          LINKED="k8s/apps/${{ steps.dev.outputs.name }}-qa.tfvars"
          if [ -f "$LINKED" ]; then
            echo "found=true"   >> "$GITHUB_OUTPUT"
            echo "path=$LINKED" >> "$GITHUB_OUTPUT"
            echo "Linked K8s QA app found: $LINKED"
          else
            echo "found=false" >> "$GITHUB_OUTPUT"
            echo "No linked K8s QA app at $LINKED — skipping chained deploy."
          fi

      - name: Chained Docker deploy
        if: steps.vm.outputs.role == 'docker' && steps.linked_docker.outputs.found == 'true'
        env:
          AWS_ACCESS_KEY_ID:     ${{ secrets.SEAWEED_ACCESS_KEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }}
        run: |
          DEV="${{ steps.dev.outputs.name }}"
          LINKED="${{ steps.linked_docker.outputs.path }}"
          APP=$(basename "$LINKED" .tfvars)
          VM_IP="${{ steps.vm.outputs.ip }}"

          echo "── Chained Docker deploy: $APP → $VM_IP ──"

          gopass show -o "infra/ssh-keys/$DEV" > /tmp/deploy_key
          chmod 600 /tmp/deploy_key

          DB_PASSWORD=$(gopass show -o "apps/$APP/db_password" 2>/dev/null || echo "")
          GIT_TOKEN=$(gopass show -o "apps/$APP/git_token" 2>/dev/null || echo "")
          echo "::add-mask::$DB_PASSWORD"
          echo "::add-mask::$GIT_TOKEN"

          tofu init \
            -backend-config="bucket=tofu-state" \
            -backend-config="key=apps/$APP.tfstate" \
            -backend-config="endpoint=${{ secrets.SEAWEED_S3_ENDPOINT }}" \
            -backend-config="region=us-east-1" \
            -backend-config="force_path_style=true"

          tofu workspace select "$APP" || tofu workspace new "$APP"

          tofu apply -auto-approve \
            -var-file="$LINKED" \
            -var="ssh_host=$VM_IP" \
            -var="ssh_key_path=/tmp/deploy_key" \
            -var="db_password=$DB_PASSWORD" \
            -var="openresty_git_token=$GIT_TOKEN"

          rm -f /tmp/deploy_key
          echo "Chained Docker deploy complete: $APP"

      - name: Chained K8s deploy
        if: steps.vm.outputs.role == 'k3s' && steps.linked_k8s.outputs.found == 'true'
        env:
          AWS_ACCESS_KEY_ID:     ${{ secrets.SEAWEED_ACCESS_KEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }}
        run: |
          DEV="${{ steps.dev.outputs.name }}"
          LINKED="${{ steps.linked_k8s.outputs.path }}"
          APP=$(basename "$LINKED" .tfvars)

          echo "── Chained K8s deploy: $APP ──"

          gopass show -o "infra/kubeconfigs/$DEV" > /tmp/kubeconfig
          chmod 600 /tmp/kubeconfig

          RABBITMQ_PASSWORD=$(gopass show -o "apps/$APP/rabbitmq_password" 2>/dev/null || echo "")
          LOKI_AUTH_TOKEN=$(gopass show -o "apps/$APP/loki_token" 2>/dev/null || echo "")
          echo "::add-mask::$RABBITMQ_PASSWORD"
          echo "::add-mask::$LOKI_AUTH_TOKEN"

          cd k8s
          tofu init \
            -backend-config="bucket=tofu-state" \
            -backend-config="key=apps-k8s/$APP.tfstate" \
            -backend-config="endpoint=${{ secrets.SEAWEED_S3_ENDPOINT }}" \
            -backend-config="region=us-east-1" \
            -backend-config="force_path_style=true"

          tofu workspace select "$APP" || tofu workspace new "$APP"

          tofu apply -auto-approve \
            -var-file="../$LINKED" \
            -var="kubeconfig_path=/tmp/kubeconfig" \
            -var="rabbitmq_password=$RABBITMQ_PASSWORD" \
            -var="loki_auth_token=$LOKI_AUTH_TOKEN"

          rm -f /tmp/kubeconfig
          echo "Chained K8s deploy complete: $APP"

      - name: Cleanup
        if: always()
        run: rm -f /tmp/deploy_key /tmp/kubeconfig

  # ─── Destroy VM (tfvars file deleted) ─────────────────────────────────────
  destroy:
    name: Destroy ${{ matrix.tfvars }}
    needs: detect-changes
    if: ${{ needs.detect-changes.outputs.deleted != '[]' }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        tfvars: ${{ fromJson(needs.detect-changes.outputs.deleted) }}

    steps:
      - uses: actions/checkout@v4
        with:
          # Check out the previous commit so the deleted tfvars file is present.
          ref: ${{ github.event.before }}

      - name: Install OpenTofu
        run: |
          curl -fsSL https://get.opentofu.org/install-opentofu.sh | sh -s -- --install-method standalone --opentofu-version ${{ env.TOFU_VERSION }}

      - name: Install gopass
        run: |
          curl -fsSL https://github.com/gopasspw/gopass/releases/latest/download/gopass-linux-amd64.tar.gz | tar xz
          sudo mv gopass /usr/local/bin/

      - name: Configure gopass
        env:
          GOPASS_GPG_KEY:    ${{ secrets.GOPASS_GPG_KEY }}
          GOPASS_STORE_REPO: ${{ secrets.GOPASS_STORE_REPO }}
        run: |
          echo "$GOPASS_GPG_KEY" | gpg --batch --import
          gopass clone "$GOPASS_STORE_REPO"

      - name: Resolve developer name
        id: dev
        run: |
          DEV=$(basename "${{ matrix.tfvars }}" .tfvars)
          echo "name=$DEV" >> "$GITHUB_OUTPUT"

      - name: tofu init
        working-directory: ${{ env.TOFU_WORKING_DIR }}
        env:
          AWS_ACCESS_KEY_ID:     ${{ secrets.SEAWEED_ACCESS_KEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }}
        run: |
          tofu init \
            -backend-config="bucket=tofu-state" \
            -backend-config="key=apps-proxmox/${{ steps.dev.outputs.name }}.tfstate" \
            -backend-config="endpoint=${{ secrets.SEAWEED_S3_ENDPOINT }}" \
            -backend-config="region=us-east-1" \
            -backend-config="force_path_style=true"

      - name: Select workspace
        working-directory: ${{ env.TOFU_WORKING_DIR }}
        run: tofu workspace select "${{ steps.dev.outputs.name }}"

      - name: tofu destroy
        working-directory: ${{ env.TOFU_WORKING_DIR }}
        env:
          AWS_ACCESS_KEY_ID:     ${{ secrets.SEAWEED_ACCESS_KEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }}
        run: |
          tofu destroy -auto-approve \
            -var-file="../${{ matrix.tfvars }}" \
            -var="proxmox_endpoint=${{ secrets.PROXMOX_ENDPOINT }}" \
            -var="proxmox_api_token=${{ secrets.PROXMOX_API_TOKEN }}" \
            -var="proxmox_tls_insecure=${{ secrets.PROXMOX_TLS_INSECURE || 'false' }}"

      - name: Delete workspace
        working-directory: ${{ env.TOFU_WORKING_DIR }}
        run: |
          tofu workspace select default
          tofu workspace delete "${{ steps.dev.outputs.name }}"

      - name: Remove credentials from gopass
        run: |
          DEV="${{ steps.dev.outputs.name }}"
          gopass rm -f "infra/ssh-keys/$DEV"    2>/dev/null || true
          gopass rm -f "infra/kubeconfigs/$DEV" 2>/dev/null || true
          echo "Removed gopass entries for developer '$DEV'."