From b2a3e0045238b7f7086e00a9a5a41072843bd15a Mon Sep 17 00:00:00 2001 From: Lutz Finsterle Date: Fri, 20 Mar 2026 10:06:05 +0100 Subject: [PATCH] New Function: DEV VM --- .gitea/workflows/deploy-proxmox.yml | 365 ++++++++++++++ .gitlab-ci.yml | 25 +- .../workflows/deploy-proxmox.gitlab-ci.yml | 300 ++++++++++++ Makefile | 15 +- README.md | 463 ++++++++++++++++-- .../vm-proxmox/cloud-init-docker.yaml.tftpl | 18 + modules/vm-proxmox/cloud-init-k3s.yaml.tftpl | 18 + modules/vm-proxmox/main.tf | 119 +++++ modules/vm-proxmox/outputs.tf | 30 ++ modules/vm-proxmox/variables.tf | 90 ++++ proxmox/apps/example-developer.tfvars | 49 ++ proxmox/backend.tf | 8 + proxmox/main.tf | 70 +++ proxmox/tests/proxmox_validation.tftest.hcl | 101 ++++ proxmox/variables.tf | 104 ++++ 15 files changed, 1733 insertions(+), 42 deletions(-) create mode 100644 .gitea/workflows/deploy-proxmox.yml create mode 100644 .gitlab/workflows/deploy-proxmox.gitlab-ci.yml create mode 100644 modules/vm-proxmox/cloud-init-docker.yaml.tftpl create mode 100644 modules/vm-proxmox/cloud-init-k3s.yaml.tftpl create mode 100644 modules/vm-proxmox/main.tf create mode 100644 modules/vm-proxmox/outputs.tf create mode 100644 modules/vm-proxmox/variables.tf create mode 100644 proxmox/apps/example-developer.tfvars create mode 100644 proxmox/backend.tf create mode 100644 proxmox/main.tf create mode 100644 proxmox/tests/proxmox_validation.tftest.hcl create mode 100644 proxmox/variables.tf diff --git a/.gitea/workflows/deploy-proxmox.yml b/.gitea/workflows/deploy-proxmox.yml new file mode 100644 index 0000000..6a191ae --- /dev/null +++ b/.gitea/workflows/deploy-proxmox.yml @@ -0,0 +1,365 @@ +name: Deploy / Update Proxmox Dev VMs + +on: + push: + branches: + - main + paths: + - "proxmox/apps/*.tfvars" + +env: + TOFU_VERSION: "1.9.0" + TOFU_WORKING_DIR: "proxmox" + +jobs: + detect-changes: + name: Detect changed Proxmox tfvars + runs-on: ubuntu-latest + outputs: + added_modified: ${{ steps.diff.outputs.added_modified }} + deleted: ${{ steps.diff.outputs.deleted }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Compute diff + id: diff + run: | + ADDED_MODIFIED=$(git diff --name-only --diff-filter=ACM HEAD~1 HEAD -- 'proxmox/apps/*.tfvars' | jq -R -s -c 'split("\n") | map(select(length > 0))') + DELETED=$(git diff --name-only --diff-filter=D HEAD~1 HEAD -- 'proxmox/apps/*.tfvars' | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "added_modified=$ADDED_MODIFIED" >> "$GITHUB_OUTPUT" + echo "deleted=$DELETED" >> "$GITHUB_OUTPUT" + + # ─── Provision / Update VM ──────────────────────────────────────────────── + provision: + name: Provision ${{ matrix.tfvars }} + needs: detect-changes + if: ${{ needs.detect-changes.outputs.added_modified != '[]' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + tfvars: ${{ fromJson(needs.detect-changes.outputs.added_modified) }} + + steps: + - uses: actions/checkout@v4 + + - name: Install OpenTofu + run: | + curl -fsSL https://get.opentofu.org/install-opentofu.sh | sh -s -- --install-method standalone --opentofu-version ${{ env.TOFU_VERSION }} + + - name: Install gopass and openssh-client + run: | + sudo apt-get install -y -qq openssh-client + curl -fsSL https://github.com/gopasspw/gopass/releases/latest/download/gopass-linux-amd64.tar.gz | tar xz + sudo mv gopass /usr/local/bin/ + + - name: Configure gopass + env: + GOPASS_GPG_KEY: ${{ secrets.GOPASS_GPG_KEY }} + GOPASS_STORE_REPO: ${{ secrets.GOPASS_STORE_REPO }} + run: | + echo "$GOPASS_GPG_KEY" | gpg --batch --import + gopass clone "$GOPASS_STORE_REPO" + + - name: Resolve developer name from tfvars filename + id: dev + run: | + DEV=$(basename "${{ matrix.tfvars }}" .tfvars) + echo "name=$DEV" >> "$GITHUB_OUTPUT" + + - name: tofu init + working-directory: ${{ env.TOFU_WORKING_DIR }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.SEAWEED_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }} + run: | + tofu init \ + -backend-config="bucket=tofu-state" \ + -backend-config="key=apps-proxmox/${{ steps.dev.outputs.name }}.tfstate" \ + -backend-config="endpoint=${{ secrets.SEAWEED_S3_ENDPOINT }}" \ + -backend-config="region=us-east-1" \ + -backend-config="force_path_style=true" + + - name: Select or create workspace + working-directory: ${{ env.TOFU_WORKING_DIR }} + run: | + tofu workspace select "${{ steps.dev.outputs.name }}" \ + || tofu workspace new "${{ steps.dev.outputs.name }}" + + - name: tofu apply (provision VM) + working-directory: ${{ env.TOFU_WORKING_DIR }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.SEAWEED_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }} + run: | + tofu apply -auto-approve \ + -var-file="../${{ matrix.tfvars }}" \ + -var="proxmox_endpoint=${{ secrets.PROXMOX_ENDPOINT }}" \ + -var="proxmox_api_token=${{ secrets.PROXMOX_API_TOKEN }}" \ + -var="proxmox_tls_insecure=${{ secrets.PROXMOX_TLS_INSECURE || 'false' }}" + + - name: Read VM outputs + id: vm + working-directory: ${{ env.TOFU_WORKING_DIR }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.SEAWEED_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }} + run: | + VM_IP=$(tofu output -raw vm_ip) + VM_ROLE=$(tofu output -raw vm_role) + KEY_GEN=$(tofu output -raw key_was_generated) + CI_USER=$(tofu output -raw cloud_init_user) + echo "ip=$VM_IP" >> "$GITHUB_OUTPUT" + echo "role=$VM_ROLE" >> "$GITHUB_OUTPUT" + echo "key_generated=$KEY_GEN" >> "$GITHUB_OUTPUT" + echo "ci_user=$CI_USER" >> "$GITHUB_OUTPUT" + echo "VM IP: $VM_IP Role: $VM_ROLE Key generated: $KEY_GEN" + + # Store auto-generated SSH key in gopass so subsequent app deploy pipelines + # can retrieve it via infra/ssh-keys/ (same convention as Docker stack). + - name: Store generated SSH key in gopass + if: steps.vm.outputs.key_generated == 'true' + working-directory: ${{ env.TOFU_WORKING_DIR }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.SEAWEED_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }} + run: | + SSH_KEY=$(tofu output -raw ssh_private_key) + echo "::add-mask::$SSH_KEY" + echo "$SSH_KEY" | gopass insert -f "infra/ssh-keys/${{ steps.dev.outputs.name }}" + echo "SSH private key stored in gopass at infra/ssh-keys/${{ steps.dev.outputs.name }}" + + # For k3s VMs: SSH in, wait for k3s to be active, fetch the kubeconfig, + # replace the loopback address with the actual VM IP, and store in gopass. + - name: Fetch and store k3s kubeconfig + if: steps.vm.outputs.role == 'k3s' + run: | + DEV="${{ steps.dev.outputs.name }}" + VM_IP="${{ steps.vm.outputs.ip }}" + CI_USER="${{ steps.vm.outputs.ci_user }}" + + gopass show -o "infra/ssh-keys/$DEV" > /tmp/deploy_key + chmod 600 /tmp/deploy_key + + echo "Waiting for k3s to become active on $VM_IP..." + for i in $(seq 1 30); do + if ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + -i /tmp/deploy_key "$CI_USER@$VM_IP" \ + "systemctl is-active --quiet k3s" 2>/dev/null; then + echo "k3s is active after $i attempt(s)." + break + fi + if [ "$i" -eq 30 ]; then + echo "ERROR: k3s did not become active within 5 minutes." + exit 1 + fi + echo " Attempt $i/30 — retrying in 10s..." + sleep 10 + done + + ssh -o StrictHostKeyChecking=no -i /tmp/deploy_key "$CI_USER@$VM_IP" \ + "cat /etc/rancher/k3s/k3s.yaml" \ + | sed "s|127.0.0.1|$VM_IP|g" \ + | gopass insert -f "infra/kubeconfigs/$DEV" + + rm -f /tmp/deploy_key + echo "kubeconfig stored in gopass at infra/kubeconfigs/$DEV" + + # ─── Chained deploy: apply linked QA app tfvars if present ───────────── + # Convention: apps/-qa.tfvars for Docker, k8s/apps/-qa.tfvars for k3s. + # The -qa suffix makes QA environments discoverable without extra config. + + - name: Check for linked Docker QA app tfvars + id: linked_docker + if: steps.vm.outputs.role == 'docker' + run: | + LINKED="apps/${{ steps.dev.outputs.name }}-qa.tfvars" + if [ -f "$LINKED" ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + echo "path=$LINKED" >> "$GITHUB_OUTPUT" + echo "Linked Docker QA app found: $LINKED" + else + echo "found=false" >> "$GITHUB_OUTPUT" + echo "No linked Docker QA app at $LINKED — skipping chained deploy." + fi + + - name: Check for linked K8s QA app tfvars + id: linked_k8s + if: steps.vm.outputs.role == 'k3s' + run: | + LINKED="k8s/apps/${{ steps.dev.outputs.name }}-qa.tfvars" + if [ -f "$LINKED" ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + echo "path=$LINKED" >> "$GITHUB_OUTPUT" + echo "Linked K8s QA app found: $LINKED" + else + echo "found=false" >> "$GITHUB_OUTPUT" + echo "No linked K8s QA app at $LINKED — skipping chained deploy." + fi + + - name: Chained Docker deploy + if: steps.vm.outputs.role == 'docker' && steps.linked_docker.outputs.found == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.SEAWEED_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }} + run: | + DEV="${{ steps.dev.outputs.name }}" + LINKED="${{ steps.linked_docker.outputs.path }}" + APP=$(basename "$LINKED" .tfvars) + VM_IP="${{ steps.vm.outputs.ip }}" + + echo "── Chained Docker deploy: $APP → $VM_IP ──" + + gopass show -o "infra/ssh-keys/$DEV" > /tmp/deploy_key + chmod 600 /tmp/deploy_key + + DB_PASSWORD=$(gopass show -o "apps/$APP/db_password" 2>/dev/null || echo "") + GIT_TOKEN=$(gopass show -o "apps/$APP/git_token" 2>/dev/null || echo "") + echo "::add-mask::$DB_PASSWORD" + echo "::add-mask::$GIT_TOKEN" + + tofu init \ + -backend-config="bucket=tofu-state" \ + -backend-config="key=apps/$APP.tfstate" \ + -backend-config="endpoint=${{ secrets.SEAWEED_S3_ENDPOINT }}" \ + -backend-config="region=us-east-1" \ + -backend-config="force_path_style=true" + + tofu workspace select "$APP" || tofu workspace new "$APP" + + tofu apply -auto-approve \ + -var-file="$LINKED" \ + -var="ssh_host=$VM_IP" \ + -var="ssh_key_path=/tmp/deploy_key" \ + -var="db_password=$DB_PASSWORD" \ + -var="openresty_git_token=$GIT_TOKEN" + + rm -f /tmp/deploy_key + echo "Chained Docker deploy complete: $APP" + + - name: Chained K8s deploy + if: steps.vm.outputs.role == 'k3s' && steps.linked_k8s.outputs.found == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.SEAWEED_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }} + run: | + DEV="${{ steps.dev.outputs.name }}" + LINKED="${{ steps.linked_k8s.outputs.path }}" + APP=$(basename "$LINKED" .tfvars) + + echo "── Chained K8s deploy: $APP ──" + + gopass show -o "infra/kubeconfigs/$DEV" > /tmp/kubeconfig + chmod 600 /tmp/kubeconfig + + RABBITMQ_PASSWORD=$(gopass show -o "apps/$APP/rabbitmq_password" 2>/dev/null || echo "") + LOKI_AUTH_TOKEN=$(gopass show -o "apps/$APP/loki_token" 2>/dev/null || echo "") + echo "::add-mask::$RABBITMQ_PASSWORD" + echo "::add-mask::$LOKI_AUTH_TOKEN" + + cd k8s + tofu init \ + -backend-config="bucket=tofu-state" \ + -backend-config="key=apps-k8s/$APP.tfstate" \ + -backend-config="endpoint=${{ secrets.SEAWEED_S3_ENDPOINT }}" \ + -backend-config="region=us-east-1" \ + -backend-config="force_path_style=true" + + tofu workspace select "$APP" || tofu workspace new "$APP" + + tofu apply -auto-approve \ + -var-file="../$LINKED" \ + -var="kubeconfig_path=/tmp/kubeconfig" \ + -var="rabbitmq_password=$RABBITMQ_PASSWORD" \ + -var="loki_auth_token=$LOKI_AUTH_TOKEN" + + rm -f /tmp/kubeconfig + echo "Chained K8s deploy complete: $APP" + + - name: Cleanup + if: always() + run: rm -f /tmp/deploy_key /tmp/kubeconfig + + # ─── Destroy VM (tfvars file deleted) ───────────────────────────────────── + destroy: + name: Destroy ${{ matrix.tfvars }} + needs: detect-changes + if: ${{ needs.detect-changes.outputs.deleted != '[]' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + tfvars: ${{ fromJson(needs.detect-changes.outputs.deleted) }} + + steps: + - uses: actions/checkout@v4 + with: + # Check out the previous commit so the deleted tfvars file is present. + ref: ${{ github.event.before }} + + - name: Install OpenTofu + run: | + curl -fsSL https://get.opentofu.org/install-opentofu.sh | sh -s -- --install-method standalone --opentofu-version ${{ env.TOFU_VERSION }} + + - name: Install gopass + run: | + curl -fsSL https://github.com/gopasspw/gopass/releases/latest/download/gopass-linux-amd64.tar.gz | tar xz + sudo mv gopass /usr/local/bin/ + + - name: Configure gopass + env: + GOPASS_GPG_KEY: ${{ secrets.GOPASS_GPG_KEY }} + GOPASS_STORE_REPO: ${{ secrets.GOPASS_STORE_REPO }} + run: | + echo "$GOPASS_GPG_KEY" | gpg --batch --import + gopass clone "$GOPASS_STORE_REPO" + + - name: Resolve developer name + id: dev + run: | + DEV=$(basename "${{ matrix.tfvars }}" .tfvars) + echo "name=$DEV" >> "$GITHUB_OUTPUT" + + - name: tofu init + working-directory: ${{ env.TOFU_WORKING_DIR }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.SEAWEED_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }} + run: | + tofu init \ + -backend-config="bucket=tofu-state" \ + -backend-config="key=apps-proxmox/${{ steps.dev.outputs.name }}.tfstate" \ + -backend-config="endpoint=${{ secrets.SEAWEED_S3_ENDPOINT }}" \ + -backend-config="region=us-east-1" \ + -backend-config="force_path_style=true" + + - name: Select workspace + working-directory: ${{ env.TOFU_WORKING_DIR }} + run: tofu workspace select "${{ steps.dev.outputs.name }}" + + - name: tofu destroy + working-directory: ${{ env.TOFU_WORKING_DIR }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.SEAWEED_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SEAWEED_SECRET_KEY }} + run: | + tofu destroy -auto-approve \ + -var-file="../${{ matrix.tfvars }}" \ + -var="proxmox_endpoint=${{ secrets.PROXMOX_ENDPOINT }}" \ + -var="proxmox_api_token=${{ secrets.PROXMOX_API_TOKEN }}" \ + -var="proxmox_tls_insecure=${{ secrets.PROXMOX_TLS_INSECURE || 'false' }}" + + - name: Delete workspace + working-directory: ${{ env.TOFU_WORKING_DIR }} + run: | + tofu workspace select default + tofu workspace delete "${{ steps.dev.outputs.name }}" + + - name: Remove credentials from gopass + run: | + DEV="${{ steps.dev.outputs.name }}" + gopass rm -f "infra/ssh-keys/$DEV" 2>/dev/null || true + gopass rm -f "infra/kubeconfigs/$DEV" 2>/dev/null || true + echo "Removed gopass entries for developer '$DEV'." diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d980db..8ee3df1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,11 @@ # ───────────────────────────────────────────────────────────────────────────── # GitLab CI/CD — OpenTofu Playground # -# Three included pipelines: -# test.gitlab-ci.yml Static analysis + unit tests + K8s integration -# deploy.gitlab-ci.yml Docker stack deploy/destroy via apps/*.tfvars -# deploy-k8s.gitlab-ci.yml K8s stack deploy/destroy via k8s/apps/*.tfvars +# Four included pipelines: +# test.gitlab-ci.yml Static analysis + unit tests + K8s integration +# deploy.gitlab-ci.yml Docker stack deploy/destroy via apps/*.tfvars +# deploy-k8s.gitlab-ci.yml K8s stack deploy/destroy via k8s/apps/*.tfvars +# deploy-proxmox.gitlab-ci.yml Proxmox dev VM provision/destroy via proxmox/apps/*.tfvars # # Required GitLab CI/CD variables (Settings > CI/CD > Variables): # TOFU_VERSION OpenTofu version to install (default: 1.9.0) @@ -13,14 +14,17 @@ # SEAWEED_S3_ENDPOINT SeaweedFS S3 endpoint, e.g. http://seaweedfs.home:8333 # SEAWEED_ACCESS_KEY SeaweedFS access key (mark as Masked) # SEAWEED_SECRET_KEY SeaweedFS secret key (mark as Masked) +# PROXMOX_ENDPOINT Proxmox API URL, e.g. https://proxmox.home:8006/ +# PROXMOX_API_TOKEN Proxmox API token: user@pam!tokenid=secret (Masked) +# PROXMOX_TLS_INSECURE "true" when using a self-signed cert (optional) # # gopass store layout expected by the deploy pipelines: -# infra/ssh-keys/ — SSH private key for Docker host -# infra/kubeconfigs/ — kubeconfig for K8s cluster -# apps//db_password — PostgreSQL password -# apps//git_token — optional git token for git_clone mode -# apps//rabbitmq_password — RabbitMQ password (K8s stack) -# apps//loki_token — optional Loki auth token (K8s stack) +# infra/ssh-keys/ — SSH private key (Docker host or dev VM) +# infra/kubeconfigs/ — kubeconfig for K8s cluster or k3s VM +# apps//db_password — PostgreSQL password +# apps//git_token — optional git token for git_clone mode +# apps//rabbitmq_password — RabbitMQ password (K8s stack) +# apps//loki_token — optional Loki auth token (K8s stack) # ───────────────────────────────────────────────────────────────────────────── stages: @@ -37,3 +41,4 @@ include: - local: .gitlab/workflows/test.gitlab-ci.yml - local: .gitlab/workflows/deploy.gitlab-ci.yml - local: .gitlab/workflows/deploy-k8s.gitlab-ci.yml + - local: .gitlab/workflows/deploy-proxmox.gitlab-ci.yml diff --git a/.gitlab/workflows/deploy-proxmox.gitlab-ci.yml b/.gitlab/workflows/deploy-proxmox.gitlab-ci.yml new file mode 100644 index 0000000..06c2078 --- /dev/null +++ b/.gitlab/workflows/deploy-proxmox.gitlab-ci.yml @@ -0,0 +1,300 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Proxmox dev VM deploy pipeline — mirrors .gitea/workflows/deploy-proxmox.yml +# +# Triggered on pushes to main when files under proxmox/apps/*.tfvars change. +# Two jobs run in sequence: +# provision-proxmox — loops over added/modified tfvars, provisions/updates VMs, +# stores credentials in gopass, runs chained app deploy +# when apps/-qa.tfvars or k8s/apps/-qa.tfvars exists +# destroy-proxmox — loops over deleted tfvars, destroys VMs, removes gopass entries +# +# Required GitLab CI/CD variables (in addition to the shared ones): +# PROXMOX_ENDPOINT — https://proxmox.example.com:8006/ +# PROXMOX_API_TOKEN — user@pam!tokenid=xxxxxxxx-... (mark as Masked) +# PROXMOX_TLS_INSECURE — "true" when using a self-signed cert (optional) +# +# State key pattern: apps-proxmox/.tfstate +# gopass paths written by this pipeline: +# infra/ssh-keys/ — SSH private key (if auto-generated) +# infra/kubeconfigs/ — k3s kubeconfig (if vm_role = "k3s") +# ───────────────────────────────────────────────────────────────────────────── + +.proxmox_base: + image: ubuntu:22.04 + variables: + GIT_DEPTH: "0" + before_script: + - apt-get update -qq && apt-get install -y -qq curl git jq gnupg openssh-client + - | + curl -fsSL https://get.opentofu.org/install-opentofu.sh \ + | sh -s -- --install-method=standalone --opentofu-version=$TOFU_VERSION + - | + curl -fsSL https://github.com/gopasspw/gopass/releases/latest/download/gopass-linux-amd64.tar.gz \ + | tar xz + mv gopass /usr/local/bin/ + - | + echo "$GOPASS_GPG_KEY" | gpg --batch --import + gopass clone "$GOPASS_STORE_REPO" + +# ─── Provision / Update ─────────────────────────────────────────────────────── + +provision-proxmox: + extends: .proxmox_base + stage: deploy + rules: + - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' + changes: + - proxmox/apps/*.tfvars + script: + - | + ADDED_MODIFIED=$(git diff --name-only --diff-filter=ACM \ + "$CI_COMMIT_BEFORE_SHA" "$CI_COMMIT_SHA" -- 'proxmox/apps/*.tfvars') + + if [ -z "$ADDED_MODIFIED" ]; then + echo "No Proxmox tfvars files added or modified. Nothing to provision." + exit 0 + fi + + echo "Files to provision: $ADDED_MODIFIED" + FAILED=0 + + for TFVARS in $ADDED_MODIFIED; do + DEV=$(basename "$TFVARS" .tfvars) + echo "" + echo "══════════════════════════════════════════" + echo " Provisioning VM: $DEV" + echo "══════════════════════════════════════════" + + if ! ( + set -e + + export AWS_ACCESS_KEY_ID="$SEAWEED_ACCESS_KEY" + export AWS_SECRET_ACCESS_KEY="$SEAWEED_SECRET_KEY" + TLS_INSECURE="${PROXMOX_TLS_INSECURE:-false}" + + # ── tofu init with SeaweedFS backend ── + cd proxmox + tofu init \ + -backend-config="bucket=tofu-state" \ + -backend-config="key=apps-proxmox/$DEV.tfstate" \ + -backend-config="endpoint=$SEAWEED_S3_ENDPOINT" \ + -backend-config="region=us-east-1" \ + -backend-config="force_path_style=true" \ + -reconfigure + + tofu workspace select "$DEV" 2>/dev/null || tofu workspace new "$DEV" + + tofu apply -auto-approve \ + -var-file="../$TFVARS" \ + -var="proxmox_endpoint=$PROXMOX_ENDPOINT" \ + -var="proxmox_api_token=$PROXMOX_API_TOKEN" \ + -var="proxmox_tls_insecure=$TLS_INSECURE" + + # ── Read VM outputs ── + VM_IP=$(tofu output -raw vm_ip) + VM_ROLE=$(tofu output -raw vm_role) + KEY_GEN=$(tofu output -raw key_was_generated) + CI_USER=$(tofu output -raw cloud_init_user) + echo " VM IP: $VM_IP Role: $VM_ROLE Key generated: $KEY_GEN" + cd .. + + # ── Store auto-generated SSH key in gopass ── + if [ "$KEY_GEN" = "true" ]; then + cd proxmox + SSH_KEY=$(tofu output -raw ssh_private_key) + echo "$SSH_KEY" | gopass insert -f "infra/ssh-keys/$DEV" + echo " SSH key stored in gopass at infra/ssh-keys/$DEV" + cd .. + fi + + # ── For k3s: wait for k3s and fetch kubeconfig ── + if [ "$VM_ROLE" = "k3s" ]; then + gopass show -o "infra/ssh-keys/$DEV" > /tmp/deploy_key + chmod 600 /tmp/deploy_key + + echo " Waiting for k3s to become active on $VM_IP..." + for i in $(seq 1 30); do + if ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + -i /tmp/deploy_key "$CI_USER@$VM_IP" \ + "systemctl is-active --quiet k3s" 2>/dev/null; then + echo " k3s is active." + break + fi + if [ "$i" -eq 30 ]; then + echo "ERROR: k3s did not become active within 5 minutes." + exit 1 + fi + echo " Attempt $i/30 — retrying in 10s..." + sleep 10 + done + + ssh -o StrictHostKeyChecking=no -i /tmp/deploy_key "$CI_USER@$VM_IP" \ + "cat /etc/rancher/k3s/k3s.yaml" \ + | sed "s|127.0.0.1|$VM_IP|g" \ + | gopass insert -f "infra/kubeconfigs/$DEV" + + rm -f /tmp/deploy_key + echo " kubeconfig stored in gopass at infra/kubeconfigs/$DEV" + fi + + # ── Chained Docker deploy ── + if [ "$VM_ROLE" = "docker" ] && [ -f "apps/${DEV}-qa.tfvars" ]; then + APP="${DEV}-qa" + echo " Chained Docker deploy: $APP → $VM_IP" + + gopass show -o "infra/ssh-keys/$DEV" > /tmp/deploy_key + chmod 600 /tmp/deploy_key + + DB_PASSWORD=$(gopass show -o "apps/$APP/db_password" 2>/dev/null || echo "") + GIT_TOKEN=$(gopass show -o "apps/$APP/git_token" 2>/dev/null || echo "") + + tofu init \ + -backend-config="bucket=tofu-state" \ + -backend-config="key=apps/$APP.tfstate" \ + -backend-config="endpoint=$SEAWEED_S3_ENDPOINT" \ + -backend-config="region=us-east-1" \ + -backend-config="force_path_style=true" \ + -reconfigure + + tofu workspace select "$APP" 2>/dev/null || tofu workspace new "$APP" + + tofu apply -auto-approve \ + -var-file="apps/${DEV}-qa.tfvars" \ + -var="ssh_host=$VM_IP" \ + -var="ssh_key_path=/tmp/deploy_key" \ + -var="db_password=$DB_PASSWORD" \ + -var="openresty_git_token=$GIT_TOKEN" + + rm -f /tmp/deploy_key + echo " Chained Docker deploy complete: $APP" + fi + + # ── Chained K8s deploy ── + if [ "$VM_ROLE" = "k3s" ] && [ -f "k8s/apps/${DEV}-qa.tfvars" ]; then + APP="${DEV}-qa" + echo " Chained K8s deploy: $APP" + + gopass show -o "infra/kubeconfigs/$DEV" > /tmp/kubeconfig + chmod 600 /tmp/kubeconfig + + RABBITMQ_PASSWORD=$(gopass show -o "apps/$APP/rabbitmq_password" 2>/dev/null || echo "") + LOKI_AUTH_TOKEN=$(gopass show -o "apps/$APP/loki_token" 2>/dev/null || echo "") + + cd k8s + tofu init \ + -backend-config="bucket=tofu-state" \ + -backend-config="key=apps-k8s/$APP.tfstate" \ + -backend-config="endpoint=$SEAWEED_S3_ENDPOINT" \ + -backend-config="region=us-east-1" \ + -backend-config="force_path_style=true" \ + -reconfigure + + tofu workspace select "$APP" 2>/dev/null || tofu workspace new "$APP" + + tofu apply -auto-approve \ + -var-file="../k8s/apps/${DEV}-qa.tfvars" \ + -var="kubeconfig_path=/tmp/kubeconfig" \ + -var="rabbitmq_password=$RABBITMQ_PASSWORD" \ + -var="loki_auth_token=$LOKI_AUTH_TOKEN" + + cd .. + rm -f /tmp/kubeconfig + echo " Chained K8s deploy complete: $APP" + fi + + echo " Provisioned: $DEV" + ); then + echo "ERROR: Provisioning of $DEV failed — continuing with remaining VMs" + FAILED=1 + rm -f /tmp/deploy_key /tmp/kubeconfig + fi + done + + if [ $FAILED -ne 0 ]; then + echo "" + echo "One or more VM provisioning operations failed. See logs above for details." + exit 1 + fi + after_script: + - rm -f /tmp/deploy_key /tmp/kubeconfig + +# ─── Destroy (tfvars file deleted) ─────────────────────────────────────────── + +destroy-proxmox: + extends: .proxmox_base + stage: destroy + rules: + - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' + changes: + - proxmox/apps/*.tfvars + script: + - | + DELETED=$(git diff --name-only --diff-filter=D \ + "$CI_COMMIT_BEFORE_SHA" "$CI_COMMIT_SHA" -- 'proxmox/apps/*.tfvars') + + if [ -z "$DELETED" ]; then + echo "No Proxmox tfvars files deleted. Nothing to destroy." + exit 0 + fi + + echo "Files to destroy: $DELETED" + FAILED=0 + + for TFVARS in $DELETED; do + DEV=$(basename "$TFVARS" .tfvars) + echo "" + echo "══════════════════════════════════════════" + echo " Destroying VM: $DEV" + echo "══════════════════════════════════════════" + + if ! ( + set -e + + # Recover deleted tfvars from git history + git show "$CI_COMMIT_BEFORE_SHA:$TFVARS" > /tmp/${DEV}.tfvars + + export AWS_ACCESS_KEY_ID="$SEAWEED_ACCESS_KEY" + export AWS_SECRET_ACCESS_KEY="$SEAWEED_SECRET_KEY" + TLS_INSECURE="${PROXMOX_TLS_INSECURE:-false}" + + cd proxmox + tofu init \ + -backend-config="bucket=tofu-state" \ + -backend-config="key=apps-proxmox/$DEV.tfstate" \ + -backend-config="endpoint=$SEAWEED_S3_ENDPOINT" \ + -backend-config="region=us-east-1" \ + -backend-config="force_path_style=true" \ + -reconfigure + + tofu workspace select "$DEV" + + tofu destroy -auto-approve \ + -var-file="/tmp/${DEV}.tfvars" \ + -var="proxmox_endpoint=$PROXMOX_ENDPOINT" \ + -var="proxmox_api_token=$PROXMOX_API_TOKEN" \ + -var="proxmox_tls_insecure=$TLS_INSECURE" + + tofu workspace select default + tofu workspace delete "$DEV" + cd .. + + # Remove credentials from gopass + gopass rm -f "infra/ssh-keys/$DEV" 2>/dev/null || true + gopass rm -f "infra/kubeconfigs/$DEV" 2>/dev/null || true + + rm -f "/tmp/${DEV}.tfvars" + echo " Destroyed: $DEV" + ); then + echo "ERROR: Destroy of $DEV failed — continuing with remaining VMs" + FAILED=1 + rm -f "/tmp/${DEV}.tfvars" + fi + done + + if [ $FAILED -ne 0 ]; then + echo "" + echo "One or more VM destroy operations failed. See logs above for details." + exit 1 + fi + after_script: + - rm -f /tmp/deploy_key /tmp/kubeconfig /tmp/*.tfvars diff --git a/Makefile b/Makefile index 36fb88d..bd853ce 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help validate fmt fmt-fix test-unit test-unit-docker test-unit-k8s \ - test-integration-docker test-integration-k8s test-all clean + test-unit-proxmox test-integration-docker test-integration-k8s test-all clean TOFU ?= tofu TFLINT ?= tflint @@ -21,7 +21,7 @@ help: ## Static analysis (no infrastructure required) ##───────────────────────────────────────────────────────────────────────────── -## validate Run 'tofu validate' on both stacks +## validate Run 'tofu validate' on all stacks validate: @echo "── Validating Docker stack ──" $(TOFU) init -backend=false -input=false > /dev/null @@ -29,6 +29,9 @@ validate: @echo "── Validating K8s stack ──" cd k8s && $(TOFU) init -backend=false -input=false > /dev/null cd k8s && $(TOFU) validate + @echo "── Validating Proxmox stack ──" + cd proxmox && $(TOFU) init -backend=false -input=false > /dev/null + cd proxmox && $(TOFU) validate @echo "All configurations valid." ## fmt-check Check HCL formatting in all .tf files @@ -62,8 +65,14 @@ test-unit-k8s: cd k8s && $(TOFU) init -backend=false -input=false > /dev/null cd k8s && $(TOFU) test +## test-unit-proxmox Run Proxmox stack unit tests (tofu test) +test-unit-proxmox: + @echo "── Proxmox stack unit tests ──" + cd proxmox && $(TOFU) init -backend=false -input=false > /dev/null + cd proxmox && $(TOFU) test + ## test-unit Run all unit tests -test-unit: test-unit-docker test-unit-k8s +test-unit: test-unit-docker test-unit-k8s test-unit-proxmox ##───────────────────────────────────────────────────────────────────────────── ## Integration tests (real local infrastructure) diff --git a/README.md b/README.md index 6121ce0..a4c604d 100644 --- a/README.md +++ b/README.md @@ -32,15 +32,23 @@ Two CI/CD systems are supported: - [Traefik Ingress Routes](#traefik-ingress-routes) - [Variables Reference](#variables-reference-kubernetes) - [Adding a New K8s App](#adding-a-new-k8s-app) -9. [CI/CD Pipelines](#cicd-pipelines) - - [Docker Pipeline](#docker-pipeline) - - [Kubernetes Pipeline](#kubernetes-pipeline) - - [Destroy on Delete](#destroy-on-delete) - - [Gitea vs GitLab differences](#gitea-vs-gitlab-differences) -10. [Running Locally](#running-locally) -11. [Adding a New Stack Module](#adding-a-new-stack-module) -12. [Testing](#testing) -13. [Caveats and Known Limitations](#caveats-and-known-limitations) +9. [Stack: Proxmox — Developer VMs](#stack-proxmox--developer-vms) + - [How It Works](#how-it-works-proxmox) + - [Proxmox Prerequisites](#proxmox-prerequisites) + - [Variables Reference](#variables-reference-proxmox) + - [Creating a New Developer VM](#creating-a-new-developer-vm) + - [Chained App Deploy](#chained-app-deploy) + - [Decommissioning a Developer VM](#decommissioning-a-developer-vm) +10. [CI/CD Pipelines](#cicd-pipelines) + - [Docker Pipeline](#docker-pipeline) + - [Kubernetes Pipeline](#kubernetes-pipeline) + - [Proxmox Pipeline](#proxmox-pipeline) + - [Destroy on Delete](#destroy-on-delete) + - [Gitea vs GitLab differences](#gitea-vs-gitlab-differences) +11. [Running Locally](#running-locally) +12. [Adding a New Stack Module](#adding-a-new-stack-module) +13. [Testing](#testing) +14. [Caveats and Known Limitations](#caveats-and-known-limitations) --- @@ -49,21 +57,27 @@ Two CI/CD systems are supported: ``` Gitea repo │ -├── apps/*.tfvars ← Docker app configs -└── k8s/apps/*.tfvars ← Kubernetes app configs +├── apps/*.tfvars ← Docker app configs +├── k8s/apps/*.tfvars ← Kubernetes app configs +└── proxmox/apps/*.tfvars ← Developer VM configs Push to main │ -├── deploy.yml ← detects apps/*.tfvars changes -│ └── tofu apply ← connects to remote Docker host via SSH +├── deploy.yml ← detects apps/*.tfvars changes +│ └── tofu apply ← connects to remote Docker host via SSH │ -└── deploy-k8s.yml ← detects k8s/apps/*.tfvars changes - └── tofu apply ← connects to K8s cluster via kubeconfig +├── deploy-k8s.yml ← detects k8s/apps/*.tfvars changes +│ └── tofu apply ← connects to K8s cluster via kubeconfig +│ +└── deploy-proxmox.yml ← detects proxmox/apps/*.tfvars changes + └── tofu apply ← provisions Proxmox VM via API token + ├── stores credentials in gopass + └── (optional) tofu apply → deploys app onto the new VM │ - └── Per-app workspace in SeaweedFS state bucket + └── Per-app/VM workspace in SeaweedFS state bucket ``` -Each `.tfvars` file = one app deployment. One Tofu workspace = one isolated state. +Each `.tfvars` file = one deployment. One Tofu workspace = one isolated state. Secrets are never stored in the repository — they are fetched from **gopass** at pipeline runtime and passed to `tofu apply` via `-var` flags. @@ -78,10 +92,16 @@ runtime and passed to `tofu apply` via `-var` flags. │ │ ├── variables.tf │ │ ├── main.tf │ │ └── outputs.tf -│ └── app-k8s-nodered-rabbitmq/ # Kubernetes stack module +│ ├── app-k8s-nodered-rabbitmq/ # Kubernetes stack module +│ │ ├── variables.tf +│ │ ├── main.tf +│ │ └── outputs.tf +│ └── vm-proxmox/ # Proxmox developer VM module │ ├── variables.tf │ ├── main.tf -│ └── outputs.tf +│ ├── outputs.tf +│ ├── cloud-init-docker.yaml.tftpl +│ └── cloud-init-k3s.yaml.tftpl │ ├── apps/ # Docker app configs (one file per app) │ ├── example-dev.tfvars @@ -103,6 +123,15 @@ runtime and passed to `tofu apply` via `-var` flags. │ └── tests/ │ └── k8s_validation.tftest.hcl # K8s stack unit tests (tofu test) │ +├── proxmox/ # Proxmox stack root +│ ├── main.tf # Proxmox provider + module call +│ ├── variables.tf +│ ├── backend.tf +│ ├── apps/ # Developer VM configs (one file per developer) +│ │ └── example-developer.tfvars +│ └── tests/ +│ └── proxmox_validation.tftest.hcl # Proxmox stack unit tests (tofu test) +│ ├── scripts/ │ └── setup-backend.sh # One-time SeaweedFS backend setup │ @@ -112,6 +141,7 @@ runtime and passed to `tofu apply` via `-var` flags. │ └── workflows/ │ ├── deploy.yml # Docker CI/CD pipeline (Gitea Actions) │ ├── deploy-k8s.yml # Kubernetes CI/CD pipeline (Gitea Actions) +│ ├── deploy-proxmox.yml # Proxmox VM pipeline (Gitea Actions) │ └── test.yml # Test pipeline (Gitea Actions) │ └── .gitlab/ @@ -119,6 +149,7 @@ runtime and passed to `tofu apply` via `-var` flags. └── workflows/ ├── deploy.gitlab-ci.yml # Docker CI/CD pipeline (GitLab CI) ├── deploy-k8s.gitlab-ci.yml # Kubernetes CI/CD pipeline (GitLab CI) + ├── deploy-proxmox.gitlab-ci.yml # Proxmox VM pipeline (GitLab CI) └── test.gitlab-ci.yml # Test pipeline (GitLab CI) ``` @@ -176,6 +207,7 @@ access key and secret key. |-------|------------------| | Docker | `apps/.tfstate` | | Kubernetes | `apps-k8s/.tfstate` | +| Proxmox | `apps-proxmox/.tfstate` | ### 4. Initialise the backend (automated) @@ -226,9 +258,9 @@ gopass store │ ├── infra/ │ ├── ssh-keys/ -│ │ └── # SSH private key for the remote Docker host +│ │ └── # SSH private key — Docker host or developer VM │ └── kubeconfigs/ -│ └── # kubeconfig for the target K8s cluster +│ └── # kubeconfig — K8s cluster or developer k3s VM │ └── apps/ └── / @@ -238,6 +270,11 @@ gopass store └── loki_token # Loki bearer token (K8s stack, optional) ``` +> **Proxmox note:** The Proxmox pipeline writes `infra/ssh-keys/` and +> `infra/kubeconfigs/` automatically — no manual `gopass insert` needed for +> the VM credentials. App-level secrets (e.g. `apps/-qa/db_password`) still +> need to be added manually before the chained deploy runs. + ### Adding a secret ```sh @@ -257,12 +294,15 @@ For SSH keys specifically: update `infra/ssh-keys/` in gopass and updat ## CI/CD Variables -The same five values are required regardless of which CI/CD system you use. +The same base values are required regardless of which CI/CD system you use. +Add the Proxmox group when using the developer VM stack. ### Gitea Repository Secrets Configure in **Gitea → Repository → Settings → Secrets**. +#### Base (all stacks) + | Secret | Purpose | |--------|---------| | `GOPASS_GPG_KEY` | GPG private key (armored) to decrypt the gopass store | @@ -271,11 +311,21 @@ Configure in **Gitea → Repository → Settings → Secrets**. | `SEAWEED_ACCESS_KEY` | SeaweedFS S3 access key | | `SEAWEED_SECRET_KEY` | SeaweedFS S3 secret key (mark as masked) | +#### Proxmox stack (add when using developer VMs) + +| Secret | Purpose | +|--------|---------| +| `PROXMOX_ENDPOINT` | Proxmox API URL, e.g. `https://proxmox.example.com:8006/` | +| `PROXMOX_API_TOKEN` | API token in `user@realm!tokenid=secret` format (mark as masked) | +| `PROXMOX_TLS_INSECURE` | `"true"` when using a self-signed TLS certificate (optional) | + ### GitLab CI/CD Variables Configure in **GitLab → Project → Settings → CI/CD → Variables**. Mark sensitive values as **Masked** so they are redacted from job logs. +#### Base (all stacks) + | Variable | Masked | Purpose | |----------|--------|---------| | `GOPASS_GPG_KEY` | yes | GPG private key (armored) to decrypt the gopass store | @@ -284,6 +334,14 @@ as **Masked** so they are redacted from job logs. | `SEAWEED_ACCESS_KEY` | yes | SeaweedFS S3 access key | | `SEAWEED_SECRET_KEY` | yes | SeaweedFS S3 secret key | +#### Proxmox stack (add when using developer VMs) + +| Variable | Masked | Purpose | +|----------|--------|---------| +| `PROXMOX_ENDPOINT` | no | Proxmox API URL, e.g. `https://proxmox.example.com:8006/` | +| `PROXMOX_API_TOKEN` | yes | API token in `user@realm!tokenid=secret` format | +| `PROXMOX_TLS_INSECURE` | no | `"true"` when using a self-signed TLS certificate (optional) | + > **Note:** GitLab CI variables are plain environment variables — no `${{ secrets.X }}` > syntax is needed. The pipelines reference them directly as `$VARIABLE_NAME`. @@ -642,15 +700,333 @@ The Traefik entrypoint (default: `"web"`) is configurable per app via `traefik_e --- +## Stack: Proxmox — Developer VMs + +Provisions persistent developer VMs on a **Proxmox VE** host via the Proxmox API. +Each developer gets an isolated VM that can run either Docker (for the Docker stack) +or k3s (for the Kubernetes stack), giving them a full QA environment that mirrors +the production deployment pipeline. + +### How It Works (Proxmox) + +``` +proxmox/apps/.tfvars → pipeline provisions VM on Proxmox + │ + ├── clones a cloud-init template + ├── installs Docker CE (vm_role = "docker") + │ or k3s (vm_role = "k3s") + ├── injects developer SSH key + ├── stores credentials in gopass automatically + │ infra/ssh-keys/ + │ infra/kubeconfigs/ (k3s only) + │ + └── (optional) chained deploy + apps/-qa.tfvars (docker) + k8s/apps/-qa.tfvars (k3s) +``` + +The VM hostname and all gopass key names are derived from the tfvars filename +(`proxmox/apps/alice.tfvars` → developer name `alice`). + +**IP address:** assigned by DHCP, read from the QEMU guest agent after boot. +With a persistent VM and stable DHCP leases this IP does not change day-to-day, +but static DHCP reservations on your router/DHCP server are recommended for production use. + +### Proxmox Prerequisites + +Before using this stack, complete the following one-time setup on Proxmox: + +#### 1. Cloud-init template + +The module clones an existing VM template. The template must: + +- Be a Linux VM (Debian 12 or Ubuntu 22.04 recommended) +- Have `cloud-init` and `qemu-guest-agent` installed and enabled +- Have the cloud-init drive attached (`--ide2 local-lvm:cloudinit` or equivalent) + +Create a template from a cloud image (example for Debian 12): + +```sh +# On the Proxmox host +VMID=9000 +wget https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2 + +qm create $VMID --name debian12-cloud --memory 1024 --cores 1 --net0 virtio,bridge=vmbr0 +qm importdisk $VMID debian-12-genericcloud-amd64.qcow2 local-lvm +qm set $VMID --scsihw virtio-scsi-pci --scsi0 local-lvm:vm-${VMID}-disk-0 +qm set $VMID --ide2 local-lvm:cloudinit --boot c --bootdisk scsi0 +qm set $VMID --serial0 socket --vga serial0 +qm set $VMID --agent enabled=1 + +# Install qemu-guest-agent inside the VM, then convert to template: +qm template $VMID +``` + +Note the VM ID (`9000` in the example) — this is your `template_vm_id` in the tfvars. + +#### 2. Enable Snippets on a datastore + +Cloud-init user-data (the Docker/k3s install script) is uploaded as a Proxmox snippet. +At least one datastore must have the **Snippets** content type enabled. + +Proxmox UI → **Datacenter → Storage → `local` → Edit → Content** → check **Snippets**. + +The default `snippets_datastore = "local"` in the tfvars assumes the `local` datastore. + +#### 3. Create an API token + +Proxmox UI → **Datacenter → Permissions → API Tokens → Add** + +| Field | Value | +|-------|-------| +| User | `root@pam` (or a dedicated CI user) | +| Token ID | `ci` (or any name you choose) | +| Privilege Separation | unchecked (inherits user permissions) | + +Copy the displayed token secret — it is shown only once. The full token string has the +form `root@pam!ci=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`. + +Store it in the Gitea secret `PROXMOX_API_TOKEN` and configure `PROXMOX_ENDPOINT` to +point at your Proxmox host, e.g. `https://proxmox.home:8006/`. + +Set `PROXMOX_TLS_INSECURE=true` if your Proxmox uses a self-signed certificate. + +### Variables Reference (Proxmox) + +#### Connection (injected at CI runtime — never in tfvars) + +| Variable | Type | Description | +|----------|------|-------------| +| `proxmox_endpoint` | string | Proxmox API URL, e.g. `https://proxmox.example.com:8006/` | +| `proxmox_api_token` | string | API token `user@realm!tokenid=secret` (sensitive) | +| `proxmox_tls_insecure` | bool | Skip TLS verification (default: `false`) | + +#### Placement + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `proxmox_node` | string | — | Proxmox node name, e.g. `"pve"` | +| `template_vm_id` | number | — | VM ID of the cloud-init template to clone | +| `storage_pool` | string | — | Disk storage pool, e.g. `"local-lvm"` | +| `snippets_datastore` | string | `"local"` | Datastore with Snippets content enabled | +| `vm_id` | number | `0` | Explicit Proxmox VM ID; `0` = auto-assign | + +#### Sizing + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `cores` | number | `2` | vCPU cores | +| `memory_mb` | number | `2048` | RAM in megabytes | +| `disk_size_gb` | number | `20` | Root disk size in gigabytes | + +#### Networking + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `bridge` | string | `"vmbr0"` | Proxmox network bridge | +| `vlan_tag` | number | `null` | VLAN tag for the VM NIC; `null` = untagged | + +#### Role & access + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `vm_role` | string | — | `"docker"` or `"k3s"` | +| `cloud_init_user` | string | `"developer"` | Username created by cloud-init | +| `ssh_public_key` | string | `""` | Developer's SSH public key. Empty = auto-generate | + +### Creating a New Developer VM + +This is the standard workflow for onboarding a developer onto their own QA environment. + +#### Step 1 — Add Proxmox secrets to Gitea (one-time, per repository) + +If not already done, add to **Gitea → Repository → Settings → Secrets**: + +``` +PROXMOX_ENDPOINT https://proxmox.example.com:8006/ +PROXMOX_API_TOKEN root@pam!ci=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +PROXMOX_TLS_INSECURE true # only if using a self-signed cert +``` + +#### Step 2 — Create the developer VM tfvars + +Copy `proxmox/apps/example-developer.tfvars` to `proxmox/apps/.tfvars` +and fill in the values for your environment: + +```hcl +# proxmox/apps/alice.tfvars + +proxmox_node = "pve" +template_vm_id = 9000 +storage_pool = "local-lvm" + +cores = 2 +memory_mb = 4096 +disk_size_gb = 40 + +bridge = "vmbr0" +# vlan_tag = 100 # optional + +vm_role = "docker" # "docker" or "k3s" +cloud_init_user = "developer" + +# Option A — let Tofu generate an ED25519 key pair (recommended): +ssh_public_key = "" + +# Option B — provide the developer's own public key: +# ssh_public_key = "ssh-ed25519 AAAA... alice@laptop" +``` + +#### Step 3 — (Optional) Add app-level secrets to gopass + +If you also want the chained deploy to run (deploying an app onto the new VM automatically), +add the app secrets **before** pushing. The app name follows the convention `-qa`: + +For a Docker app (`apps/alice-qa.tfvars`): +```sh +gopass insert apps/alice-qa/db_password +gopass insert apps/alice-qa/git_token # only if using git_clone mode +``` + +For a K8s app (`k8s/apps/alice-qa.tfvars`): +```sh +gopass insert apps/alice-qa/rabbitmq_password # only if enable_rabbitmq = true +gopass insert apps/alice-qa/loki_token # only if Loki requires auth +``` + +#### Step 4 — (Optional) Create the app QA tfvars + +If you want the chained deploy to fire, commit an app tfvars alongside the VM tfvars. +Use any placeholder for `ssh_host` — the pipeline always overrides it with the actual VM IP. + +```hcl +# apps/alice-qa.tfvars (for vm_role = "docker") + +ssh_host = "placeholder" # overridden by pipeline with actual VM IP +ssh_user = "developer" + +app_name = "alice-qa" +environment = "dev" + +openresty_source_type = "git_clone" +openresty_git_repo = "https://gitea.example.com/alice/myapp-openresty.git" +openresty_git_ref = "v0.1.0" +openresty_external_port = 8080 + +db_name = "alice_qa" +db_user = "alice" +``` + +#### Step 5 — Commit and push to main + +```sh +git add proxmox/apps/alice.tfvars +git add apps/alice-qa.tfvars # optional, for chained deploy +git commit -m "chore: add developer VM for alice" +git push origin main +``` + +#### What the pipeline does + +``` +deploy-proxmox.yml +│ +├── tofu apply (proxmox/) +│ └── Proxmox VM created, cloud-init installs Docker or k3s +│ +├── Read outputs: VM IP, vm_role, key_was_generated +│ +├── If key_was_generated == true: +│ └── gopass insert infra/ssh-keys/alice ← private key from Tofu state +│ +├── If vm_role == "k3s": +│ ├── SSH to VM, wait for k3s to be active (up to 5 minutes) +│ ├── Fetch /etc/rancher/k3s/k3s.yaml +│ ├── Replace 127.0.0.1 with actual VM IP +│ └── gopass insert infra/kubeconfigs/alice +│ +├── If apps/alice-qa.tfvars exists (vm_role = "docker"): +│ └── tofu apply (root/) with -var="ssh_host=" +│ └── Deploys OpenResty + PostgreSQL + Redis onto the new VM +│ +└── If k8s/apps/alice-qa.tfvars exists (vm_role = "k3s"): + └── tofu apply (k8s/) with retrieved kubeconfig + └── Deploys Node-RED + optional RabbitMQ onto the new VM +``` + +#### Accessing the VM after provisioning + +The developer can SSH using: + +```sh +# If Tofu generated the key — retrieve it from gopass: +gopass show -o infra/ssh-keys/alice > ~/.ssh/alice-vm && chmod 600 ~/.ssh/alice-vm +ssh -i ~/.ssh/alice-vm developer@ + +# If the developer provided their own key: +ssh developer@ +``` + +The VM IP is visible in the pipeline logs and in the Tofu state: + +```sh +cd proxmox +tofu workspace select alice +tofu output vm_ip +``` + +### Chained App Deploy + +When a `-qa.tfvars` file exists at commit time, the Proxmox pipeline +automatically deploys the app onto the freshly provisioned VM in the same pipeline run. +No separate trigger or manual step is required. + +After initial provisioning, subsequent changes to the app tfvars (e.g. updating +`openresty_git_ref`) are handled by the **regular Docker or K8s deploy pipeline**, +not the Proxmox pipeline. The developer simply commits the updated app tfvars to `main` +as they would for any production app. + +**Naming convention summary:** + +| VM tfvars | App tfvars (Docker) | App tfvars (K8s) | +|-----------|---------------------|-----------------| +| `proxmox/apps/alice.tfvars` | `apps/alice-qa.tfvars` | `k8s/apps/alice-qa.tfvars` | +| `proxmox/apps/bob.tfvars` | `apps/bob-qa.tfvars` | `k8s/apps/bob-qa.tfvars` | + +### Decommissioning a Developer VM + +To tear down a developer's VM and remove all associated credentials: + +```sh +git rm proxmox/apps/alice.tfvars +git commit -m "chore: decommission developer VM for alice" +git push origin main +``` + +The `destroy` job in the pipeline will: + +1. Run `tofu destroy` to delete the Proxmox VM +2. Delete the Tofu workspace +3. Remove `infra/ssh-keys/alice` from gopass +4. Remove `infra/kubeconfigs/alice` from gopass (if it existed) + +> **Note:** The app QA deployment (`apps/alice-qa.tfvars`) is **not** automatically +> destroyed when the VM is decommissioned. Delete that file in a separate commit to +> trigger the standard Docker/K8s destroy pipeline, or clean it up manually with +> `tofu destroy` before removing the VM. + +--- + ## CI/CD Pipelines -Both Gitea Actions and GitLab CI implement the same three pipelines: +Both Gitea Actions and GitLab CI implement the same four pipelines: | Pipeline | Gitea file | GitLab file | Trigger | |----------|-----------|-------------|---------| | Test | `.gitea/workflows/test.yml` | `.gitlab/workflows/test.gitlab-ci.yml` | all branches + MRs | | Docker deploy | `.gitea/workflows/deploy.yml` | `.gitlab/workflows/deploy.gitlab-ci.yml` | `main` + `apps/*.tfvars` changed | | K8s deploy | `.gitea/workflows/deploy-k8s.yml` | `.gitlab/workflows/deploy-k8s.gitlab-ci.yml` | `main` + `k8s/apps/*.tfvars` changed | +| Proxmox deploy | `.gitea/workflows/deploy-proxmox.yml` | `.gitlab/workflows/deploy-proxmox.gitlab-ci.yml` | `main` + `proxmox/apps/*.tfvars` changed | ### Docker Pipeline @@ -680,6 +1056,22 @@ Both Gitea Actions and GitLab CI implement the same three pipelines: 6. `tofu apply` (working dir: `k8s/`) with tfvars + runtime secrets 7. Cleanup: `rm /tmp/kubeconfig` +### Proxmox Pipeline + +**Trigger:** push to `main` with changes to `proxmox/apps/*.tfvars` + +**Steps per changed file:** + +1. Resolve developer name from filename (`proxmox/apps/alice.tfvars` → `alice`) +2. `tofu init` against SeaweedFS backend (state key: `apps-proxmox/alice.tfstate`) +3. `tofu workspace select alice` or create if new +4. `tofu apply` with the tfvars + Proxmox API credentials via `-var` flags +5. Read VM outputs: IP, role, whether a key was generated +6. If key was generated → store SSH private key in gopass `infra/ssh-keys/alice` +7. If `vm_role = k3s` → SSH to VM, wait for k3s, fetch kubeconfig, store in gopass `infra/kubeconfigs/alice` +8. If `apps/alice-qa.tfvars` exists (`vm_role = docker`) → run Docker `tofu apply` with VM IP override +9. If `k8s/apps/alice-qa.tfvars` exists (`vm_role = k3s`) → run K8s `tofu apply` with retrieved kubeconfig + ### Destroy on Delete Both pipelines include a `destroy` job. When a `.tfvars` file is **deleted** from the @@ -816,9 +1208,10 @@ intercepted and return mock values. **Requires OpenTofu >= 1.7.0.** ```sh -make test-unit # both stacks -make test-unit-docker # Docker stack only -make test-unit-k8s # K8s stack only +make test-unit # all stacks +make test-unit-docker # Docker stack only +make test-unit-k8s # K8s stack only +make test-unit-proxmox # Proxmox stack only ``` #### What is tested @@ -845,6 +1238,18 @@ make test-unit-k8s # K8s stack only | `output.namespace == var.app_name` | Assertion | | `output.app_url_path == var.app_path_prefix` | Assertion | +**Proxmox stack** ([proxmox/tests/proxmox_validation.tftest.hcl](proxmox/tests/proxmox_validation.tftest.hcl)): + +| Test | Type | +|------|------| +| `vm_role = "docker"` plans and output is propagated | Smoke | +| `vm_role = "k3s"` plans and output is propagated | Smoke | +| `vm_role = "nomad"` is rejected | Validation | +| `vm_role = "both"` is rejected | Validation | +| `key_was_generated == true` when `ssh_public_key = ""` | Assertion | +| `key_was_generated == false` when `ssh_public_key` is provided | Assertion | +| Default `cloud_init_user` is `"developer"` | Assertion | + #### Running a single test ```sh @@ -931,10 +1336,10 @@ static ──► unit-docker ──► ### Quick reference ```sh -make test-all # L1 + L2 (fast, no infra needed) -make test-integration-k8s # L3 K8s (needs k3d) +make test-all # L1 + L2 (fast, no infra needed) — all three stacks +make test-integration-k8s # L3 K8s (needs k3d) make test-integration-docker # L3 Docker (needs local Docker + SSH) -make clean # remove .terraform dirs and local state files +make clean # remove .terraform dirs and local state files ``` --- diff --git a/modules/vm-proxmox/cloud-init-docker.yaml.tftpl b/modules/vm-proxmox/cloud-init-docker.yaml.tftpl new file mode 100644 index 0000000..2c11d06 --- /dev/null +++ b/modules/vm-proxmox/cloud-init-docker.yaml.tftpl @@ -0,0 +1,18 @@ +#cloud-config +# Installs Docker CE on first boot using the official convenience script. +# The cloud-init user is added to the docker group for rootless CLI access. + +package_update: true +package_upgrade: false + +packages: + - curl + - ca-certificates + - gnupg + +runcmd: + - curl -fsSL https://get.docker.com | sh + - usermod -aG docker ${cloud_init_user} + - systemctl enable --now docker + +final_message: "Docker installation complete. System ready after $UPTIME seconds." diff --git a/modules/vm-proxmox/cloud-init-k3s.yaml.tftpl b/modules/vm-proxmox/cloud-init-k3s.yaml.tftpl new file mode 100644 index 0000000..0ceee04 --- /dev/null +++ b/modules/vm-proxmox/cloud-init-k3s.yaml.tftpl @@ -0,0 +1,18 @@ +#cloud-config +# Installs k3s single-node on first boot using the official installer. +# The kubeconfig at /etc/rancher/k3s/k3s.yaml is made world-readable so the +# CI pipeline can retrieve it via SSH without requiring sudo. + +package_update: true +package_upgrade: false + +packages: + - curl + +runcmd: + - curl -sfL https://get.k3s.io | sh - + - systemctl enable k3s + - until systemctl is-active --quiet k3s; do sleep 3; done + - chmod 644 /etc/rancher/k3s/k3s.yaml + +final_message: "k3s installation complete. System ready after $UPTIME seconds." diff --git a/modules/vm-proxmox/main.tf b/modules/vm-proxmox/main.tf new file mode 100644 index 0000000..6256067 --- /dev/null +++ b/modules/vm-proxmox/main.tf @@ -0,0 +1,119 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = ">= 0.66" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + } +} + +# ─── SSH key handling ───────────────────────────────────────────────────────── +# Generate an ED25519 key pair when the developer did not supply a public key. + +resource "tls_private_key" "generated" { + count = var.ssh_public_key == "" ? 1 : 0 + algorithm = "ED25519" +} + +locals { + ssh_public_key = var.ssh_public_key != "" ? var.ssh_public_key : tls_private_key.generated[0].public_key_openssh + ssh_private_key = var.ssh_public_key != "" ? null : tls_private_key.generated[0].private_key_openssh +} + +# ─── Cloud-init user-data ───────────────────────────────────────────────────── +# Uploads a snippets file to Proxmox so cloud-init can install Docker or k3s +# on first boot. The target datastore must have "Snippets" content enabled +# (Proxmox UI → Datacenter → Storage → → Content → Snippets). + +resource "proxmox_virtual_environment_file" "cloud_init_config" { + content_type = "snippets" + datastore_id = var.snippets_datastore + node_name = var.proxmox_node + + source_raw { + data = templatefile("${path.module}/cloud-init-${var.vm_role}.yaml.tftpl", { + cloud_init_user = var.cloud_init_user + }) + file_name = "${var.vm_name}-cloud-init.yaml" + } +} + +# ─── Virtual machine ────────────────────────────────────────────────────────── + +resource "proxmox_virtual_environment_vm" "dev_vm" { + name = var.vm_name + node_name = var.proxmox_node + vm_id = var.vm_id > 0 ? var.vm_id : null + + clone { + vm_id = var.template_vm_id + full = true + } + + # QEMU guest agent must be pre-installed in the template. + # Required for Tofu to read the DHCP-assigned IP after boot. + agent { + enabled = true + } + + cpu { + cores = var.cores + type = "host" + } + + memory { + dedicated = var.memory_mb + } + + disk { + datastore_id = var.storage_pool + size = var.disk_size_gb + interface = "scsi0" + file_format = "raw" + discard = "on" + iothread = true + } + + network_device { + bridge = var.bridge + vlan_id = var.vlan_tag + } + + initialization { + ip_config { + ipv4 { + address = "dhcp" + } + } + + user_account { + username = var.cloud_init_user + keys = [local.ssh_public_key] + } + + # Custom user-data installs Docker CE or k3s via cloud-init runcmd. + user_data_file_id = proxmox_virtual_environment_file.cloud_init_config.id + } + + operating_system { + type = "l26" + } + + # Wait up to 10 minutes for the VM to boot and the guest agent to report an IP. + timeout_create = 600 +} + +# ─── Derived locals ─────────────────────────────────────────────────────────── + +locals { + # Collect all IPv4 addresses from non-loopback interfaces (reported by QEMU agent). + vm_ips = flatten([ + for i, name in proxmox_virtual_environment_vm.dev_vm.network_interface_names : + proxmox_virtual_environment_vm.dev_vm.ipv4_addresses[i] + if name != "lo" && length(proxmox_virtual_environment_vm.dev_vm.ipv4_addresses[i]) > 0 + ]) +} diff --git a/modules/vm-proxmox/outputs.tf b/modules/vm-proxmox/outputs.tf new file mode 100644 index 0000000..283cf08 --- /dev/null +++ b/modules/vm-proxmox/outputs.tf @@ -0,0 +1,30 @@ +output "vm_ip" { + description = "Primary IPv4 address of the VM (DHCP-assigned, read via QEMU guest agent)." + value = local.vm_ips[0] +} + +output "vm_id" { + description = "Proxmox VM ID assigned to the virtual machine." + value = proxmox_virtual_environment_vm.dev_vm.vm_id +} + +output "vm_role" { + description = "Software role installed on the VM: 'docker' or 'k3s'." + value = var.vm_role +} + +output "cloud_init_user" { + description = "Username created on the VM by cloud-init." + value = var.cloud_init_user +} + +output "ssh_private_key" { + description = "Generated SSH private key in OpenSSH format. Empty string when the caller supplied their own ssh_public_key." + value = local.ssh_private_key != null ? local.ssh_private_key : "" + sensitive = true +} + +output "key_was_generated" { + description = "True when Tofu generated the SSH key pair; false when the caller provided ssh_public_key." + value = var.ssh_public_key == "" +} diff --git a/modules/vm-proxmox/variables.tf b/modules/vm-proxmox/variables.tf new file mode 100644 index 0000000..9ee93d8 --- /dev/null +++ b/modules/vm-proxmox/variables.tf @@ -0,0 +1,90 @@ +# ─── Proxmox placement ──────────────────────────────────────────────────────── + +variable "proxmox_node" { + description = "Proxmox cluster node name where the VM will be created." + type = string +} + +variable "vm_name" { + description = "VM hostname and Proxmox display name. Derived from the Tofu workspace (developer name)." + type = string +} + +variable "vm_id" { + description = "Explicit Proxmox VM ID. Set to 0 to let Proxmox auto-assign." + type = number + default = 0 +} + +variable "template_vm_id" { + description = "VM ID of the cloud-init template to clone. Must have qemu-guest-agent pre-installed." + type = number +} + +variable "storage_pool" { + description = "Proxmox storage pool for the VM disk (e.g. 'local-lvm', 'ceph-pool')." + type = string +} + +variable "snippets_datastore" { + description = "Proxmox datastore with 'snippets' content enabled, used to upload cloud-init user-data." + type = string + default = "local" +} + +# ─── VM sizing ──────────────────────────────────────────────────────────────── + +variable "cores" { + description = "Number of vCPU cores." + type = number + default = 2 +} + +variable "memory_mb" { + description = "RAM in megabytes." + type = number + default = 2048 +} + +variable "disk_size_gb" { + description = "Root disk size in gigabytes." + type = number + default = 20 +} + +# ─── Networking ─────────────────────────────────────────────────────────────── + +variable "bridge" { + description = "Proxmox network bridge (e.g. 'vmbr0', 'vmbr1')." + type = string + default = "vmbr0" +} + +variable "vlan_tag" { + description = "Optional VLAN tag for the VM NIC. null = untagged/native VLAN." + type = number + default = null +} + +# ─── VM role & access ───────────────────────────────────────────────────────── + +variable "vm_role" { + description = "Software to install on the VM: 'docker' installs Docker CE, 'k3s' installs k3s single-node." + type = string + validation { + condition = contains(["docker", "k3s"], var.vm_role) + error_message = "vm_role must be 'docker' or 'k3s'." + } +} + +variable "cloud_init_user" { + description = "Username to create via cloud-init on the VM." + type = string + default = "developer" +} + +variable "ssh_public_key" { + description = "Developer's SSH public key to inject. Leave empty to auto-generate an ED25519 key pair (private key stored in Tofu state, exposed as a sensitive output)." + type = string + default = "" +} diff --git a/proxmox/apps/example-developer.tfvars b/proxmox/apps/example-developer.tfvars new file mode 100644 index 0000000..65a78b7 --- /dev/null +++ b/proxmox/apps/example-developer.tfvars @@ -0,0 +1,49 @@ +# ─── Example developer VM configuration ────────────────────────────────────── +# +# File naming convention: proxmox/apps/.tfvars +# The filename becomes the Tofu workspace name, the VM hostname, and the +# key name used for gopass secrets (infra/ssh-keys/). +# +# Required Gitea secrets (set once in the repository settings): +# PROXMOX_ENDPOINT — https://proxmox.example.com:8006/ +# PROXMOX_API_TOKEN — user@pam!tokenid=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# PROXMOX_TLS_INSECURE — true (if using a self-signed cert), otherwise omit +# +# Chained deploy convention: +# vm_role = "docker" → also applies apps/-qa.tfvars (if present) +# vm_role = "k3s" → also applies k8s/apps/-qa.tfvars (if present) +# ───────────────────────────────────────────────────────────────────────────── + +# ─── Proxmox placement ─────────────────────────────────────────────────────── + +proxmox_node = "pve" # your Proxmox node name +template_vm_id = 9000 # VM ID of your cloud-init template (must have qemu-guest-agent) +storage_pool = "local-lvm" + +# snippets_datastore = "local" # default; must have Snippets content type enabled + +# ─── VM sizing ─────────────────────────────────────────────────────────────── + +cores = 2 +memory_mb = 2048 +disk_size_gb = 30 + +# ─── Networking ────────────────────────────────────────────────────────────── + +bridge = "vmbr0" + +# vlan_tag = 100 # uncomment to place the VM in a specific VLAN + +# ─── Role & access ─────────────────────────────────────────────────────────── + +# "docker" → installs Docker CE; pairs with apps/-qa.tfvars +# "k3s" → installs k3s; pairs with k8s/apps/-qa.tfvars +vm_role = "docker" + +cloud_init_user = "developer" + +# Leave empty to auto-generate an ED25519 key pair. +# The private key is stored in Tofu state and automatically saved to gopass +# as infra/ssh-keys/ by the CI pipeline. +# Or paste the developer's SSH public key here: +ssh_public_key = "" diff --git a/proxmox/backend.tf b/proxmox/backend.tf new file mode 100644 index 0000000..b6a5a34 --- /dev/null +++ b/proxmox/backend.tf @@ -0,0 +1,8 @@ +terraform { + # Local backend is the default — safe for local testing and tofu validate. + # Uncomment the block below and run `tofu init` with backend-config flags + # (or scripts/setup-backend.sh) to switch to SeaweedFS S3. + # State key pattern: apps-proxmox/.tfstate + + # backend "s3" {} +} diff --git a/proxmox/main.tf b/proxmox/main.tf new file mode 100644 index 0000000..0db95d8 --- /dev/null +++ b/proxmox/main.tf @@ -0,0 +1,70 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = ">= 0.66" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + } +} + +provider "proxmox" { + endpoint = var.proxmox_endpoint + api_token = var.proxmox_api_token + insecure = var.proxmox_tls_insecure +} + +# The Tofu workspace name becomes the VM name and developer identifier. +# One workspace = one developer VM = one proxmox/apps/.tfvars file. +module "dev_vm" { + source = "../modules/vm-proxmox" + + proxmox_node = var.proxmox_node + vm_name = terraform.workspace + vm_id = var.vm_id + template_vm_id = var.template_vm_id + storage_pool = var.storage_pool + snippets_datastore = var.snippets_datastore + cores = var.cores + memory_mb = var.memory_mb + disk_size_gb = var.disk_size_gb + bridge = var.bridge + vlan_tag = var.vlan_tag + vm_role = var.vm_role + cloud_init_user = var.cloud_init_user + ssh_public_key = var.ssh_public_key +} + +output "vm_ip" { + description = "VM IPv4 address (DHCP-assigned, read via QEMU guest agent)." + value = module.dev_vm.vm_ip +} + +output "vm_id" { + description = "Proxmox VM ID." + value = module.dev_vm.vm_id +} + +output "vm_role" { + description = "Software role installed: 'docker' or 'k3s'." + value = module.dev_vm.vm_role +} + +output "cloud_init_user" { + description = "Username on the VM." + value = module.dev_vm.cloud_init_user +} + +output "ssh_private_key" { + description = "Generated SSH private key (empty string when caller provided ssh_public_key)." + value = module.dev_vm.ssh_private_key + sensitive = true +} + +output "key_was_generated" { + description = "True when Tofu generated the SSH key pair." + value = module.dev_vm.key_was_generated +} diff --git a/proxmox/tests/proxmox_validation.tftest.hcl b/proxmox/tests/proxmox_validation.tftest.hcl new file mode 100644 index 0000000..e261878 --- /dev/null +++ b/proxmox/tests/proxmox_validation.tftest.hcl @@ -0,0 +1,101 @@ +# Unit tests for the Proxmox stack using mock providers. +# No real Proxmox or TLS infrastructure is required. + +mock_provider "proxmox" {} +mock_provider "tls" {} + +# ─── Shared variable defaults ───────────────────────────────────────────────── + +variables { + proxmox_endpoint = "https://proxmox.example.com:8006/" + proxmox_api_token = "test@pam!ci=00000000-0000-0000-0000-000000000000" + proxmox_node = "pve" + template_vm_id = 9000 + storage_pool = "local-lvm" + vm_role = "docker" +} + +# ─── Role: docker ───────────────────────────────────────────────────────────── + +run "docker_vm_role_is_propagated" { + command = plan + + assert { + condition = module.dev_vm.vm_role == "docker" + error_message = "vm_role output should equal the input 'docker'." + } +} + +run "docker_vm_key_generated_when_no_public_key_provided" { + command = plan + + variables { + ssh_public_key = "" + } + + assert { + condition = module.dev_vm.key_was_generated == true + error_message = "key_was_generated should be true when ssh_public_key is empty." + } +} + +run "docker_vm_key_not_generated_when_public_key_provided" { + command = plan + + variables { + ssh_public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExampleKeyForTesting developer@example.com" + } + + assert { + condition = module.dev_vm.key_was_generated == false + error_message = "key_was_generated should be false when ssh_public_key is provided." + } +} + +# ─── Role: k3s ──────────────────────────────────────────────────────────────── + +run "k3s_vm_role_is_propagated" { + command = plan + + variables { + vm_role = "k3s" + } + + assert { + condition = module.dev_vm.vm_role == "k3s" + error_message = "vm_role output should equal the input 'k3s'." + } +} + +# ─── Variable validation ────────────────────────────────────────────────────── + +run "reject_invalid_vm_role" { + command = plan + + variables { + vm_role = "nomad" + } + + expect_failures = [var.vm_role] +} + +run "reject_another_invalid_role" { + command = plan + + variables { + vm_role = "both" + } + + expect_failures = [var.vm_role] +} + +# ─── Sizing defaults are applied ────────────────────────────────────────────── + +run "default_cloud_init_user" { + command = plan + + assert { + condition = module.dev_vm.cloud_init_user == "developer" + error_message = "Default cloud_init_user should be 'developer'." + } +} diff --git a/proxmox/variables.tf b/proxmox/variables.tf new file mode 100644 index 0000000..f2441b5 --- /dev/null +++ b/proxmox/variables.tf @@ -0,0 +1,104 @@ +# ─── Proxmox connection (injected at CI runtime from Gitea secrets) ─────────── + +variable "proxmox_endpoint" { + description = "Proxmox API endpoint URL, e.g. 'https://proxmox.example.com:8006/'." + type = string +} + +variable "proxmox_api_token" { + description = "Proxmox API token in 'user@realm!tokenid=secret' format." + type = string + sensitive = true +} + +variable "proxmox_tls_insecure" { + description = "Skip TLS verification for the Proxmox API. Set to true when using a self-signed certificate." + type = bool + default = false +} + +# ─── Proxmox placement ──────────────────────────────────────────────────────── + +variable "proxmox_node" { + description = "Proxmox cluster node name where the VM will be created." + type = string +} + +variable "template_vm_id" { + description = "VM ID of the cloud-init-enabled template to clone. Must have qemu-guest-agent pre-installed." + type = number +} + +variable "storage_pool" { + description = "Proxmox storage pool for the VM disk (e.g. 'local-lvm', 'ceph-pool')." + type = string +} + +variable "snippets_datastore" { + description = "Proxmox datastore with 'Snippets' content enabled, used for cloud-init user-data upload." + type = string + default = "local" +} + +# ─── VM sizing ──────────────────────────────────────────────────────────────── + +variable "vm_id" { + description = "Explicit Proxmox VM ID. Set to 0 for Proxmox to auto-assign." + type = number + default = 0 +} + +variable "cores" { + description = "Number of vCPU cores." + type = number + default = 2 +} + +variable "memory_mb" { + description = "RAM in megabytes." + type = number + default = 2048 +} + +variable "disk_size_gb" { + description = "Root disk size in gigabytes." + type = number + default = 20 +} + +# ─── Networking ─────────────────────────────────────────────────────────────── + +variable "bridge" { + description = "Proxmox network bridge (e.g. 'vmbr0', 'vmbr1')." + type = string + default = "vmbr0" +} + +variable "vlan_tag" { + description = "Optional VLAN tag for the VM NIC. null = untagged/native VLAN." + type = number + default = null +} + +# ─── VM role & access ───────────────────────────────────────────────────────── + +variable "vm_role" { + description = "Software to install on the VM: 'docker' installs Docker CE, 'k3s' installs k3s single-node." + type = string + validation { + condition = contains(["docker", "k3s"], var.vm_role) + error_message = "vm_role must be 'docker' or 'k3s'." + } +} + +variable "cloud_init_user" { + description = "Username to create via cloud-init on the VM." + type = string + default = "developer" +} + +variable "ssh_public_key" { + description = "Developer's SSH public key to inject into the VM. Leave empty to auto-generate an ED25519 key pair." + type = string + default = "" +}