OpenTofuPlayground/modules/app-k8s-nodered-rabbitmq/main.tf

terraform {
  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.31"
    }
    kubectl = {
      source  = "gavinbunney/kubectl"
      version = "~> 1.14"
    }
  }
}

locals {
  # Alloy config: bearer_token block only when a token is provided
  alloy_auth_line = var.loki_auth_token != "" ? "    bearer_token = env(\"LOKI_AUTH_TOKEN\")\n" : ""

  # Init container shell script:
  # - Copies all files from the image's data dir to the shared PVC mount (/data)
  # - Skips flows.json and flows_cred.json if they already exist on the PVC
  #   (preserves user-modified flows across pod restarts / redeployments)
  # Shell variables ($SRC, $DEST etc.) are NOT HCL interpolations — only
  # ${var.init_data_src_path} is resolved by Tofu before the string reaches K8s.
  init_script = <<-EOT
    set -e
    SRC="${var.init_data_src_path}"
    DEST="/data_init"
    find "$SRC" -type f | while IFS= read -r src_file; do
      rel=$(echo "$src_file" | sed "s|^$SRC/||")
      dest_file="$DEST/$rel"
      if [ "$rel" = "flows.json" ] || [ "$rel" = "flows_cred.json" ]; then
        if [ -f "$dest_file" ]; then
          echo "Preserving existing $rel"
          continue
        fi
      fi
      mkdir -p "$(dirname "$dest_file")"
      cp "$src_file" "$dest_file"
      echo "Copied $rel"
    done
    echo "Init complete."
  EOT
}

# ─── Namespace ────────────────────────────────────────────────────────────────

resource "kubernetes_namespace" "app" {
  metadata {
    name = var.app_name
    labels = {
      "app.kubernetes.io/managed-by" = "opentofu"
    }
  }
}

# ─── Secrets ──────────────────────────────────────────────────────────────────

resource "kubernetes_secret" "app" {
  metadata {
    name      = "${var.app_name}-secret"
    namespace = kubernetes_namespace.app.metadata[0].name
  }

  data = {
    LOKI_AUTH_TOKEN   = var.loki_auth_token
    RABBITMQ_USER     = var.rabbitmq_user
    RABBITMQ_PASSWORD = var.rabbitmq_password
  }
}

# ─── Alloy ConfigMap ──────────────────────────────────────────────────────────
# Generates the Grafana Alloy River config to tail /data/logs/*.log and push to Loki.

resource "kubernetes_config_map" "alloy" {
  metadata {
    name      = "${var.app_name}-alloy-config"
    namespace = kubernetes_namespace.app.metadata[0].name
  }

  data = {
    "config.alloy" = <<-EOT
      local.file_match "app_logs" {
        path_targets = [{
          __path__ = "/data/logs/*.log",
          app      = "${var.app_name}",
          env      = "${var.environment}",
        }]
      }

      loki.source.file "app_logs" {
        targets    = local.file_match.app_logs.targets
        forward_to = [loki.write.default.receiver]
      }

      loki.write "default" {
        endpoint {
          url = "${var.loki_endpoint}"
      ${local.alloy_auth_line}    }
        external_labels = {
          app = "${var.app_name}",
          env = "${var.environment}",
        }
      }
    EOT
  }
}

# ─── PVC: Alloy WAL ───────────────────────────────────────────────────────────
# Small dedicated volume for the Grafana Alloy write-ahead log.
# Storing the WAL here means Alloy remembers which log lines it has already sent
# to Loki, preventing duplicate log entries after pod restarts.

resource "kubernetes_persistent_volume_claim" "alloy_wal" {
  wait_until_bound = false

  metadata {
    name      = "${var.app_name}-alloy-wal"
    namespace = kubernetes_namespace.app.metadata[0].name
  }

  spec {
    access_modes = ["ReadWriteOnce"]

    resources {
      requests = {
        storage = "100Mi"
      }
    }

    storage_class_name = var.storage_class != "" ? var.storage_class : null
  }
}

# ─── PVC: App Data ────────────────────────────────────────────────────────────
# Persists Node-RED data directory across pod restarts.
# The init container writes defaults here on first start; flows.json is never overwritten.

resource "kubernetes_persistent_volume_claim" "app_data" {
  wait_until_bound = false

  metadata {
    name      = "${var.app_name}-data"
    namespace = kubernetes_namespace.app.metadata[0].name
  }

  spec {
    access_modes = ["ReadWriteOnce"]

    resources {
      requests = {
        storage = var.app_pvc_size
      }
    }

    storage_class_name = var.storage_class != "" ? var.storage_class : null
  }
}

# ─── Deployment ───────────────────────────────────────────────────────────────

resource "kubernetes_deployment" "app" {
  metadata {
    name      = var.app_name
    namespace = kubernetes_namespace.app.metadata[0].name
    labels = {
      "app.kubernetes.io/name"       = var.app_name
      "app.kubernetes.io/managed-by" = "opentofu"
    }
  }

  spec {
    replicas = 1

    selector {
      match_labels = {
        app = var.app_name
      }
    }

    template {
      metadata {
        labels = {
          app = var.app_name
        }
      }

      spec {
        # ── Init container ──────────────────────────────────────────────────
        # Copies default configs and modules from the init image to the shared PVC.
        # flows.json and flows_cred.json are left untouched if they already exist.
        init_container {
          name  = "${var.app_name}-init"
          image = var.init_container_image

          command = ["/bin/sh", "-c", local.init_script]

          volume_mount {
            name       = "app-data"
            mount_path = "/data_init"
          }
        }

        # ── App container ───────────────────────────────────────────────────
        container {
          name  = "app"
          image = var.app_image

          port {
            name           = "http"
            container_port = var.app_port
          }

          volume_mount {
            name       = "app-data"
            mount_path = "/data"
          }

          # RabbitMQ connection info (available even if RabbitMQ is disabled)
          env {
            name  = "RABBITMQ_HOST"
            value = var.enable_rabbitmq ? "${var.app_name}-rabbitmq" : ""
          }
          env {
            name  = "RABBITMQ_VHOST"
            value = var.rabbitmq_vhost
          }
          env {
            name = "RABBITMQ_USER"
            value_from {
              secret_key_ref {
                name     = kubernetes_secret.app.metadata[0].name
                key      = "RABBITMQ_USER"
                optional = true
              }
            }
          }
          env {
            name = "RABBITMQ_PASSWORD"
            value_from {
              secret_key_ref {
                name     = kubernetes_secret.app.metadata[0].name
                key      = "RABBITMQ_PASSWORD"
                optional = true
              }
            }
          }
        }

        # ── Alloy sidecar ───────────────────────────────────────────────────
        # Tails /data/logs/*.log and forwards to Loki.
        # WAL is stored on its own PVC so log positions survive pod restarts —
        # no duplicate entries sent to Loki after a restart.
        container {
          name  = "alloy"
          image = var.alloy_image

          args = [
            "run",
            "--server.http.listen-addr=0.0.0.0:12345",
            "--storage.path=/var/lib/alloy/data",
            "/etc/alloy/config.alloy",
          ]

          port {
            name           = "alloy-http"
            container_port = 12345
          }

          # App data mounted read-only — Alloy only reads logs, never writes to /data
          volume_mount {
            name       = "app-data"
            mount_path = "/data"
            read_only  = true
          }

          volume_mount {
            name       = "alloy-config"
            mount_path = "/etc/alloy"
            read_only  = true
          }

          # Persistent WAL — survives pod restarts, prevents duplicate log shipping
          volume_mount {
            name       = "alloy-wal"
            mount_path = "/var/lib/alloy/data"
          }

          env {
            name = "LOKI_AUTH_TOKEN"
            value_from {
              secret_key_ref {
                name     = kubernetes_secret.app.metadata[0].name
                key      = "LOKI_AUTH_TOKEN"
                optional = true
              }
            }
          }
        }

        # ── Volumes ─────────────────────────────────────────────────────────
        volume {
          name = "app-data"
          persistent_volume_claim {
            claim_name = kubernetes_persistent_volume_claim.app_data.metadata[0].name
          }
        }

        volume {
          name = "alloy-config"
          config_map {
            name = kubernetes_config_map.alloy.metadata[0].name
          }
        }

        volume {
          name = "alloy-wal"
          persistent_volume_claim {
            claim_name = kubernetes_persistent_volume_claim.alloy_wal.metadata[0].name
          }
        }
      }
    }
  }
}

# ─── Service: App ─────────────────────────────────────────────────────────────

resource "kubernetes_service" "app" {
  metadata {
    name      = var.app_name
    namespace = kubernetes_namespace.app.metadata[0].name
  }

  spec {
    selector = {
      app = var.app_name
    }

    port {
      name        = "http"
      port        = var.app_port
      target_port = var.app_port
    }
  }
}

# ─── Traefik: App IngressRoute + StripPrefix Middleware ───────────────────────
# Uses kubectl_manifest (gavinbunney/kubectl provider) instead of kubernetes_manifest.
# kubectl_manifest does NOT validate CRD schemas at plan time, so tofu plan succeeds
# even on a fresh cluster before Traefik CRDs are installed.
# The API group is configurable via var.traefik_api_group to support both
# k3s >= 1.27 (traefik.io/v1alpha1) and older clusters (traefik.containo.us/v1alpha1).

resource "kubectl_manifest" "app_middleware" {
  yaml_body = <<-YAML
    apiVersion: ${var.traefik_api_group}
    kind: Middleware
    metadata:
      name: ${var.app_name}-strip-prefix
      namespace: ${kubernetes_namespace.app.metadata[0].name}
    spec:
      stripPrefix:
        prefixes:
          - "${var.app_path_prefix}"
  YAML
}

resource "kubectl_manifest" "app_ingress" {
  yaml_body = <<-YAML
    apiVersion: ${var.traefik_api_group}
    kind: IngressRoute
    metadata:
      name: ${var.app_name}
      namespace: ${kubernetes_namespace.app.metadata[0].name}
    spec:
      entryPoints:
        - ${var.traefik_entrypoint}
      routes:
        - match: "PathPrefix(`${var.app_path_prefix}`)"
          kind: Rule
          middlewares:
            - name: ${var.app_name}-strip-prefix
              namespace: ${kubernetes_namespace.app.metadata[0].name}
          services:
            - name: ${var.app_name}
              port: ${var.app_port}
  YAML

  depends_on = [kubectl_manifest.app_middleware]
}

# ─── RabbitMQ (optional) ──────────────────────────────────────────────────────

resource "kubernetes_stateful_set" "rabbitmq" {
  count = var.enable_rabbitmq ? 1 : 0

  metadata {
    name      = "${var.app_name}-rabbitmq"
    namespace = kubernetes_namespace.app.metadata[0].name
    labels = {
      "app.kubernetes.io/name"       = "${var.app_name}-rabbitmq"
      "app.kubernetes.io/managed-by" = "opentofu"
    }
  }

  spec {
    service_name = "${var.app_name}-rabbitmq-headless"
    replicas     = 1

    selector {
      match_labels = {
        app = "${var.app_name}-rabbitmq"
      }
    }

    template {
      metadata {
        labels = {
          app = "${var.app_name}-rabbitmq"
        }
      }

      spec {
        container {
          name  = "rabbitmq"
          image = var.rabbitmq_image

          port {
            name           = "amqp"
            container_port = 5672
          }
          port {
            name           = "management"
            container_port = 15672
          }

          env {
            name = "RABBITMQ_DEFAULT_USER"
            value_from {
              secret_key_ref {
                name = kubernetes_secret.app.metadata[0].name
                key  = "RABBITMQ_USER"
              }
            }
          }
          env {
            name = "RABBITMQ_DEFAULT_PASS"
            value_from {
              secret_key_ref {
                name = kubernetes_secret.app.metadata[0].name
                key  = "RABBITMQ_PASSWORD"
              }
            }
          }
          env {
            name  = "RABBITMQ_DEFAULT_VHOST"
            value = var.rabbitmq_vhost
          }

          volume_mount {
            name       = "rabbitmq-data"
            mount_path = "/var/lib/rabbitmq"
          }
        }
      }
    }

    volume_claim_template {
      metadata {
        name = "rabbitmq-data"
      }
      spec {
        access_modes = ["ReadWriteOnce"]
        resources {
          requests = {
            storage = var.rabbitmq_pvc_size
          }
        }
        storage_class_name = var.storage_class != "" ? var.storage_class : null
      }
    }
  }
}

# Headless service — required by the StatefulSet for stable pod DNS names
resource "kubernetes_service" "rabbitmq_headless" {
  count = var.enable_rabbitmq ? 1 : 0

  metadata {
    name      = "${var.app_name}-rabbitmq-headless"
    namespace = kubernetes_namespace.app.metadata[0].name
  }

  spec {
    cluster_ip = "None"
    selector = {
      app = "${var.app_name}-rabbitmq"
    }
    port {
      name = "amqp"
      port = 5672
    }
    port {
      name = "management"
      port = 15672
    }
  }
}

# ClusterIP service — used by the app container and the management IngressRoute
resource "kubernetes_service" "rabbitmq" {
  count = var.enable_rabbitmq ? 1 : 0

  metadata {
    name      = "${var.app_name}-rabbitmq"
    namespace = kubernetes_namespace.app.metadata[0].name
  }

  spec {
    selector = {
      app = "${var.app_name}-rabbitmq"
    }
    port {
      name        = "amqp"
      port        = 5672
      target_port = 5672
    }
    port {
      name        = "management"
      port        = 15672
      target_port = 15672
    }
  }
}

# Traefik Middleware: StripPrefix for RabbitMQ management UI
resource "kubectl_manifest" "rabbitmq_middleware" {
  count = var.enable_rabbitmq ? 1 : 0

  yaml_body = <<-YAML
    apiVersion: ${var.traefik_api_group}
    kind: Middleware
    metadata:
      name: ${var.app_name}-rabbitmq-strip-prefix
      namespace: ${kubernetes_namespace.app.metadata[0].name}
    spec:
      stripPrefix:
        prefixes:
          - "${var.rabbitmq_path_prefix}"
  YAML
}

# Traefik IngressRoute for RabbitMQ management UI (port 15672)
resource "kubectl_manifest" "rabbitmq_ingress" {
  count = var.enable_rabbitmq ? 1 : 0

  yaml_body = <<-YAML
    apiVersion: ${var.traefik_api_group}
    kind: IngressRoute
    metadata:
      name: ${var.app_name}-rabbitmq
      namespace: ${kubernetes_namespace.app.metadata[0].name}
    spec:
      entryPoints:
        - ${var.traefik_entrypoint}
      routes:
        - match: "PathPrefix(`${var.rabbitmq_path_prefix}`)"
          kind: Rule
          middlewares:
            - name: ${var.app_name}-rabbitmq-strip-prefix
              namespace: ${kubernetes_namespace.app.metadata[0].name}
          services:
            - name: ${var.app_name}-rabbitmq
              port: 15672
  YAML

  depends_on = [kubectl_manifest.rabbitmq_middleware]
}