#!/usr/bin/env bash
# OnFailure handler for the OpenBAO systemd timers. Invoked as:
#   send-failure-alert.sh <failed-unit-name>
# by openbao-alert@.service, which the timer units reference via OnFailure=.
#
# Backups died silently for 24 days (2026-07-29..08-22) because a failed
# oneshot unit makes no noise. This makes it make noise.
#
# CIRCULAR-DEPENDENCY NOTE: the SMTP password lives in OpenBAO, but the most
# likely reason one of these units failed is that OpenBAO itself is down or
# sealed -- in which case fetching the password would fail too, and the alert
# would be lost exactly when it matters most. So:
#   1. every alert is ALWAYS appended to $ALERT_LOG first, before any network
#      call, so a durable record exists even with no OpenBAO and no internet;
#   2. creds are fetched from OpenBAO when it is reachable, and cached to a
#      root-only 0600 file that is used as the fallback when it is not.
# The cache is a deliberate trade-off, consistent with the house rule that
# secrets live in root-owned 0600 files or come from the store at runtime.
set -uo pipefail

UNIT="${1:-unknown.unit}"
HOST="$(hostname -s 2>/dev/null || echo pi)"
TS="$(date '+%F %T %Z')"

BAO_ADDR="${BAO_ADDR:-http://127.0.0.1:8200}"
TOKEN_FILE="/etc/openbao-alert-smtp.token"
CACHE_FILE="/etc/openbao-smtp-cache"
ALERT_LOG="/var/log/openbao-alerts.log"

SMTP_HOST="securesmtp.t-online.de"; SMTP_PORT="587"
MAIL_FROM="lutz.finsterle@t-online.de"; MAIL_TO="lutz.finsterle@t-online.de"

# --- 1. gather context ------------------------------------------------------
STATUS="$(systemctl status --no-pager --full "$UNIT" 2>&1 | head -40)"
JOURNAL="$(journalctl -u "$UNIT" -n 40 --no-pager 2>&1 | tail -40)"
RESULT="$(systemctl show -p Result --value "$UNIT" 2>/dev/null)"
EXECMAIN="$(systemctl show -p ExecMainStatus --value "$UNIT" 2>/dev/null)"

BODY="OpenBAO maintenance unit FAILED on ${HOST}.

Unit:        ${UNIT}
When:        ${TS}
Result:      ${RESULT:-unknown} (exit status ${EXECMAIN:-?})

This unit is part of the OpenBAO safety net (raft snapshots, TLS cert renewal,
scoped-token renewal). A failure here is silent by default -- if you are reading
this, the alerting is doing its job. Investigate promptly: a lapsed token or a
missed snapshot degrades quietly and is easy to miss for weeks.

--- systemctl status ---
${STATUS}

--- last 40 journal lines ---
${JOURNAL}
"

# --- 2. durable local record FIRST (never depends on OpenBAO or the network) --
{
  echo "===== ${TS} ${UNIT} ====="
  printf '%s\n\n' "$BODY"
} >> "$ALERT_LOG" 2>/dev/null
chmod 0600 "$ALERT_LOG" 2>/dev/null || true

# --- 3. resolve SMTP creds: OpenBAO first, cached copy as fallback -----------
user=""; pass=""; cred_src=""

if [ -r "$TOKEN_FILE" ]; then
  tok="$(cat "$TOKEN_FILE" 2>/dev/null)"
  if [ -n "$tok" ]; then
    resp="$(curl -sS --max-time 10 -H "X-Vault-Token: $tok" \
              "$BAO_ADDR/v1/secret/data/smtp/healthcheck" 2>/dev/null)"
    user="$(printf '%s' "$resp" | jq -r '.data.data.username // empty' 2>/dev/null)"
    pass="$(printf '%s' "$resp" | jq -r '.data.data.password // empty' 2>/dev/null)"
    if [ -n "$user" ] && [ -n "$pass" ]; then
      cred_src="openbao"
      # refresh the offline fallback copy
      umask 077
      printf '%s\n%s\n' "$user" "$pass" > "${CACHE_FILE}.tmp" 2>/dev/null \
        && chmod 0600 "${CACHE_FILE}.tmp" 2>/dev/null \
        && mv -f "${CACHE_FILE}.tmp" "$CACHE_FILE" 2>/dev/null
    fi
  fi
fi

if { [ -z "$user" ] || [ -z "$pass" ]; } && [ -r "$CACHE_FILE" ]; then
  user="$(sed -n 1p "$CACHE_FILE" 2>/dev/null)"
  pass="$(sed -n 2p "$CACHE_FILE" 2>/dev/null)"
  cred_src="cache (OpenBAO unreachable -- it may itself be the problem)"
fi

if [ -z "$user" ] || [ -z "$pass" ]; then
  echo "${TS} ${UNIT}: ALERT EMAIL NOT SENT -- no SMTP creds from OpenBAO or cache" >> "$ALERT_LOG"
  logger -t openbao-alert "FAILED unit ${UNIT}; could not send email (no SMTP creds)"
  exit 0   # never fail the handler: that would just add noise, not signal
fi

# --- 4. send ----------------------------------------------------------------
msg="$(printf 'From: %s\r\nTo: %s\r\nSubject: [ALERT] %s: %s failed\r\nDate: %s\r\nContent-Type: text/plain; charset=UTF-8\r\n\r\n%s\r\n' \
      "$MAIL_FROM" "$MAIL_TO" "$HOST" "$UNIT" "$(date -R)" "$BODY (creds via ${cred_src})")"

if printf '%s' "$msg" | curl -sS --max-time 30 --ssl-reqd \
     --url "smtp://$SMTP_HOST:$SMTP_PORT" --user "$user:$pass" \
     --mail-from "$MAIL_FROM" --mail-rcpt "$MAIL_TO" --upload-file - 2>>"$ALERT_LOG"; then
  echo "${TS} ${UNIT}: alert email sent (creds via ${cred_src})" >> "$ALERT_LOG"
  logger -t openbao-alert "FAILED unit ${UNIT}; alert email sent"
else
  echo "${TS} ${UNIT}: ALERT EMAIL FAILED TO SEND (creds via ${cred_src})" >> "$ALERT_LOG"
  logger -t openbao-alert "FAILED unit ${UNIT}; alert email could NOT be sent"
fi

# Keep the log bounded.
tail -n 2000 "$ALERT_LOG" > "${ALERT_LOG}.tmp" 2>/dev/null && mv -f "${ALERT_LOG}.tmp" "$ALERT_LOG" 2>/dev/null
chmod 0600 "$ALERT_LOG" 2>/dev/null || true
exit 0