From ee0b8c818f7c2101fa336ce590be7522f80b8589 Mon Sep 17 00:00:00 2001 From: Lutz Finsterle Date: Sat, 22 Aug 2026 09:22:05 +0200 Subject: [PATCH] Alert by email when an OpenBAO maintenance unit fails A failed oneshot unit is silent by default, which is why the backup failures went unnoticed for 24 days. All three maintenance units now carry OnFailure=openbao-alert@%n.service, which mails the failure with systemctl status and the last 40 journal lines. Reuses the existing ClaudeAdmin notifier pattern: curl --ssl-reqd to securesmtp.t-online.de:587, password read from KV secret/data/smtp/healthcheck via a new root-owned periodic token /etc/openbao-alert-smtp.token bound to smtp-healthcheck-ro. That path matches /etc/openbao-*.token, so the token-renew loop picks it up automatically and it cannot lapse the way the others did (verified: renewed=9 failed=0). Two failure modes designed around: - Circular dependency. The SMTP password lives in OpenBAO, but the likeliest cause of these units failing is OpenBAO being down or sealed -- so fetching the password would fail exactly when the alert matters most. The alert is therefore always written to /var/log/openbao-alerts.log (0600) BEFORE any network call, and successful fetches refresh a root-only 0600 credential cache used as fallback. Verified with BAO_ADDR pointed at a dead port: the email still went out, tagged as sent via cache. - Recursion. The alert unit has no OnFailure of its own and always exits 0; a send failure is logged and syslogged rather than raised. Uses %i, not %I: unit names contain '-', which systemd unescaping would turn into '/' (openbao-backup.service -> openbao/backup.service). Verified end to end with a throwaway failing unit (since removed): failure -> OnFailure -> alert unit -> email delivered. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NFtVLA7VVqXL5G2S18c4Jk --- scripts/send-failure-alert.sh | 114 ++++++++++++++++++++++++++++ systemd/openbao-alert@.service | 13 ++++ systemd/openbao-backup.service | 1 + systemd/openbao-cert-renew.service | 1 + systemd/openbao-token-renew.service | 1 + 5 files changed, 130 insertions(+) create mode 100755 scripts/send-failure-alert.sh create mode 100644 systemd/openbao-alert@.service diff --git a/scripts/send-failure-alert.sh b/scripts/send-failure-alert.sh new file mode 100755 index 0000000..4371ecf --- /dev/null +++ b/scripts/send-failure-alert.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# OnFailure handler for the OpenBAO systemd timers. Invoked as: +# send-failure-alert.sh +# by openbao-alert@.service, which the timer units reference via OnFailure=. +# +# Backups died silently for 24 days (2026-07-29..08-22) because a failed +# oneshot unit makes no noise. This makes it make noise. +# +# CIRCULAR-DEPENDENCY NOTE: the SMTP password lives in OpenBAO, but the most +# likely reason one of these units failed is that OpenBAO itself is down or +# sealed -- in which case fetching the password would fail too, and the alert +# would be lost exactly when it matters most. So: +# 1. every alert is ALWAYS appended to $ALERT_LOG first, before any network +# call, so a durable record exists even with no OpenBAO and no internet; +# 2. creds are fetched from OpenBAO when it is reachable, and cached to a +# root-only 0600 file that is used as the fallback when it is not. +# The cache is a deliberate trade-off, consistent with the house rule that +# secrets live in root-owned 0600 files or come from the store at runtime. +set -uo pipefail + +UNIT="${1:-unknown.unit}" +HOST="$(hostname -s 2>/dev/null || echo pi)" +TS="$(date '+%F %T %Z')" + +BAO_ADDR="${BAO_ADDR:-http://127.0.0.1:8200}" +TOKEN_FILE="/etc/openbao-alert-smtp.token" +CACHE_FILE="/etc/openbao-smtp-cache" +ALERT_LOG="/var/log/openbao-alerts.log" + +SMTP_HOST="securesmtp.t-online.de"; SMTP_PORT="587" +MAIL_FROM="lutz.finsterle@t-online.de"; MAIL_TO="lutz.finsterle@t-online.de" + +# --- 1. gather context ------------------------------------------------------ +STATUS="$(systemctl status --no-pager --full "$UNIT" 2>&1 | head -40)" +JOURNAL="$(journalctl -u "$UNIT" -n 40 --no-pager 2>&1 | tail -40)" +RESULT="$(systemctl show -p Result --value "$UNIT" 2>/dev/null)" +EXECMAIN="$(systemctl show -p ExecMainStatus --value "$UNIT" 2>/dev/null)" + +BODY="OpenBAO maintenance unit FAILED on ${HOST}. + +Unit: ${UNIT} +When: ${TS} +Result: ${RESULT:-unknown} (exit status ${EXECMAIN:-?}) + +This unit is part of the OpenBAO safety net (raft snapshots, TLS cert renewal, +scoped-token renewal). A failure here is silent by default -- if you are reading +this, the alerting is doing its job. Investigate promptly: a lapsed token or a +missed snapshot degrades quietly and is easy to miss for weeks. + +--- systemctl status --- +${STATUS} + +--- last 40 journal lines --- +${JOURNAL} +" + +# --- 2. durable local record FIRST (never depends on OpenBAO or the network) -- +{ + echo "===== ${TS} ${UNIT} =====" + printf '%s\n\n' "$BODY" +} >> "$ALERT_LOG" 2>/dev/null +chmod 0600 "$ALERT_LOG" 2>/dev/null || true + +# --- 3. resolve SMTP creds: OpenBAO first, cached copy as fallback ----------- +user=""; pass=""; cred_src="" + +if [ -r "$TOKEN_FILE" ]; then + tok="$(cat "$TOKEN_FILE" 2>/dev/null)" + if [ -n "$tok" ]; then + resp="$(curl -sS --max-time 10 -H "X-Vault-Token: $tok" \ + "$BAO_ADDR/v1/secret/data/smtp/healthcheck" 2>/dev/null)" + user="$(printf '%s' "$resp" | jq -r '.data.data.username // empty' 2>/dev/null)" + pass="$(printf '%s' "$resp" | jq -r '.data.data.password // empty' 2>/dev/null)" + if [ -n "$user" ] && [ -n "$pass" ]; then + cred_src="openbao" + # refresh the offline fallback copy + umask 077 + printf '%s\n%s\n' "$user" "$pass" > "${CACHE_FILE}.tmp" 2>/dev/null \ + && chmod 0600 "${CACHE_FILE}.tmp" 2>/dev/null \ + && mv -f "${CACHE_FILE}.tmp" "$CACHE_FILE" 2>/dev/null + fi + fi +fi + +if { [ -z "$user" ] || [ -z "$pass" ]; } && [ -r "$CACHE_FILE" ]; then + user="$(sed -n 1p "$CACHE_FILE" 2>/dev/null)" + pass="$(sed -n 2p "$CACHE_FILE" 2>/dev/null)" + cred_src="cache (OpenBAO unreachable -- it may itself be the problem)" +fi + +if [ -z "$user" ] || [ -z "$pass" ]; then + echo "${TS} ${UNIT}: ALERT EMAIL NOT SENT -- no SMTP creds from OpenBAO or cache" >> "$ALERT_LOG" + logger -t openbao-alert "FAILED unit ${UNIT}; could not send email (no SMTP creds)" + exit 0 # never fail the handler: that would just add noise, not signal +fi + +# --- 4. send ---------------------------------------------------------------- +msg="$(printf 'From: %s\r\nTo: %s\r\nSubject: [ALERT] %s: %s failed\r\nDate: %s\r\nContent-Type: text/plain; charset=UTF-8\r\n\r\n%s\r\n' \ + "$MAIL_FROM" "$MAIL_TO" "$HOST" "$UNIT" "$(date -R)" "$BODY (creds via ${cred_src})")" + +if printf '%s' "$msg" | curl -sS --max-time 30 --ssl-reqd \ + --url "smtp://$SMTP_HOST:$SMTP_PORT" --user "$user:$pass" \ + --mail-from "$MAIL_FROM" --mail-rcpt "$MAIL_TO" --upload-file - 2>>"$ALERT_LOG"; then + echo "${TS} ${UNIT}: alert email sent (creds via ${cred_src})" >> "$ALERT_LOG" + logger -t openbao-alert "FAILED unit ${UNIT}; alert email sent" +else + echo "${TS} ${UNIT}: ALERT EMAIL FAILED TO SEND (creds via ${cred_src})" >> "$ALERT_LOG" + logger -t openbao-alert "FAILED unit ${UNIT}; alert email could NOT be sent" +fi + +# Keep the log bounded. +tail -n 2000 "$ALERT_LOG" > "${ALERT_LOG}.tmp" 2>/dev/null && mv -f "${ALERT_LOG}.tmp" "$ALERT_LOG" 2>/dev/null +chmod 0600 "$ALERT_LOG" 2>/dev/null || true +exit 0 diff --git a/systemd/openbao-alert@.service b/systemd/openbao-alert@.service new file mode 100644 index 0000000..ad832ec --- /dev/null +++ b/systemd/openbao-alert@.service @@ -0,0 +1,13 @@ +[Unit] +Description=Email alert for failed OpenBAO maintenance unit %i +# Deliberately no OnFailure= here: if the alerter itself fails it must not +# recurse. It exits 0 on send failure and logs to /var/log/openbao-alerts.log. +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +# %i (escaped instance), NOT %I: unit names contain '-', which systemd's +# unescaping would turn into '/' (openbao-backup.service -> openbao/backup). +ExecStart=/home/lutz/Projects/OpenBAO/scripts/send-failure-alert.sh %i +# Runs as root: reads /etc/openbao-alert-smtp.token and journalctl -u. diff --git a/systemd/openbao-backup.service b/systemd/openbao-backup.service index 991dfdc..9056a3f 100644 --- a/systemd/openbao-backup.service +++ b/systemd/openbao-backup.service @@ -3,6 +3,7 @@ Description=Raft snapshot backup of both OpenBAO instances (main + unsealer) After=network-online.target docker.service Wants=network-online.target Requires=docker.service +OnFailure=openbao-alert@%n.service [Service] Type=oneshot diff --git a/systemd/openbao-cert-renew.service b/systemd/openbao-cert-renew.service index 4664168..1cca434 100644 --- a/systemd/openbao-cert-renew.service +++ b/systemd/openbao-cert-renew.service @@ -3,6 +3,7 @@ Description=Renew openbao.famfi.home cert from OpenBAO PKI and reload Traefik After=network-online.target docker.service Wants=network-online.target Requires=docker.service +OnFailure=openbao-alert@%n.service [Service] Type=oneshot diff --git a/systemd/openbao-token-renew.service b/systemd/openbao-token-renew.service index 4dd0f41..3c6439e 100644 --- a/systemd/openbao-token-renew.service +++ b/systemd/openbao-token-renew.service @@ -3,6 +3,7 @@ Description=Renew scoped OpenBAO app tokens so periodic tokens never lapse After=network-online.target docker.service Wants=network-online.target Requires=docker.service +OnFailure=openbao-alert@%n.service [Service] Type=oneshot