810d22f0a6
Migrates the last plaintext-at-rest monitoring backup off rclone-sync onto the same restic pipeline as loki/grafana/k8s-resources. Closes the Hetzner Object Storage gap flagged in DEV-482 — no SSE-S3/SSE-KMS means every object we push must be encrypted client-side. - apps/monitoring/prometheus-backup-cronjob.yaml: pin the image to the Harbor mirror `harbor.basicstack.de/library/restic:0.17.3` (DEV-493) so the pull path matches the sibling CronJobs. - docs/monitoring/restic-restore.md: append the DEV-492 drill entry — cold restic init, 8.017 GiB → 2.441 GiB stored, `restic check` clean, restore of `latest` into a scratch namespace, then `promtool tsdb list` + `promtool tsdb analyze` against all 19 blocks (0 failed). Confirms the compaction-race mitigation (`--exclude wal/*`/`chunks_head/*` + accept exit 3) does not leave a corrupt snapshot. Also documents the DEV-492 Prometheus repo in the layout table + per-repo restore commands. - apps/monitoring/README.md: point the restore section at all four repos and cite DEV-488 (loki/k8s) + DEV-492 (prometheus) drill entries. Definition of done (DEV-492): - backup ships client-side-encrypted to Hetzner: ✅ (restic init succeeded on the manual run; snapshot 71420465 written) - restore drill promtool tsdb analyze clean: ✅ (19/19 blocks OK) - OBSERVABILITY_BASELINE.md + apps/monitoring/README.md updated: ✅ - restic-password ownership documented: ✅ (README shared-SealedSecret table already covered all four repos after this change) Co-Authored-By: Paperclip <noreply@paperclip.ing>
228 lines
9.5 KiB
YAML
228 lines
9.5 KiB
YAML
---
|
|
# Prometheus data backup via restic to Hetzner Object Storage
|
|
# (DEV-492, DEV-482 Option 4). Replaces the DEV-465 rclone-sync job so
|
|
# every monitoring backup ships client-side-encrypted; Hetzner Object
|
|
# Storage has no SSE-S3/SSE-KMS, so the previous plaintext-at-rest
|
|
# object layout was the only remaining gap.
|
|
#
|
|
# Streams the RWO PVC `prometheus-data-encrypted` (mounted read-only)
|
|
# into `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/prometheus`, a client-side
|
|
# encrypted restic repository (tag=`prometheus`, host=`k3s`).
|
|
#
|
|
# Node scheduling matches the previous job: podAffinity co-schedules
|
|
# with the Prometheus pod (app=prometheus, topology
|
|
# kubernetes.io/hostname). Hetzner CSI RWO permits additional read-only
|
|
# mounts on the node that holds the PVC's VolumeAttachment, so this
|
|
# survives Prometheus being rescheduled to a different worker.
|
|
#
|
|
# Prometheus TSDB compaction race
|
|
# --------------------------------
|
|
# Prometheus rewrites the on-disk store roughly every 2 h: it creates a
|
|
# new block dir, then deletes the source dirs. restic walks the source
|
|
# tree once and may catch a file that disappeared mid-walk; restic
|
|
# 0.17.3 exits 3 ("at least one source file could not be read") in
|
|
# that case, and the snapshot excludes only the missing file. The
|
|
# next daily run picks up the successor block, so the race is not a
|
|
# data-loss risk — but we must not treat exit 3 as a hard failure, or
|
|
# the daily job will alert-flap.
|
|
#
|
|
# Mitigation:
|
|
# - exclude `wal/*` (WAL is replayed from a fresh instance on
|
|
# restart; we accept losing the last ~15 s of ingested samples
|
|
# rather than snapshotting a moving segment)
|
|
# - exclude `chunks_head/*` (in-memory head block; ephemeral, would
|
|
# be rebuilt from WAL which we do not keep)
|
|
# - exclude Prometheus lock/scratch files (`lock`, `queries.active`,
|
|
# `*.tmp`, `lost+found/*`)
|
|
# - treat restic exit code 3 as a soft warning (log, continue);
|
|
# any other non-zero exit is still fatal
|
|
# - restore drill re-runs `promtool tsdb analyze` against every
|
|
# block so a corrupted snapshot is caught end-to-end
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: prometheus-backup
|
|
namespace: monitoring
|
|
labels:
|
|
app: backup
|
|
type: prometheus
|
|
backend: restic
|
|
spec:
|
|
schedule: "30 3 * * *" # daily 03:30, offset from loki/grafana/k8s-resources
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
type: prometheus
|
|
backend: restic
|
|
spec:
|
|
backoffLimit: 2
|
|
activeDeadlineSeconds: 3600
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
type: prometheus
|
|
backend: restic
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
affinity:
|
|
podAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchExpressions:
|
|
- key: app
|
|
operator: In
|
|
values:
|
|
- prometheus
|
|
topologyKey: kubernetes.io/hostname
|
|
containers:
|
|
- name: restic
|
|
image: harbor.basicstack.de/library/restic:0.17.3
|
|
env:
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: access-key
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: secret-key
|
|
- name: RESTIC_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: restic-password
|
|
- name: S3_ENDPOINT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: endpoint
|
|
- name: S3_BUCKET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: bucket
|
|
- name: RESTIC_REPOSITORY
|
|
value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/prometheus"
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -eu
|
|
echo "=== backup-prometheus-restic started at $(date -u +%FT%TZ) ==="
|
|
echo "Repository: ${RESTIC_REPOSITORY}"
|
|
|
|
# First-run tolerance: init if the repo isn't there yet.
|
|
if restic snapshots >/dev/null 2>&1; then
|
|
echo "Repo exists, skipping init."
|
|
else
|
|
echo "Repo missing, initialising..."
|
|
restic init
|
|
fi
|
|
|
|
# See "Prometheus TSDB compaction race" in the file
|
|
# header. WAL + chunks_head are excluded on purpose;
|
|
# exit code 3 (source file vanished mid-walk during a
|
|
# compaction) is accepted and reported, any other
|
|
# non-zero exit is fatal.
|
|
echo "--- restic backup /source (exclude wal/chunks_head + lock files) ---"
|
|
BACKUP_STATUS=0
|
|
restic backup /source \
|
|
--tag prometheus \
|
|
--host k3s \
|
|
--exclude 'wal/*' \
|
|
--exclude 'chunks_head/*' \
|
|
--exclude 'lock' \
|
|
--exclude 'queries.active' \
|
|
--exclude 'lost+found/*' \
|
|
--exclude '*.tmp' || BACKUP_STATUS=$?
|
|
echo "restic backup exit: ${BACKUP_STATUS}"
|
|
if [ "${BACKUP_STATUS}" -eq 0 ]; then
|
|
echo "backup: all files captured cleanly"
|
|
elif [ "${BACKUP_STATUS}" -eq 3 ]; then
|
|
echo "backup: exit 3 (source files vanished mid-walk) — expected under Prometheus compaction, continuing"
|
|
else
|
|
echo "backup: FATAL exit ${BACKUP_STATUS} (not compaction-race)"
|
|
exit ${BACKUP_STATUS}
|
|
fi
|
|
|
|
echo "--- restic forget/prune ---"
|
|
restic forget --tag prometheus \
|
|
--keep-daily 7 \
|
|
--keep-weekly 4 \
|
|
--keep-monthly 6 \
|
|
--prune
|
|
|
|
echo "--- restic check --read-data-subset=5% ---"
|
|
CHECK_STATUS=0
|
|
restic check --read-data-subset=5% || CHECK_STATUS=$?
|
|
echo "restic check exit: ${CHECK_STATUS}"
|
|
|
|
echo "--- restic stats (repo size) ---"
|
|
REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
|
|
| grep -oE '"total_size":[0-9]+' \
|
|
| head -1 \
|
|
| cut -d: -f2)
|
|
REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
|
|
echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
|
|
|
|
# Textfile-collector metrics. Same hostPath pattern as
|
|
# the loki/grafana/k8s-resources siblings (DEV-494) —
|
|
# written atomically via `.tmp` + rename so a mid-write
|
|
# read never surfaces a truncated sample.
|
|
# backup_prometheus_success rolls in both stages: the
|
|
# backup step (accepting exit 3) and restic check.
|
|
BACKUP_OK=0
|
|
if [ "${BACKUP_STATUS}" -eq 0 ] || [ "${BACKUP_STATUS}" -eq 3 ]; then
|
|
BACKUP_OK=1
|
|
fi
|
|
SUCCESS=0
|
|
if [ "${BACKUP_OK}" -eq 1 ] && [ "${CHECK_STATUS}" -eq 0 ]; then
|
|
SUCCESS=1
|
|
fi
|
|
{
|
|
echo "backup_prometheus_success ${SUCCESS}"
|
|
echo "backup_prometheus_timestamp_seconds $(date +%s)"
|
|
echo "backup_prometheus_check_status ${CHECK_STATUS}"
|
|
echo "backup_prometheus_backup_status ${BACKUP_STATUS}"
|
|
echo "restic_repo_size_bytes{repo=\"prometheus\"} ${REPO_SIZE_BYTES}"
|
|
} > /metrics/backup_prometheus.prom.tmp
|
|
mv /metrics/backup_prometheus.prom.tmp /metrics/backup_prometheus.prom
|
|
|
|
echo "=== backup-prometheus-restic finished at $(date -u +%FT%TZ) ==="
|
|
exit ${CHECK_STATUS}
|
|
volumeMounts:
|
|
- name: prometheus-data
|
|
mountPath: /source
|
|
readOnly: true
|
|
- name: metrics
|
|
mountPath: /metrics
|
|
- name: cache
|
|
mountPath: /root/.cache/restic
|
|
resources:
|
|
# Prometheus TSDB is ~8-10 GiB; give restic room to
|
|
# burst during pack/check but keep steady-state small.
|
|
requests:
|
|
cpu: 100m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: 1500m
|
|
memory: 1Gi
|
|
volumes:
|
|
- name: prometheus-data
|
|
persistentVolumeClaim:
|
|
claimName: prometheus-data-encrypted
|
|
- name: metrics
|
|
hostPath:
|
|
# node-exporter's textfile-collector directory
|
|
# (DEV-494). See sibling loki cronjob for detail.
|
|
path: /var/lib/node_exporter/textfile_collector
|
|
type: DirectoryOrCreate
|
|
- name: cache
|
|
emptyDir: {}
|