63aa116ef1
After the DEV-464 split, `monitoring/backup-volumes` only backs up grafana + loki (pinned to k3s-worker-2 with `backup-storage`), leaving Prometheus data unbacked. `prometheus-data-encrypted` is an RWO Hetzner Cloud volume attached to whichever node currently runs the Prometheus pod (typically k3s-worker-1), so it cannot join the shared backup-volumes job without provoking Multi-Attach errors. This introduces a dedicated `monitoring/prometheus-backup` CronJob that: - Streams `prometheus-data-encrypted` to Hetzner S3 via rclone (`basicstack-backup/prometheus/prometheus-<DATE>/`). - Uses `podAffinity` to co-schedule with the Prometheus pod so the RWO PVC always attaches on the same node. - Runs at 03:30 daily, `Forbid` concurrency, 60m hard deadline. - Retains 7 days of dated backups (rclone delete --min-age 7d). - Tolerates the expected TSDB compaction race (Prometheus deletes old block dirs mid-copy): rclone's non-zero exit from those transient errors is captured, then success is validated by comparing dest bytes to source bytes (>= 80% and > 100 MiB floor). S3 credentials are the same Hetzner Object Storage account used by `opencloud-backup` and `stalwart-backup`, resealed for the `monitoring` namespace as `SealedSecret monitoring-s3-backup`. Verified with a manual job on k3s-worker-1 (see DEV-465 for logs). Co-Authored-By: Paperclip <noreply@paperclip.ing>
176 lines
7.5 KiB
YAML
176 lines
7.5 KiB
YAML
---
|
|
# Prometheus data backup (DEV-465).
|
|
#
|
|
# Prometheus data lives on the RWO PVC `prometheus-data-encrypted` in
|
|
# namespace `monitoring`. That PVC is mounted by the Prometheus pod which
|
|
# currently lives on k3s-worker-1, and the Hetzner CSI volume can only be
|
|
# attached to one node at a time. The shared `backup-volumes` CronJob (see
|
|
# `apps/monitoring/backup-volumes-cronjob.yaml`) is pinned to k3s-worker-2
|
|
# (where grafana + loki live) and therefore cannot back up Prometheus.
|
|
#
|
|
# This CronJob co-schedules with the Prometheus pod via podAffinity, so it
|
|
# lands on whichever node currently holds `prometheus-data-encrypted`. The
|
|
# PVC is mounted read-only alongside the running Prometheus pod (RWO permits
|
|
# additional read-only mounts on the same node) and streamed to Hetzner S3
|
|
# via rclone under `basicstack-backup/prometheus/prometheus-<DATE>/`. Old
|
|
# snapshots are pruned after 7 days.
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: prometheus-backup
|
|
namespace: monitoring
|
|
labels:
|
|
app: backup
|
|
type: prometheus
|
|
spec:
|
|
schedule: "30 3 * * *" # daily 03:30, offset from backup-volumes (03:00)
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
type: prometheus
|
|
spec:
|
|
backoffLimit: 2
|
|
activeDeadlineSeconds: 3600 # 60 min hard cap; Prometheus data is ~9GB
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
type: prometheus
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
# Co-schedule with the Prometheus pod so the RWO PVC attaches on
|
|
# the same node. This survives Prometheus being rescheduled to a
|
|
# different worker (the backup follows).
|
|
affinity:
|
|
podAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchExpressions:
|
|
- key: app
|
|
operator: In
|
|
values:
|
|
- prometheus
|
|
topologyKey: kubernetes.io/hostname
|
|
containers:
|
|
- name: prometheus-backup
|
|
image: rclone/rclone:1.68
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -eu
|
|
DATE=$(date +%Y%m%d-%H%M%S)
|
|
echo "=== Prometheus backup started at $(date) (target prefix: prometheus-${DATE}) ==="
|
|
|
|
mkdir -p /root/.config/rclone
|
|
cat > /root/.config/rclone/rclone.conf <<EOC
|
|
[hetzner-s3]
|
|
type = s3
|
|
provider = Other
|
|
access_key_id = ${S3_ACCESS_KEY}
|
|
secret_access_key = ${S3_SECRET_KEY}
|
|
endpoint = ${S3_ENDPOINT}
|
|
acl = private
|
|
EOC
|
|
|
|
SOURCE_BYTES=$(du -sb /source 2>/dev/null | cut -f1 || echo 0)
|
|
SOURCE_HUMAN=$(du -sh /source 2>/dev/null | cut -f1 || echo unknown)
|
|
echo "Source /source size: ${SOURCE_HUMAN} (${SOURCE_BYTES} bytes)"
|
|
|
|
# Prometheus TSDB is largely immutable chunk files plus an
|
|
# append-only WAL. rclone sync is safe with the database
|
|
# live: on restore the WAL is replayed. HOWEVER Prometheus
|
|
# compacts blocks every ~2h and deletes their source dirs,
|
|
# which races with the copy and produces "no such file or
|
|
# directory" errors mid-run. Those are expected and do
|
|
# not indicate data loss — the compacted successor blocks
|
|
# are picked up on the same or the next daily run. We
|
|
# therefore do not fail the job on rclone's non-zero exit
|
|
# from those transient errors; instead we validate the
|
|
# backup by comparing destination size to source (must be
|
|
# >= 80% of source bytes and > 100 MiB).
|
|
DEST="hetzner-s3:${S3_BUCKET}/prometheus/prometheus-${DATE}/"
|
|
echo "Streaming Prometheus data to ${DEST} ..."
|
|
RCLONE_EXIT=0
|
|
rclone sync /source "${DEST}" \
|
|
--transfers 4 \
|
|
--checkers 4 \
|
|
--stats 30s \
|
|
--stats-log-level NOTICE \
|
|
--s3-chunk-size 32M \
|
|
--s3-upload-concurrency 4 \
|
|
--retries 3 \
|
|
--retries-sleep 30s || RCLONE_EXIT=$?
|
|
echo "rclone sync exit code: ${RCLONE_EXIT}"
|
|
|
|
echo "Measuring destination size..."
|
|
DEST_BYTES=$(rclone size "${DEST}" --json 2>/dev/null | \
|
|
sed -n 's/.*"bytes":\s*\([0-9]\+\).*/\1/p' | head -1)
|
|
DEST_BYTES=${DEST_BYTES:-0}
|
|
DEST_HUMAN=$(rclone size "${DEST}" 2>/dev/null | \
|
|
grep -oE 'Total size:.*' || echo "Total size: unknown")
|
|
echo "Destination bytes: ${DEST_BYTES}"
|
|
echo "Destination summary: ${DEST_HUMAN}"
|
|
|
|
MIN_ACCEPTABLE=$(( SOURCE_BYTES * 80 / 100 ))
|
|
FLOOR=104857600 # 100 MiB absolute floor
|
|
echo "Acceptance threshold: dest >= ${MIN_ACCEPTABLE} bytes and > ${FLOOR} bytes"
|
|
if [ "${DEST_BYTES}" -lt "${FLOOR}" ]; then
|
|
echo "ERROR: destination is below hard floor (100 MiB) — backup failed."
|
|
exit 2
|
|
fi
|
|
if [ "${DEST_BYTES}" -lt "${MIN_ACCEPTABLE}" ]; then
|
|
echo "ERROR: destination is < 80% of source (${DEST_BYTES} < ${MIN_ACCEPTABLE}) — backup incomplete."
|
|
exit 3
|
|
fi
|
|
echo "OK: destination size acceptable."
|
|
|
|
echo "Pruning prometheus backups older than 7 days..."
|
|
rclone delete "hetzner-s3:${S3_BUCKET}/prometheus/" \
|
|
--min-age 7d || echo "prune step reported errors (continuing)"
|
|
rclone rmdirs "hetzner-s3:${S3_BUCKET}/prometheus/" --leave-root || true
|
|
|
|
echo "Post-run inventory (prometheus/ prefixes):"
|
|
rclone lsd "hetzner-s3:${S3_BUCKET}/prometheus/" || true
|
|
|
|
echo "=== Prometheus backup completed at $(date) ==="
|
|
env:
|
|
- name: S3_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: access-key
|
|
- name: S3_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: secret-key
|
|
- name: S3_ENDPOINT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: endpoint
|
|
- name: S3_BUCKET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: bucket
|
|
volumeMounts:
|
|
- name: prometheus-data
|
|
mountPath: /source
|
|
readOnly: true
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 1000m
|
|
memory: 512Mi
|
|
volumes:
|
|
- name: prometheus-data
|
|
persistentVolumeClaim:
|
|
claimName: prometheus-data-encrypted
|