a5ceedf4da
The original `rclone delete --min-age 7d` step ate the freshly uploaded backup: rclone preserves each source file's mtime, and Prometheus TSDB chunk/block files retain very old mtimes (weeks-old immutable blocks), so filtering by file age deleted ~72% of the objects immediately after sync (verified: 8.764 GiB destination reduced to 2.310 GiB / 37 objects before we noticed). Replaced the mtime prune with a prefix-name-based prune: every top-level prefix is `prometheus-YYYYMMDD-HHMMSS`, so we parse the encoded date and `rclone purge` whole prefixes older than 7 days. This keeps the latest 7 daily snapshots intact regardless of the Prometheus block ages. Co-Authored-By: Paperclip <noreply@paperclip.ing>
211 lines
9.5 KiB
YAML
211 lines
9.5 KiB
YAML
---
|
|
# Prometheus data backup (DEV-465).
|
|
#
|
|
# Prometheus data lives on the RWO PVC `prometheus-data-encrypted` in
|
|
# namespace `monitoring`. That PVC is mounted by the Prometheus pod which
|
|
# currently lives on k3s-worker-1, and the Hetzner CSI volume can only be
|
|
# attached to one node at a time. The shared `backup-volumes` CronJob (see
|
|
# `apps/monitoring/backup-volumes-cronjob.yaml`) is pinned to k3s-worker-2
|
|
# (where grafana + loki live) and therefore cannot back up Prometheus.
|
|
#
|
|
# This CronJob co-schedules with the Prometheus pod via podAffinity, so it
|
|
# lands on whichever node currently holds `prometheus-data-encrypted`. The
|
|
# PVC is mounted read-only alongside the running Prometheus pod (RWO permits
|
|
# additional read-only mounts on the same node) and streamed to Hetzner S3
|
|
# via rclone under `basicstack-backup/prometheus/prometheus-<DATE>/`. Old
|
|
# snapshots are pruned after 7 days.
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: prometheus-backup
|
|
namespace: monitoring
|
|
labels:
|
|
app: backup
|
|
type: prometheus
|
|
spec:
|
|
schedule: "30 3 * * *" # daily 03:30, offset from backup-volumes (03:00)
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
type: prometheus
|
|
spec:
|
|
backoffLimit: 2
|
|
activeDeadlineSeconds: 3600 # 60 min hard cap; Prometheus data is ~9GB
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
type: prometheus
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
# Co-schedule with the Prometheus pod so the RWO PVC attaches on
|
|
# the same node. This survives Prometheus being rescheduled to a
|
|
# different worker (the backup follows).
|
|
affinity:
|
|
podAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchExpressions:
|
|
- key: app
|
|
operator: In
|
|
values:
|
|
- prometheus
|
|
topologyKey: kubernetes.io/hostname
|
|
containers:
|
|
- name: prometheus-backup
|
|
image: rclone/rclone:1.68
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -eu
|
|
DATE=$(date +%Y%m%d-%H%M%S)
|
|
echo "=== Prometheus backup started at $(date) (target prefix: prometheus-${DATE}) ==="
|
|
|
|
mkdir -p /root/.config/rclone
|
|
cat > /root/.config/rclone/rclone.conf <<EOC
|
|
[hetzner-s3]
|
|
type = s3
|
|
provider = Other
|
|
access_key_id = ${S3_ACCESS_KEY}
|
|
secret_access_key = ${S3_SECRET_KEY}
|
|
endpoint = ${S3_ENDPOINT}
|
|
acl = private
|
|
EOC
|
|
|
|
SOURCE_BYTES=$(du -sb /source 2>/dev/null | cut -f1 || echo 0)
|
|
SOURCE_HUMAN=$(du -sh /source 2>/dev/null | cut -f1 || echo unknown)
|
|
echo "Source /source size: ${SOURCE_HUMAN} (${SOURCE_BYTES} bytes)"
|
|
|
|
# Prometheus TSDB is largely immutable chunk files plus an
|
|
# append-only WAL. rclone sync is safe with the database
|
|
# live: on restore the WAL is replayed. HOWEVER Prometheus
|
|
# compacts blocks every ~2h and deletes their source dirs,
|
|
# which races with the copy and produces "no such file or
|
|
# directory" errors mid-run. Those are expected and do
|
|
# not indicate data loss — the compacted successor blocks
|
|
# are picked up on the same or the next daily run. We
|
|
# therefore do not fail the job on rclone's non-zero exit
|
|
# from those transient errors; instead we validate the
|
|
# backup by comparing destination size to source (must be
|
|
# >= 80% of source bytes and > 100 MiB).
|
|
DEST="hetzner-s3:${S3_BUCKET}/prometheus/prometheus-${DATE}/"
|
|
echo "Streaming Prometheus data to ${DEST} ..."
|
|
RCLONE_EXIT=0
|
|
rclone sync /source "${DEST}" \
|
|
--transfers 4 \
|
|
--checkers 4 \
|
|
--stats 30s \
|
|
--stats-log-level NOTICE \
|
|
--s3-chunk-size 32M \
|
|
--s3-upload-concurrency 4 \
|
|
--retries 3 \
|
|
--retries-sleep 30s || RCLONE_EXIT=$?
|
|
echo "rclone sync exit code: ${RCLONE_EXIT}"
|
|
|
|
echo "Measuring destination size..."
|
|
DEST_BYTES=$(rclone size "${DEST}" --json 2>/dev/null | \
|
|
sed -n 's/.*"bytes":\s*\([0-9]\+\).*/\1/p' | head -1)
|
|
DEST_BYTES=${DEST_BYTES:-0}
|
|
DEST_HUMAN=$(rclone size "${DEST}" 2>/dev/null | \
|
|
grep -oE 'Total size:.*' || echo "Total size: unknown")
|
|
echo "Destination bytes: ${DEST_BYTES}"
|
|
echo "Destination summary: ${DEST_HUMAN}"
|
|
|
|
MIN_ACCEPTABLE=$(( SOURCE_BYTES * 80 / 100 ))
|
|
FLOOR=104857600 # 100 MiB absolute floor
|
|
echo "Acceptance threshold: dest >= ${MIN_ACCEPTABLE} bytes and > ${FLOOR} bytes"
|
|
if [ "${DEST_BYTES}" -lt "${FLOOR}" ]; then
|
|
echo "ERROR: destination is below hard floor (100 MiB) — backup failed."
|
|
exit 2
|
|
fi
|
|
if [ "${DEST_BYTES}" -lt "${MIN_ACCEPTABLE}" ]; then
|
|
echo "ERROR: destination is < 80% of source (${DEST_BYTES} < ${MIN_ACCEPTABLE}) — backup incomplete."
|
|
exit 3
|
|
fi
|
|
echo "OK: destination size acceptable."
|
|
|
|
# Prune by prefix name, NOT by file mtime. rclone preserves
|
|
# each source file's mtime on upload, and Prometheus TSDB
|
|
# chunk files retain very old mtimes (weeks-old blocks),
|
|
# so `rclone delete --min-age 7d` would eat the just-
|
|
# uploaded backup. Every top-level prefix is named
|
|
# `prometheus-YYYYMMDD-HHMMSS`, so we compare the encoded
|
|
# date to a 7-day threshold and purge whole prefixes.
|
|
echo "Pruning prometheus backup prefixes older than 7 days..."
|
|
# BusyBox date lacks GNU's `-d "7 days ago"` and BSD's `-v -7d`,
|
|
# so compute the cutoff via @epoch which BusyBox does support.
|
|
CUTOFF_EPOCH=$(( $(date +%s) - 7 * 86400 ))
|
|
CUTOFF=$(date -d "@${CUTOFF_EPOCH}" +%Y%m%d)
|
|
echo "Cutoff (delete prefixes with date < ${CUTOFF}):"
|
|
KEEP=0
|
|
PRUNE=0
|
|
for PREFIX in $(rclone lsf --dirs-only "hetzner-s3:${S3_BUCKET}/prometheus/" 2>/dev/null | sed 's:/$::'); do
|
|
case "${PREFIX}" in
|
|
prometheus-*)
|
|
PDATE=$(echo "${PREFIX}" | sed -n 's/^prometheus-\([0-9]\{8\}\)-.*/\1/p')
|
|
if [ -z "${PDATE}" ]; then
|
|
echo " SKIP ${PREFIX} (unparseable name)"
|
|
continue
|
|
fi
|
|
if [ "${PDATE}" -lt "${CUTOFF}" ]; then
|
|
echo " PURGE ${PREFIX} (date ${PDATE} < ${CUTOFF})"
|
|
rclone purge "hetzner-s3:${S3_BUCKET}/prometheus/${PREFIX}" \
|
|
|| echo " WARN purge failed for ${PREFIX} (continuing)"
|
|
PRUNE=$((PRUNE + 1))
|
|
else
|
|
echo " KEEP ${PREFIX} (date ${PDATE})"
|
|
KEEP=$((KEEP + 1))
|
|
fi
|
|
;;
|
|
*)
|
|
echo " SKIP ${PREFIX} (not a prometheus- prefix)"
|
|
;;
|
|
esac
|
|
done
|
|
echo "Prune summary: kept ${KEEP}, purged ${PRUNE}."
|
|
|
|
echo "Post-run inventory (prometheus/ prefixes):"
|
|
rclone lsd "hetzner-s3:${S3_BUCKET}/prometheus/" || true
|
|
|
|
echo "=== Prometheus backup completed at $(date) ==="
|
|
env:
|
|
- name: S3_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: access-key
|
|
- name: S3_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: secret-key
|
|
- name: S3_ENDPOINT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: endpoint
|
|
- name: S3_BUCKET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: bucket
|
|
volumeMounts:
|
|
- name: prometheus-data
|
|
mountPath: /source
|
|
readOnly: true
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 1000m
|
|
memory: 512Mi
|
|
volumes:
|
|
- name: prometheus-data
|
|
persistentVolumeClaim:
|
|
claimName: prometheus-data-encrypted
|