stack.basicstack.de/apps/monitoring/prometheus-backup-cronjob.yaml

---
# Prometheus data backup (DEV-465).
#
# Prometheus data lives on the RWO PVC `prometheus-data-encrypted` in
# namespace `monitoring`. That PVC is mounted by the Prometheus pod which
# currently lives on k3s-worker-1, and the Hetzner CSI volume can only be
# attached to one node at a time. The shared `backup-volumes` CronJob (see
# `apps/monitoring/backup-volumes-cronjob.yaml`) is pinned to k3s-worker-2
# (where grafana + loki live) and therefore cannot back up Prometheus.
#
# This CronJob co-schedules with the Prometheus pod via podAffinity, so it
# lands on whichever node currently holds `prometheus-data-encrypted`. The
# PVC is mounted read-only alongside the running Prometheus pod (RWO permits
# additional read-only mounts on the same node) and streamed to Hetzner S3
# via rclone under `basicstack-backup/prometheus/prometheus-<DATE>/`. Old
# snapshots are pruned after 7 days.
apiVersion: batch/v1
kind: CronJob
metadata:
  name: prometheus-backup
  namespace: monitoring
  labels:
    app: backup
    type: prometheus
spec:
  schedule: "30 3 * * *"  # daily 03:30, offset from backup-volumes (03:00)
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    metadata:
      labels:
        app: backup
        type: prometheus
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 3600  # 60 min hard cap; Prometheus data is ~9GB
      template:
        metadata:
          labels:
            app: backup
            type: prometheus
        spec:
          restartPolicy: OnFailure
          # Co-schedule with the Prometheus pod so the RWO PVC attaches on
          # the same node. This survives Prometheus being rescheduled to a
          # different worker (the backup follows).
          affinity:
            podAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                - labelSelector:
                    matchExpressions:
                      - key: app
                        operator: In
                        values:
                          - prometheus
                  topologyKey: kubernetes.io/hostname
          containers:
            - name: prometheus-backup
              image: rclone/rclone:1.68
              command:
                - /bin/sh
                - -c
                - |
                  set -eu
                  DATE=$(date +%Y%m%d-%H%M%S)
                  echo "=== Prometheus backup started at $(date) (target prefix: prometheus-${DATE}) ==="

                  mkdir -p /root/.config/rclone
                  cat > /root/.config/rclone/rclone.conf <<EOC
                  [hetzner-s3]
                  type = s3
                  provider = Other
                  access_key_id = ${S3_ACCESS_KEY}
                  secret_access_key = ${S3_SECRET_KEY}
                  endpoint = ${S3_ENDPOINT}
                  acl = private
                  EOC

                  SOURCE_BYTES=$(du -sb /source 2>/dev/null | cut -f1 || echo 0)
                  SOURCE_HUMAN=$(du -sh /source 2>/dev/null | cut -f1 || echo unknown)
                  echo "Source /source size: ${SOURCE_HUMAN} (${SOURCE_BYTES} bytes)"

                  # Prometheus TSDB is largely immutable chunk files plus an
                  # append-only WAL. rclone sync is safe with the database
                  # live: on restore the WAL is replayed. HOWEVER Prometheus
                  # compacts blocks every ~2h and deletes their source dirs,
                  # which races with the copy and produces "no such file or
                  # directory" errors mid-run. Those are expected and do
                  # not indicate data loss — the compacted successor blocks
                  # are picked up on the same or the next daily run. We
                  # therefore do not fail the job on rclone's non-zero exit
                  # from those transient errors; instead we validate the
                  # backup by comparing destination size to source (must be
                  # >= 80% of source bytes and > 100 MiB).
                  DEST="hetzner-s3:${S3_BUCKET}/prometheus/prometheus-${DATE}/"
                  echo "Streaming Prometheus data to ${DEST} ..."
                  RCLONE_EXIT=0
                  rclone sync /source "${DEST}" \
                    --transfers 4 \
                    --checkers 4 \
                    --stats 30s \
                    --stats-log-level NOTICE \
                    --s3-chunk-size 32M \
                    --s3-upload-concurrency 4 \
                    --retries 3 \
                    --retries-sleep 30s || RCLONE_EXIT=$?
                  echo "rclone sync exit code: ${RCLONE_EXIT}"

                  echo "Measuring destination size..."
                  DEST_BYTES=$(rclone size "${DEST}" --json 2>/dev/null | \
                    sed -n 's/.*"bytes":\s*\([0-9]\+\).*/\1/p' | head -1)
                  DEST_BYTES=${DEST_BYTES:-0}
                  DEST_HUMAN=$(rclone size "${DEST}" 2>/dev/null | \
                    grep -oE 'Total size:.*' || echo "Total size: unknown")
                  echo "Destination bytes: ${DEST_BYTES}"
                  echo "Destination summary: ${DEST_HUMAN}"

                  MIN_ACCEPTABLE=$(( SOURCE_BYTES * 80 / 100 ))
                  FLOOR=104857600 # 100 MiB absolute floor
                  echo "Acceptance threshold: dest >= ${MIN_ACCEPTABLE} bytes and > ${FLOOR} bytes"
                  if [ "${DEST_BYTES}" -lt "${FLOOR}" ]; then
                    echo "ERROR: destination is below hard floor (100 MiB) — backup failed."
                    exit 2
                  fi
                  if [ "${DEST_BYTES}" -lt "${MIN_ACCEPTABLE}" ]; then
                    echo "ERROR: destination is < 80% of source (${DEST_BYTES} < ${MIN_ACCEPTABLE}) — backup incomplete."
                    exit 3
                  fi
                  echo "OK: destination size acceptable."

                  # Prune by prefix name, NOT by file mtime. rclone preserves
                  # each source file's mtime on upload, and Prometheus TSDB
                  # chunk files retain very old mtimes (weeks-old blocks),
                  # so `rclone delete --min-age 7d` would eat the just-
                  # uploaded backup. Every top-level prefix is named
                  # `prometheus-YYYYMMDD-HHMMSS`, so we compare the encoded
                  # date to a 7-day threshold and purge whole prefixes.
                  echo "Pruning prometheus backup prefixes older than 7 days..."
                  # BusyBox date lacks GNU's `-d "7 days ago"` and BSD's `-v -7d`,
                  # so compute the cutoff via @epoch which BusyBox does support.
                  CUTOFF_EPOCH=$(( $(date +%s) - 7 * 86400 ))
                  CUTOFF=$(date -d "@${CUTOFF_EPOCH}" +%Y%m%d)
                  echo "Cutoff (delete prefixes with date < ${CUTOFF}):"
                  KEEP=0
                  PRUNE=0
                  for PREFIX in $(rclone lsf --dirs-only "hetzner-s3:${S3_BUCKET}/prometheus/" 2>/dev/null | sed 's:/$::'); do
                    case "${PREFIX}" in
                      prometheus-*)
                        PDATE=$(echo "${PREFIX}" | sed -n 's/^prometheus-\([0-9]\{8\}\)-.*/\1/p')
                        if [ -z "${PDATE}" ]; then
                          echo "  SKIP ${PREFIX} (unparseable name)"
                          continue
                        fi
                        if [ "${PDATE}" -lt "${CUTOFF}" ]; then
                          echo "  PURGE ${PREFIX} (date ${PDATE} < ${CUTOFF})"
                          rclone purge "hetzner-s3:${S3_BUCKET}/prometheus/${PREFIX}" \
                            || echo "  WARN purge failed for ${PREFIX} (continuing)"
                          PRUNE=$((PRUNE + 1))
                        else
                          echo "  KEEP  ${PREFIX} (date ${PDATE})"
                          KEEP=$((KEEP + 1))
                        fi
                        ;;
                      *)
                        echo "  SKIP ${PREFIX} (not a prometheus- prefix)"
                        ;;
                    esac
                  done
                  echo "Prune summary: kept ${KEEP}, purged ${PRUNE}."

                  echo "Post-run inventory (prometheus/ prefixes):"
                  rclone lsd "hetzner-s3:${S3_BUCKET}/prometheus/" || true

                  echo "=== Prometheus backup completed at $(date) ==="
              env:
                - name: S3_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: access-key
                - name: S3_SECRET_KEY
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: secret-key
                - name: S3_ENDPOINT
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: endpoint
                - name: S3_BUCKET
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: bucket
              volumeMounts:
                - name: prometheus-data
                  mountPath: /source
                  readOnly: true
              resources:
                requests:
                  cpu: 100m
                  memory: 128Mi
                limits:
                  cpu: 1000m
                  memory: 512Mi
          volumes:
            - name: prometheus-data
              persistentVolumeClaim:
                claimName: prometheus-data-encrypted