stack.basicstack.de/apps/monitoring/prometheus-backup-cronjob.yaml

---
# Prometheus data backup via restic to Hetzner Object Storage
# (DEV-492, DEV-482 Option 4). Replaces the DEV-465 rclone-sync job so
# every monitoring backup ships client-side-encrypted; Hetzner Object
# Storage has no SSE-S3/SSE-KMS, so the previous plaintext-at-rest
# object layout was the only remaining gap.
#
# Streams the RWO PVC `prometheus-data-encrypted` (mounted read-only)
# into `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/prometheus`, a client-side
# encrypted restic repository (tag=`prometheus`, host=`k3s`).
#
# Node scheduling matches the previous job: podAffinity co-schedules
# with the Prometheus pod (app=prometheus, topology
# kubernetes.io/hostname). Hetzner CSI RWO permits additional read-only
# mounts on the node that holds the PVC's VolumeAttachment, so this
# survives Prometheus being rescheduled to a different worker.
#
# Prometheus TSDB compaction race
# --------------------------------
# Prometheus rewrites the on-disk store roughly every 2 h: it creates a
# new block dir, then deletes the source dirs. restic walks the source
# tree once and may catch a file that disappeared mid-walk; restic
# 0.17.3 exits 3 ("at least one source file could not be read") in
# that case, and the snapshot excludes only the missing file. The
# next daily run picks up the successor block, so the race is not a
# data-loss risk — but we must not treat exit 3 as a hard failure, or
# the daily job will alert-flap.
#
# Mitigation:
#   - exclude `wal/*` (WAL is replayed from a fresh instance on
#     restart; we accept losing the last ~15 s of ingested samples
#     rather than snapshotting a moving segment)
#   - exclude `chunks_head/*` (in-memory head block; ephemeral, would
#     be rebuilt from WAL which we do not keep)
#   - exclude Prometheus lock/scratch files (`lock`, `queries.active`,
#     `*.tmp`, `lost+found/*`)
#   - treat restic exit code 3 as a soft warning (log, continue);
#     any other non-zero exit is still fatal
#   - restore drill re-runs `promtool tsdb analyze` against every
#     block so a corrupted snapshot is caught end-to-end
apiVersion: batch/v1
kind: CronJob
metadata:
  name: prometheus-backup
  namespace: monitoring
  labels:
    app: backup
    type: prometheus
    backend: restic
spec:
  schedule: "30 3 * * *"  # daily 03:30, offset from loki/grafana/k8s-resources
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    metadata:
      labels:
        app: backup
        type: prometheus
        backend: restic
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 3600
      template:
        metadata:
          labels:
            app: backup
            type: prometheus
            backend: restic
        spec:
          restartPolicy: OnFailure
          affinity:
            podAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                - labelSelector:
                    matchExpressions:
                      - key: app
                        operator: In
                        values:
                          - prometheus
                  topologyKey: kubernetes.io/hostname
          containers:
            - name: restic
              image: harbor.basicstack.de/library/restic:0.19.1
              env:
                - name: AWS_ACCESS_KEY_ID
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: access-key
                - name: AWS_SECRET_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: secret-key
                - name: RESTIC_PASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: restic-password
                - name: S3_ENDPOINT
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: endpoint
                - name: S3_BUCKET
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: bucket
                - name: RESTIC_REPOSITORY
                  value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/prometheus"
              command:
                - /bin/sh
                - -c
                - |
                  set -eu
                  echo "=== backup-prometheus-restic started at $(date -u +%FT%TZ) ==="
                  echo "Repository: ${RESTIC_REPOSITORY}"

                  # First-run tolerance: init if the repo isn't there yet.
                  if restic snapshots >/dev/null 2>&1; then
                    echo "Repo exists, skipping init."
                  else
                    echo "Repo missing, initialising..."
                    restic init
                  fi

                  # See "Prometheus TSDB compaction race" in the file
                  # header. WAL + chunks_head are excluded on purpose;
                  # exit code 3 (source file vanished mid-walk during a
                  # compaction) is accepted and reported, any other
                  # non-zero exit is fatal.
                  echo "--- restic backup /source (exclude wal/chunks_head + lock files) ---"
                  BACKUP_STATUS=0
                  restic backup /source \
                    --tag prometheus \
                    --host k3s \
                    --exclude 'wal/*' \
                    --exclude 'chunks_head/*' \
                    --exclude 'lock' \
                    --exclude 'queries.active' \
                    --exclude 'lost+found/*' \
                    --exclude '*.tmp' || BACKUP_STATUS=$?
                  echo "restic backup exit: ${BACKUP_STATUS}"
                  if [ "${BACKUP_STATUS}" -eq 0 ]; then
                    echo "backup: all files captured cleanly"
                  elif [ "${BACKUP_STATUS}" -eq 3 ]; then
                    echo "backup: exit 3 (source files vanished mid-walk) — expected under Prometheus compaction, continuing"
                  else
                    echo "backup: FATAL exit ${BACKUP_STATUS} (not compaction-race)"
                    exit ${BACKUP_STATUS}
                  fi

                  echo "--- restic forget/prune ---"
                  restic forget --tag prometheus \
                    --keep-daily 7 \
                    --keep-weekly 4 \
                    --keep-monthly 6 \
                    --prune

                  echo "--- restic check --read-data-subset=5% ---"
                  CHECK_STATUS=0
                  restic check --read-data-subset=5% || CHECK_STATUS=$?
                  echo "restic check exit: ${CHECK_STATUS}"

                  echo "--- restic stats (repo size) ---"
                  REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
                    | grep -oE '"total_size":[0-9]+' \
                    | head -1 \
                    | cut -d: -f2)
                  REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
                  echo "restic repo size: ${REPO_SIZE_BYTES} bytes"

                  # Textfile-collector metrics. Same hostPath pattern as
                  # the loki/grafana/k8s-resources siblings (DEV-494) —
                  # written atomically via `.tmp` + rename so a mid-write
                  # read never surfaces a truncated sample.
                  # backup_prometheus_success rolls in both stages: the
                  # backup step (accepting exit 3) and restic check.
                  BACKUP_OK=0
                  if [ "${BACKUP_STATUS}" -eq 0 ] || [ "${BACKUP_STATUS}" -eq 3 ]; then
                    BACKUP_OK=1
                  fi
                  SUCCESS=0
                  if [ "${BACKUP_OK}" -eq 1 ] && [ "${CHECK_STATUS}" -eq 0 ]; then
                    SUCCESS=1
                  fi
                  {
                    echo "backup_prometheus_success ${SUCCESS}"
                    echo "backup_prometheus_timestamp_seconds $(date +%s)"
                    echo "backup_prometheus_check_status ${CHECK_STATUS}"
                    echo "backup_prometheus_backup_status ${BACKUP_STATUS}"
                    echo "restic_repo_size_bytes{repo=\"prometheus\"} ${REPO_SIZE_BYTES}"
                  } > /metrics/backup_prometheus.prom.tmp
                  mv /metrics/backup_prometheus.prom.tmp /metrics/backup_prometheus.prom

                  echo "=== backup-prometheus-restic finished at $(date -u +%FT%TZ) ==="
                  exit ${CHECK_STATUS}
              volumeMounts:
                - name: prometheus-data
                  mountPath: /source
                  readOnly: true
                - name: metrics
                  mountPath: /metrics
                - name: cache
                  mountPath: /root/.cache/restic
              resources:
                # Prometheus TSDB is ~8-10 GiB; give restic room to
                # burst during pack/check but keep steady-state small.
                requests:
                  cpu: 100m
                  memory: 256Mi
                limits:
                  cpu: 1500m
                  memory: 1Gi
          volumes:
            - name: prometheus-data
              persistentVolumeClaim:
                claimName: prometheus-data-encrypted
            - name: metrics
              hostPath:
                # node-exporter's textfile-collector directory
                # (DEV-494). See sibling loki cronjob for detail.
                path: /var/lib/node_exporter/textfile_collector
                type: DirectoryOrCreate
            - name: cache
              emptyDir: {}