--- # Prometheus data backup via restic to Hetzner Object Storage # (DEV-492, DEV-482 Option 4). Replaces the DEV-465 rclone-sync job so # every monitoring backup ships client-side-encrypted; Hetzner Object # Storage has no SSE-S3/SSE-KMS, so the previous plaintext-at-rest # object layout was the only remaining gap. # # Streams the RWO PVC `prometheus-data-encrypted` (mounted read-only) # into `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/prometheus`, a client-side # encrypted restic repository (tag=`prometheus`, host=`k3s`). # # Node scheduling matches the previous job: podAffinity co-schedules # with the Prometheus pod (app=prometheus, topology # kubernetes.io/hostname). Hetzner CSI RWO permits additional read-only # mounts on the node that holds the PVC's VolumeAttachment, so this # survives Prometheus being rescheduled to a different worker. # # Prometheus TSDB compaction race # -------------------------------- # Prometheus rewrites the on-disk store roughly every 2 h: it creates a # new block dir, then deletes the source dirs. restic walks the source # tree once and may catch a file that disappeared mid-walk; restic # 0.17.3 exits 3 ("at least one source file could not be read") in # that case, and the snapshot excludes only the missing file. The # next daily run picks up the successor block, so the race is not a # data-loss risk — but we must not treat exit 3 as a hard failure, or # the daily job will alert-flap. # # Mitigation: # - exclude `wal/*` (WAL is replayed from a fresh instance on # restart; we accept losing the last ~15 s of ingested samples # rather than snapshotting a moving segment) # - exclude `chunks_head/*` (in-memory head block; ephemeral, would # be rebuilt from WAL which we do not keep) # - exclude Prometheus lock/scratch files (`lock`, `queries.active`, # `*.tmp`, `lost+found/*`) # - treat restic exit code 3 as a soft warning (log, continue); # any other non-zero exit is still fatal # - restore drill re-runs `promtool tsdb analyze` against every # block so a corrupted snapshot is caught end-to-end apiVersion: batch/v1 kind: CronJob metadata: name: prometheus-backup namespace: monitoring labels: app: backup type: prometheus backend: restic spec: schedule: "30 3 * * *" # daily 03:30, offset from loki/grafana/k8s-resources concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 jobTemplate: metadata: labels: app: backup type: prometheus backend: restic spec: backoffLimit: 2 activeDeadlineSeconds: 3600 template: metadata: labels: app: backup type: prometheus backend: restic spec: restartPolicy: OnFailure affinity: podAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: app operator: In values: - prometheus topologyKey: kubernetes.io/hostname containers: - name: restic image: harbor.basicstack.de/library/restic:0.17.3 env: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: name: monitoring-s3-backup key: access-key - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: name: monitoring-s3-backup key: secret-key - name: RESTIC_PASSWORD valueFrom: secretKeyRef: name: monitoring-s3-backup key: restic-password - name: S3_ENDPOINT valueFrom: secretKeyRef: name: monitoring-s3-backup key: endpoint - name: S3_BUCKET valueFrom: secretKeyRef: name: monitoring-s3-backup key: bucket - name: RESTIC_REPOSITORY value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/prometheus" command: - /bin/sh - -c - | set -eu echo "=== backup-prometheus-restic started at $(date -u +%FT%TZ) ===" echo "Repository: ${RESTIC_REPOSITORY}" # First-run tolerance: init if the repo isn't there yet. if restic snapshots >/dev/null 2>&1; then echo "Repo exists, skipping init." else echo "Repo missing, initialising..." restic init fi # See "Prometheus TSDB compaction race" in the file # header. WAL + chunks_head are excluded on purpose; # exit code 3 (source file vanished mid-walk during a # compaction) is accepted and reported, any other # non-zero exit is fatal. echo "--- restic backup /source (exclude wal/chunks_head + lock files) ---" BACKUP_STATUS=0 restic backup /source \ --tag prometheus \ --host k3s \ --exclude 'wal/*' \ --exclude 'chunks_head/*' \ --exclude 'lock' \ --exclude 'queries.active' \ --exclude 'lost+found/*' \ --exclude '*.tmp' || BACKUP_STATUS=$? echo "restic backup exit: ${BACKUP_STATUS}" if [ "${BACKUP_STATUS}" -eq 0 ]; then echo "backup: all files captured cleanly" elif [ "${BACKUP_STATUS}" -eq 3 ]; then echo "backup: exit 3 (source files vanished mid-walk) — expected under Prometheus compaction, continuing" else echo "backup: FATAL exit ${BACKUP_STATUS} (not compaction-race)" exit ${BACKUP_STATUS} fi echo "--- restic forget/prune ---" restic forget --tag prometheus \ --keep-daily 7 \ --keep-weekly 4 \ --keep-monthly 6 \ --prune echo "--- restic check --read-data-subset=5% ---" CHECK_STATUS=0 restic check --read-data-subset=5% || CHECK_STATUS=$? echo "restic check exit: ${CHECK_STATUS}" echo "--- restic stats (repo size) ---" REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \ | grep -oE '"total_size":[0-9]+' \ | head -1 \ | cut -d: -f2) REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0} echo "restic repo size: ${REPO_SIZE_BYTES} bytes" # Textfile-collector metrics. Same hostPath pattern as # the loki/grafana/k8s-resources siblings (DEV-494) — # written atomically via `.tmp` + rename so a mid-write # read never surfaces a truncated sample. # backup_prometheus_success rolls in both stages: the # backup step (accepting exit 3) and restic check. BACKUP_OK=0 if [ "${BACKUP_STATUS}" -eq 0 ] || [ "${BACKUP_STATUS}" -eq 3 ]; then BACKUP_OK=1 fi SUCCESS=0 if [ "${BACKUP_OK}" -eq 1 ] && [ "${CHECK_STATUS}" -eq 0 ]; then SUCCESS=1 fi { echo "backup_prometheus_success ${SUCCESS}" echo "backup_prometheus_timestamp_seconds $(date +%s)" echo "backup_prometheus_check_status ${CHECK_STATUS}" echo "backup_prometheus_backup_status ${BACKUP_STATUS}" echo "restic_repo_size_bytes{repo=\"prometheus\"} ${REPO_SIZE_BYTES}" } > /metrics/backup_prometheus.prom.tmp mv /metrics/backup_prometheus.prom.tmp /metrics/backup_prometheus.prom echo "=== backup-prometheus-restic finished at $(date -u +%FT%TZ) ===" exit ${CHECK_STATUS} volumeMounts: - name: prometheus-data mountPath: /source readOnly: true - name: metrics mountPath: /metrics - name: cache mountPath: /root/.cache/restic resources: # Prometheus TSDB is ~8-10 GiB; give restic room to # burst during pack/check but keep steady-state small. requests: cpu: 100m memory: 256Mi limits: cpu: 1500m memory: 1Gi volumes: - name: prometheus-data persistentVolumeClaim: claimName: prometheus-data-encrypted - name: metrics hostPath: # node-exporter's textfile-collector directory # (DEV-494). See sibling loki cronjob for detail. path: /var/lib/node_exporter/textfile_collector type: DirectoryOrCreate - name: cache emptyDir: {}