stack.basicstack.de/apps/monitoring/backup-grafana-restic-cronjob.yaml

---
# Grafana data backup via restic to Hetzner Object Storage (DEV-486,
# DEV-482 Option 4). Step 3 of the Option 4 rollout.
#
# Streams the RWO PVC `grafana-storage` (mounted read-only) into
# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/grafana`, a client-side
# encrypted restic repository.
#
# `grafana-storage` is a local-path PV anchored on k3s-worker-2, so
# the grafana pod is already pinned there; a plain nodeSelector on
# the same host is enough (no podAffinity like the loki job needed).
apiVersion: batch/v1
kind: CronJob
metadata:
  name: backup-grafana-restic
  namespace: monitoring
  labels:
    app: backup
    type: grafana
    backend: restic
spec:
  schedule: "15 3 * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    metadata:
      labels:
        app: backup
        type: grafana
        backend: restic
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 3600
      template:
        metadata:
          labels:
            app: backup
            type: grafana
            backend: restic
        spec:
          restartPolicy: OnFailure
          nodeSelector:
            kubernetes.io/hostname: k3s-worker-2
          containers:
            - name: restic
              image: harbor.basicstack.de/library/restic:0.19.1
              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
              # deterministic ingress via Harbor. Retag procedure in
              # docs/monitoring/restic-restore.md § "Tag-bump procedure".
              env:
                - name: AWS_ACCESS_KEY_ID
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: access-key
                - name: AWS_SECRET_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: secret-key
                - name: RESTIC_PASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: restic-password
                - name: S3_ENDPOINT
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: endpoint
                - name: S3_BUCKET
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: bucket
                - name: RESTIC_REPOSITORY
                  value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/grafana"
              command:
                - /bin/sh
                - -c
                - |
                  set -eu
                  echo "=== backup-grafana-restic started at $(date -u +%FT%TZ) ==="
                  echo "Repository: ${RESTIC_REPOSITORY}"

                  # First-run tolerance: init if the repo isn't there yet.
                  if restic snapshots >/dev/null 2>&1; then
                    echo "Repo exists, skipping init."
                  else
                    echo "Repo missing, initialising..."
                    restic init
                  fi

                  echo "--- restic backup /source ---"
                  restic backup /source \
                    --tag grafana \
                    --host k3s \
                    --exclude '*.tmp'

                  echo "--- restic forget/prune ---"
                  restic forget --tag grafana \
                    --keep-daily 7 \
                    --keep-weekly 4 \
                    --keep-monthly 6 \
                    --prune

                  echo "--- restic check --read-data-subset=5% ---"
                  CHECK_STATUS=0
                  restic check --read-data-subset=5% || CHECK_STATUS=$?
                  echo "restic check exit: ${CHECK_STATUS}"

                  echo "--- restic stats (repo size) ---"
                  REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
                    | grep -oE '"total_size":[0-9]+' \
                    | head -1 \
                    | cut -d: -f2)
                  REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
                  echo "restic repo size: ${REPO_SIZE_BYTES} bytes"

                  # Textfile-collector metrics; identical wiring to the
                  # loki sibling. See that file for the atomic-write
                  # rationale (DEV-494).
                  {
                    echo "backup_grafana_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
                    echo "backup_grafana_timestamp_seconds $(date +%s)"
                    echo "backup_grafana_check_status ${CHECK_STATUS}"
                    echo "restic_repo_size_bytes{repo=\"grafana\"} ${REPO_SIZE_BYTES}"
                  } > /metrics/backup_grafana.prom.tmp
                  mv /metrics/backup_grafana.prom.tmp /metrics/backup_grafana.prom

                  echo "=== backup-grafana-restic finished at $(date -u +%FT%TZ) ==="
                  exit ${CHECK_STATUS}
              volumeMounts:
                - name: grafana-data
                  mountPath: /source
                  readOnly: true
                - name: metrics
                  mountPath: /metrics
                - name: cache
                  mountPath: /root/.cache/restic
              resources:
                requests:
                  cpu: 100m
                  memory: 128Mi
                limits:
                  cpu: 1500m
                  memory: 1Gi
          volumes:
            - name: grafana-data
              persistentVolumeClaim:
                claimName: grafana-storage
            - name: metrics
              hostPath:
                # node-exporter's textfile-collector directory
                # (DEV-494). See sibling loki cronjob for detail.
                path: /var/lib/node_exporter/textfile_collector
                type: DirectoryOrCreate
            - name: cache
              emptyDir: {}