stack.basicstack.de/apps/monitoring/backup-loki-restic-cronjob.yaml

---
# Loki data backup via restic to Hetzner Object Storage (DEV-485,
# DEV-482 Option 4). Step 2 of the Option 4 rollout.
#
# Streams the RWO PVC `loki-storage-encrypted` (mounted read-only)
# into `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/loki`, a client-side
# encrypted restic repository.
#
# podAffinity co-schedules with the Loki pod (app=loki, topology
# kubernetes.io/hostname). RWO permits additional read-only mounts
# on the node that holds the PVC's VolumeAttachment, so this
# survives Loki being rescheduled to a different worker.
apiVersion: batch/v1
kind: CronJob
metadata:
  name: backup-loki-restic
  namespace: monitoring
  labels:
    app: backup
    type: loki
    backend: restic
spec:
  schedule: "0 3 * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    metadata:
      labels:
        app: backup
        type: loki
        backend: restic
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 3600
      template:
        metadata:
          labels:
            app: backup
            type: loki
            backend: restic
        spec:
          restartPolicy: OnFailure
          affinity:
            podAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                - labelSelector:
                    matchExpressions:
                      - key: app
                        operator: In
                        values:
                          - loki
                  topologyKey: kubernetes.io/hostname
          containers:
            - name: restic
              image: harbor.basicstack.de/library/restic:0.19.1
              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
              # deterministic ingress via Harbor. Retag procedure in
              # docs/monitoring/restic-restore.md § "Tag-bump procedure".
              env:
                - name: AWS_ACCESS_KEY_ID
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: access-key
                - name: AWS_SECRET_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: secret-key
                - name: RESTIC_PASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: restic-password
                - name: S3_ENDPOINT
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: endpoint
                - name: S3_BUCKET
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: bucket
                - name: RESTIC_REPOSITORY
                  value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/loki"
              command:
                - /bin/sh
                - -c
                - |
                  set -eu
                  echo "=== backup-loki-restic started at $(date -u +%FT%TZ) ==="
                  echo "Repository: ${RESTIC_REPOSITORY}"

                  # First-run tolerance: init if the repo isn't there yet.
                  # `restic cat config` is the tightest existence probe; use
                  # `snapshots` per plan spec — either exits 0 iff the repo
                  # is initialised.
                  if restic snapshots >/dev/null 2>&1; then
                    echo "Repo exists, skipping init."
                  else
                    echo "Repo missing, initialising..."
                    restic init
                  fi

                  echo "--- restic backup /source ---"
                  restic backup /source \
                    --tag loki \
                    --host k3s \
                    --exclude '*.tmp'

                  echo "--- restic forget/prune ---"
                  restic forget --tag loki \
                    --keep-daily 7 \
                    --keep-weekly 4 \
                    --keep-monthly 6 \
                    --prune

                  echo "--- restic check --read-data-subset=5% ---"
                  CHECK_STATUS=0
                  restic check --read-data-subset=5% || CHECK_STATUS=$?
                  echo "restic check exit: ${CHECK_STATUS}"

                  echo "--- restic stats (repo size) ---"
                  # `restic stats --json --mode raw-data` prints e.g.
                  # {"total_size":123,"total_file_count":45,...}. Extract
                  # total_size without jq (not present in the restic image)
                  # via grep/cut; fall back to 0 on empty output.
                  REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
                    | grep -oE '"total_size":[0-9]+' \
                    | head -1 \
                    | cut -d: -f2)
                  REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
                  echo "restic repo size: ${REPO_SIZE_BYTES} bytes"

                  # Textfile-collector metrics. Written to the shared
                  # host directory that node-exporter's textfile collector
                  # scrapes (DEV-494). Atomic write: build the file with a
                  # `.tmp` extension (ignored by node-exporter) and rename
                  # into place, so a mid-write read never surfaces a
                  # truncated sample.
                  {
                    echo "backup_loki_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
                    echo "backup_loki_timestamp_seconds $(date +%s)"
                    echo "backup_loki_check_status ${CHECK_STATUS}"
                    echo "restic_repo_size_bytes{repo=\"loki\"} ${REPO_SIZE_BYTES}"
                  } > /metrics/backup_loki.prom.tmp
                  mv /metrics/backup_loki.prom.tmp /metrics/backup_loki.prom

                  echo "=== backup-loki-restic finished at $(date -u +%FT%TZ) ==="
                  exit ${CHECK_STATUS}
              volumeMounts:
                - name: loki-data
                  mountPath: /source
                  readOnly: true
                - name: metrics
                  mountPath: /metrics
                - name: cache
                  mountPath: /root/.cache/restic
              resources:
                # Requests deliberately lowered from the plan doc's
                # 200m/256Mi — worker-2 (Loki node) has ~150m free CPU
                # and podAffinity forces us onto it. 100m/128Mi mirrors
                # the sibling backup CronJobs; limits stay generous so
                # restic can burst during pack/check.
                requests:
                  cpu: 100m
                  memory: 128Mi
                limits:
                  cpu: 1500m
                  memory: 1Gi
          volumes:
            - name: loki-data
              persistentVolumeClaim:
                claimName: loki-storage-encrypted
            - name: metrics
              hostPath:
                # node-exporter's textfile-collector directory
                # (DEV-494). `DirectoryOrCreate` lets kubelet create
                # the dir on the current node if it does not yet
                # exist — nodes were pre-created out-of-band, this
                # is a safety net.
                path: /var/lib/node_exporter/textfile_collector
                type: DirectoryOrCreate
            - name: cache
              emptyDir: {}