stack.basicstack.de/apps/monitoring/backup-k8s-resources-cronjob.yaml

---
# Kubernetes-resource backup via restic to Hetzner Object Storage
# (DEV-487, DEV-482 Option 4). Step 4 of the Option 4 rollout.
#
# Streams a concatenated YAML dump of cluster-scoped and per-namespace
# resources through `restic backup --stdin` into
# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/k8s-resources`. No PVC mount
# (drops the local-path `backup-storage` dependency), no node pin.
#
# Two-container pattern:
#   1. `kubectl-dump` init container (`alpine/k8s:1.29.4`) writes
#      /dump/cluster.yaml into an emptyDir. Uses `serviceAccountName:
#      backup-sa` (unchanged from the legacy job).
#   2. `restic` main container (`restic/restic:0.19.1`, matches the
#      loki/grafana siblings) reads that file on stdin and streams it
#      into the restic repo with `--stdin-filename cluster.yaml`.
apiVersion: batch/v1
kind: CronJob
metadata:
  name: backup-k8s-resources
  namespace: monitoring
  labels:
    app: backup
    type: k8s-resources
    backend: restic
spec:
  schedule: "0 2 * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    metadata:
      labels:
        app: backup
        type: k8s-resources
        backend: restic
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 3600
      template:
        metadata:
          labels:
            app: backup
            type: k8s-resources
            backend: restic
        spec:
          restartPolicy: OnFailure
          serviceAccountName: backup-sa
          initContainers:
            - name: kubectl-dump
              image: alpine/k8s:1.29.4
              command:
                - /bin/sh
                - -c
                - |
                  set -eu
                  echo "=== kubectl-dump started at $(date -u +%FT%TZ) ==="
                  DUMP=/dump/cluster.yaml
                  : > "${DUMP}"

                  echo "--- namespaces ---"
                  kubectl get namespaces -o yaml >> "${DUMP}"
                  echo "---" >> "${DUMP}"

                  echo "--- cluster-scoped resources ---"
                  kubectl get persistentvolumes,storageclasses,clusterroles,clusterrolebindings \
                    -o yaml >> "${DUMP}"
                  echo "---" >> "${DUMP}"

                  echo "--- namespaced resources ---"
                  for ns in $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}'); do
                    echo "  ns=${ns}"
                    kubectl get \
                      configmaps,secrets,services,deployments,statefulsets,daemonsets,jobs,cronjobs,ingresses,persistentvolumeclaims \
                      -n "${ns}" -o yaml >> "${DUMP}" 2>/dev/null || true
                    echo "---" >> "${DUMP}"
                  done

                  echo "dump size: $(wc -c < ${DUMP}) bytes"
                  echo "=== kubectl-dump finished at $(date -u +%FT%TZ) ==="
              resources:
                requests:
                  cpu: 50m
                  memory: 128Mi
                limits:
                  cpu: 500m
                  memory: 512Mi
              volumeMounts:
                - name: dump
                  mountPath: /dump
          containers:
            - name: restic
              image: harbor.basicstack.de/library/restic:0.19.1
              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
              # deterministic ingress via Harbor. Retag procedure in
              # docs/monitoring/restic-restore.md § "Tag-bump procedure".
              env:
                - name: AWS_ACCESS_KEY_ID
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: access-key
                - name: AWS_SECRET_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: secret-key
                - name: RESTIC_PASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: restic-password
                - name: S3_ENDPOINT
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: endpoint
                - name: S3_BUCKET
                  valueFrom:
                    secretKeyRef:
                      name: monitoring-s3-backup
                      key: bucket
                - name: RESTIC_REPOSITORY
                  value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/k8s-resources"
              command:
                - /bin/sh
                - -c
                - |
                  set -eu
                  echo "=== backup-k8s-resources-restic started at $(date -u +%FT%TZ) ==="
                  echo "Repository: ${RESTIC_REPOSITORY}"

                  # First-run tolerance: init if the repo isn't there yet.
                  if restic snapshots >/dev/null 2>&1; then
                    echo "Repo exists, skipping init."
                  else
                    echo "Repo missing, initialising..."
                    restic init
                  fi

                  echo "--- restic backup --stdin cluster.yaml ---"
                  restic backup --stdin \
                    --stdin-filename cluster.yaml \
                    --tag k8s-resources \
                    --host k3s < /dump/cluster.yaml

                  echo "--- restic forget/prune ---"
                  restic forget --tag k8s-resources \
                    --keep-daily 7 \
                    --keep-weekly 4 \
                    --keep-monthly 6 \
                    --prune

                  echo "--- restic check --read-data-subset=5% ---"
                  CHECK_STATUS=0
                  restic check --read-data-subset=5% || CHECK_STATUS=$?
                  echo "restic check exit: ${CHECK_STATUS}"

                  echo "--- restic stats (repo size) ---"
                  REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
                    | grep -oE '"total_size":[0-9]+' \
                    | head -1 \
                    | cut -d: -f2)
                  REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
                  echo "restic repo size: ${REPO_SIZE_BYTES} bytes"

                  # Textfile-collector metrics; identical wiring to the
                  # loki/grafana siblings. See loki cronjob for the
                  # atomic-write rationale (DEV-494).
                  {
                    echo "backup_k8s_resources_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
                    echo "backup_k8s_resources_timestamp_seconds $(date +%s)"
                    echo "backup_k8s_resources_check_status ${CHECK_STATUS}"
                    echo "restic_repo_size_bytes{repo=\"k8s-resources\"} ${REPO_SIZE_BYTES}"
                  } > /metrics/backup_k8s_resources.prom.tmp
                  mv /metrics/backup_k8s_resources.prom.tmp /metrics/backup_k8s_resources.prom

                  echo "=== backup-k8s-resources-restic finished at $(date -u +%FT%TZ) ==="
                  exit ${CHECK_STATUS}
              volumeMounts:
                - name: dump
                  mountPath: /dump
                  readOnly: true
                - name: metrics
                  mountPath: /metrics
                - name: cache
                  mountPath: /root/.cache/restic
              resources:
                requests:
                  cpu: 100m
                  memory: 128Mi
                limits:
                  cpu: 1500m
                  memory: 1Gi
          volumes:
            - name: dump
              emptyDir: {}
            - name: metrics
              hostPath:
                # node-exporter's textfile-collector directory
                # (DEV-494). See sibling loki cronjob for detail.
                path: /var/lib/node_exporter/textfile_collector
                type: DirectoryOrCreate
            - name: cache
              emptyDir: {}