From 9d1996051ec724c3dcec4f4806a15afc7afbb7c8 Mon Sep 17 00:00:00 2001 From: CTO Agent Date: Sun, 9 Aug 2026 16:50:51 +0000 Subject: [PATCH] fix(backups): Repair three broken CronJobs blocking weekly OS updates (DEV-464) - Create the missing forgejo/platform-backup-data PVC that forgejo-backup references (20Gi hcloud-volumes-encrypted). - Record monitoring/backup-k8s-resources with a k3s-worker-2 nodeSelector (backup-storage PVC is local-path pinned there), lower memory request (128Mi) so it fits worker-2 pressure, and switch to alpine/k8s image (bitnami/kubectl is no longer resolvable). - Rewrite monitoring/backup-volumes to only back up grafana + loki co-located with backup-storage on k3s-worker-2. Prometheus data lives on k3s-worker-1 and is intentionally excluded here; a dedicated Prometheus data backup follows in a separate ticket. The three CronJobs previously left Pending/ContainerCreating pods that blocked the OS-update health guard in DEV-463. Co-Authored-By: Paperclip --- apps/forgejo/platform-backup-data-pvc.yaml | 14 +++ apps/monitoring/README.md | 10 ++ .../backup-k8s-resources-cronjob.yaml | 84 +++++++++++++++ apps/monitoring/backup-volumes-cronjob.yaml | 102 ++++++++++++++++++ 4 files changed, 210 insertions(+) create mode 100644 apps/forgejo/platform-backup-data-pvc.yaml create mode 100644 apps/monitoring/README.md create mode 100644 apps/monitoring/backup-k8s-resources-cronjob.yaml create mode 100644 apps/monitoring/backup-volumes-cronjob.yaml diff --git a/apps/forgejo/platform-backup-data-pvc.yaml b/apps/forgejo/platform-backup-data-pvc.yaml new file mode 100644 index 0000000..6a4b630 --- /dev/null +++ b/apps/forgejo/platform-backup-data-pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: platform-backup-data + namespace: forgejo + labels: + app: forgejo-backup +spec: + accessModes: + - ReadWriteOnce + storageClassName: hcloud-volumes-encrypted + resources: + requests: + storage: 20Gi diff --git a/apps/monitoring/README.md b/apps/monitoring/README.md new file mode 100644 index 0000000..8ab565e --- /dev/null +++ b/apps/monitoring/README.md @@ -0,0 +1,10 @@ +# monitoring — backup CronJobs + +Manifests recording the cluster-side monitoring backup CronJobs that were previously applied out-of-band. These files are the authoritative source (`kubectl apply -f apps/monitoring/`). See [DEV-464](/DEV/issues/DEV-464) for the repair context. + +- `backup-k8s-resources-cronjob.yaml` — daily dump of Kubernetes resources into `backup-storage` PVC. +- `backup-volumes-cronjob.yaml` — daily rsync/tar of Grafana + Loki PVCs into `backup-storage`. Prometheus data backup is **not** included here; it needs a separate on-node backup (tracked as a follow-up because Prometheus is on a different node than `backup-storage`). + +The `backup-storage` PVC (100Gi, local-path, bound to k3s-worker-2) is the shared destination for both jobs. + +Both CronJobs pin themselves to `k3s-worker-2` via `nodeSelector` because that is the node that holds all destination + source PVCs used here. diff --git a/apps/monitoring/backup-k8s-resources-cronjob.yaml b/apps/monitoring/backup-k8s-resources-cronjob.yaml new file mode 100644 index 0000000..6a6f64a --- /dev/null +++ b/apps/monitoring/backup-k8s-resources-cronjob.yaml @@ -0,0 +1,84 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup-k8s-resources + namespace: monitoring + labels: + app: backup + type: k8s-resources +spec: + schedule: "0 2 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + metadata: + annotations: + prometheus.io/scrape: "true" + labels: + app: backup + type: k8s-resources + spec: + backoffLimit: 2 + template: + metadata: + labels: + app: backup + spec: + restartPolicy: OnFailure + serviceAccountName: backup-sa + # backup-storage PVC (local-path) is bound to k3s-worker-2, so pin here. + nodeSelector: + kubernetes.io/hostname: k3s-worker-2 + containers: + - name: kubectl-backup + image: alpine/k8s:1.29.4 + command: + - /bin/sh + - -c + - | + set -e + BACKUP_DIR="/backup/k8s-$(date +%Y%m%d-%H%M%S)" + mkdir -p "$BACKUP_DIR" + + echo "Starting Kubernetes resources backup to $BACKUP_DIR" + + kubectl get namespaces -o yaml > "$BACKUP_DIR/namespaces.yaml" + + for ns in $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}'); do + mkdir -p "$BACKUP_DIR/$ns" + kubectl get configmaps,secrets,services,deployments,statefulsets,daemonsets,jobs,cronjobs,ingresses,persistentvolumeclaims \ + -n "$ns" -o yaml > "$BACKUP_DIR/$ns/resources.yaml" 2>/dev/null || true + done + + kubectl get persistentvolumes,storageclasses,clusterroles,clusterrolebindings \ + -o yaml > "$BACKUP_DIR/cluster-resources.yaml" + + cd /backup + tar -czf "k8s-backup-$(date +%Y%m%d-%H%M%S).tar.gz" "$(basename $BACKUP_DIR)" + rm -rf "$BACKUP_DIR" + + find /backup -name "k8s-backup-*.tar.gz" -mtime +7 -delete + + echo "Backup completed successfully" + + echo "backup_k8s_resources_success 1" > /metrics/backup_success.prom + echo "backup_k8s_resources_timestamp $(date +%s)" >> /metrics/backup_success.prom + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 384Mi + volumeMounts: + - mountPath: /backup + name: backup-storage + - mountPath: /metrics + name: metrics + volumes: + - name: backup-storage + persistentVolumeClaim: + claimName: backup-storage + - name: metrics + emptyDir: {} diff --git a/apps/monitoring/backup-volumes-cronjob.yaml b/apps/monitoring/backup-volumes-cronjob.yaml new file mode 100644 index 0000000..a03c7bc --- /dev/null +++ b/apps/monitoring/backup-volumes-cronjob.yaml @@ -0,0 +1,102 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup-volumes + namespace: monitoring + labels: + app: backup + type: volumes +spec: + schedule: "0 3 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + metadata: + annotations: + prometheus.io/scrape: "true" + labels: + app: backup + type: volumes + spec: + backoffLimit: 2 + template: + metadata: + labels: + app: backup + spec: + restartPolicy: OnFailure + # backup-storage, grafana-storage and loki-storage-encrypted are all + # RWO PVCs pinned to k3s-worker-2. Prometheus data lives on + # k3s-worker-1 and is intentionally NOT backed up here — see + # DEV-464 for the split rationale and the follow-up ticket for + # a dedicated Prometheus data backup. + nodeSelector: + kubernetes.io/hostname: k3s-worker-2 + containers: + - name: volume-backup + image: alpine:3.19 + command: + - /bin/sh + - -c + - | + set -e + apk add --no-cache rsync + + BACKUP_DATE=$(date +%Y%m%d-%H%M%S) + BACKUP_DIR="/backup/volumes-$BACKUP_DATE" + mkdir -p "$BACKUP_DIR" + + echo "Starting volume backup to $BACKUP_DIR" + + if [ -d "/source/grafana" ]; then + echo "Backing up Grafana data..." + rsync -a /source/grafana/ "$BACKUP_DIR/grafana/" || echo "Warning: Grafana backup incomplete" + fi + + if [ -d "/source/loki" ]; then + echo "Backing up Loki data..." + rsync -a /source/loki/ "$BACKUP_DIR/loki/" || echo "Warning: Loki backup incomplete" + fi + + cd /backup + tar -czf "volumes-backup-$BACKUP_DATE.tar.gz" "$(basename $BACKUP_DIR)" + rm -rf "$BACKUP_DIR" + + find /backup -name "volumes-backup-*.tar.gz" -mtime +7 -delete + + BACKUP_SIZE=$(du -sh "/backup/volumes-backup-$BACKUP_DATE.tar.gz" | cut -f1) + echo "Volume backup completed successfully: $BACKUP_SIZE" + + echo "backup_volumes_success 1" > /metrics/backup_success.prom + echo "backup_volumes_timestamp $(date +%s)" >> /metrics/backup_success.prom + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1000m + memory: 512Mi + volumeMounts: + - mountPath: /backup + name: backup-storage + - mountPath: /source/grafana + name: grafana-data + readOnly: true + - mountPath: /source/loki + name: loki-data + readOnly: true + - mountPath: /metrics + name: metrics + volumes: + - name: backup-storage + persistentVolumeClaim: + claimName: backup-storage + - name: grafana-data + persistentVolumeClaim: + claimName: grafana-storage + - name: loki-data + persistentVolumeClaim: + claimName: loki-storage-encrypted + - name: metrics + emptyDir: {}