9d1996051e
- Create the missing forgejo/platform-backup-data PVC that forgejo-backup references (20Gi hcloud-volumes-encrypted). - Record monitoring/backup-k8s-resources with a k3s-worker-2 nodeSelector (backup-storage PVC is local-path pinned there), lower memory request (128Mi) so it fits worker-2 pressure, and switch to alpine/k8s image (bitnami/kubectl is no longer resolvable). - Rewrite monitoring/backup-volumes to only back up grafana + loki co-located with backup-storage on k3s-worker-2. Prometheus data lives on k3s-worker-1 and is intentionally excluded here; a dedicated Prometheus data backup follows in a separate ticket. The three CronJobs previously left Pending/ContainerCreating pods that blocked the OS-update health guard in DEV-463. Co-Authored-By: Paperclip <noreply@paperclip.ing>
102 lines
3.2 KiB
YAML
102 lines
3.2 KiB
YAML
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: backup-volumes
|
|
namespace: monitoring
|
|
labels:
|
|
app: backup
|
|
type: volumes
|
|
spec:
|
|
schedule: "0 3 * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
metadata:
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
labels:
|
|
app: backup
|
|
type: volumes
|
|
spec:
|
|
backoffLimit: 2
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
# backup-storage, grafana-storage and loki-storage-encrypted are all
|
|
# RWO PVCs pinned to k3s-worker-2. Prometheus data lives on
|
|
# k3s-worker-1 and is intentionally NOT backed up here — see
|
|
# DEV-464 for the split rationale and the follow-up ticket for
|
|
# a dedicated Prometheus data backup.
|
|
nodeSelector:
|
|
kubernetes.io/hostname: k3s-worker-2
|
|
containers:
|
|
- name: volume-backup
|
|
image: alpine:3.19
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
apk add --no-cache rsync
|
|
|
|
BACKUP_DATE=$(date +%Y%m%d-%H%M%S)
|
|
BACKUP_DIR="/backup/volumes-$BACKUP_DATE"
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
echo "Starting volume backup to $BACKUP_DIR"
|
|
|
|
if [ -d "/source/grafana" ]; then
|
|
echo "Backing up Grafana data..."
|
|
rsync -a /source/grafana/ "$BACKUP_DIR/grafana/" || echo "Warning: Grafana backup incomplete"
|
|
fi
|
|
|
|
if [ -d "/source/loki" ]; then
|
|
echo "Backing up Loki data..."
|
|
rsync -a /source/loki/ "$BACKUP_DIR/loki/" || echo "Warning: Loki backup incomplete"
|
|
fi
|
|
|
|
cd /backup
|
|
tar -czf "volumes-backup-$BACKUP_DATE.tar.gz" "$(basename $BACKUP_DIR)"
|
|
rm -rf "$BACKUP_DIR"
|
|
|
|
find /backup -name "volumes-backup-*.tar.gz" -mtime +7 -delete
|
|
|
|
BACKUP_SIZE=$(du -sh "/backup/volumes-backup-$BACKUP_DATE.tar.gz" | cut -f1)
|
|
echo "Volume backup completed successfully: $BACKUP_SIZE"
|
|
|
|
echo "backup_volumes_success 1" > /metrics/backup_success.prom
|
|
echo "backup_volumes_timestamp $(date +%s)" >> /metrics/backup_success.prom
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 1000m
|
|
memory: 512Mi
|
|
volumeMounts:
|
|
- mountPath: /backup
|
|
name: backup-storage
|
|
- mountPath: /source/grafana
|
|
name: grafana-data
|
|
readOnly: true
|
|
- mountPath: /source/loki
|
|
name: loki-data
|
|
readOnly: true
|
|
- mountPath: /metrics
|
|
name: metrics
|
|
volumes:
|
|
- name: backup-storage
|
|
persistentVolumeClaim:
|
|
claimName: backup-storage
|
|
- name: grafana-data
|
|
persistentVolumeClaim:
|
|
claimName: grafana-storage
|
|
- name: loki-data
|
|
persistentVolumeClaim:
|
|
claimName: loki-storage-encrypted
|
|
- name: metrics
|
|
emptyDir: {}
|