9d1996051e
- Create the missing forgejo/platform-backup-data PVC that forgejo-backup references (20Gi hcloud-volumes-encrypted). - Record monitoring/backup-k8s-resources with a k3s-worker-2 nodeSelector (backup-storage PVC is local-path pinned there), lower memory request (128Mi) so it fits worker-2 pressure, and switch to alpine/k8s image (bitnami/kubectl is no longer resolvable). - Rewrite monitoring/backup-volumes to only back up grafana + loki co-located with backup-storage on k3s-worker-2. Prometheus data lives on k3s-worker-1 and is intentionally excluded here; a dedicated Prometheus data backup follows in a separate ticket. The three CronJobs previously left Pending/ContainerCreating pods that blocked the OS-update health guard in DEV-463. Co-Authored-By: Paperclip <noreply@paperclip.ing>
84 lines
2.7 KiB
YAML
84 lines
2.7 KiB
YAML
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: backup-k8s-resources
|
|
namespace: monitoring
|
|
labels:
|
|
app: backup
|
|
type: k8s-resources
|
|
spec:
|
|
schedule: "0 2 * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
metadata:
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
labels:
|
|
app: backup
|
|
type: k8s-resources
|
|
spec:
|
|
backoffLimit: 2
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
serviceAccountName: backup-sa
|
|
# backup-storage PVC (local-path) is bound to k3s-worker-2, so pin here.
|
|
nodeSelector:
|
|
kubernetes.io/hostname: k3s-worker-2
|
|
containers:
|
|
- name: kubectl-backup
|
|
image: alpine/k8s:1.29.4
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
BACKUP_DIR="/backup/k8s-$(date +%Y%m%d-%H%M%S)"
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
echo "Starting Kubernetes resources backup to $BACKUP_DIR"
|
|
|
|
kubectl get namespaces -o yaml > "$BACKUP_DIR/namespaces.yaml"
|
|
|
|
for ns in $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}'); do
|
|
mkdir -p "$BACKUP_DIR/$ns"
|
|
kubectl get configmaps,secrets,services,deployments,statefulsets,daemonsets,jobs,cronjobs,ingresses,persistentvolumeclaims \
|
|
-n "$ns" -o yaml > "$BACKUP_DIR/$ns/resources.yaml" 2>/dev/null || true
|
|
done
|
|
|
|
kubectl get persistentvolumes,storageclasses,clusterroles,clusterrolebindings \
|
|
-o yaml > "$BACKUP_DIR/cluster-resources.yaml"
|
|
|
|
cd /backup
|
|
tar -czf "k8s-backup-$(date +%Y%m%d-%H%M%S).tar.gz" "$(basename $BACKUP_DIR)"
|
|
rm -rf "$BACKUP_DIR"
|
|
|
|
find /backup -name "k8s-backup-*.tar.gz" -mtime +7 -delete
|
|
|
|
echo "Backup completed successfully"
|
|
|
|
echo "backup_k8s_resources_success 1" > /metrics/backup_success.prom
|
|
echo "backup_k8s_resources_timestamp $(date +%s)" >> /metrics/backup_success.prom
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 384Mi
|
|
volumeMounts:
|
|
- mountPath: /backup
|
|
name: backup-storage
|
|
- mountPath: /metrics
|
|
name: metrics
|
|
volumes:
|
|
- name: backup-storage
|
|
persistentVolumeClaim:
|
|
claimName: backup-storage
|
|
- name: metrics
|
|
emptyDir: {}
|