d495a5936f
Add a daily 03:15 UTC restic backup of the grafana-storage PVC to
s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/grafana. Pinned to
k3s-worker-2 via nodeSelector because grafana-storage is a
local-path PV anchored there — no podAffinity needed. Same restic
retention as the loki sibling (7d/4w/6m + prune + 5% check) with
metrics backup_grafana_{success,timestamp_seconds,check_status}
written to the emptyDir textfile path. Deployed in parallel with
the legacy backup-volumes CronJob (Option 4 rollout, DEV-482).
Co-Authored-By: Paperclip <noreply@paperclip.ing>
145 lines
5 KiB
YAML
145 lines
5 KiB
YAML
---
|
|
# Grafana data backup via restic to Hetzner Object Storage (DEV-486,
|
|
# DEV-482 Option 4). Step 3 of the Option 4 rollout.
|
|
#
|
|
# Streams the RWO PVC `grafana-storage` (mounted read-only) into
|
|
# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/grafana`, a client-side
|
|
# encrypted restic repository. Deployed in parallel with the legacy
|
|
# `backup-volumes` CronJob — do not retire that job until DEV-482
|
|
# step 6 (restore drill passed).
|
|
#
|
|
# `grafana-storage` is a local-path PV anchored on k3s-worker-2, so
|
|
# the grafana pod is already pinned there; a plain nodeSelector on
|
|
# the same host is enough (no podAffinity like the loki job needed).
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: backup-grafana-restic
|
|
namespace: monitoring
|
|
labels:
|
|
app: backup
|
|
type: grafana
|
|
backend: restic
|
|
spec:
|
|
schedule: "15 3 * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
type: grafana
|
|
backend: restic
|
|
spec:
|
|
backoffLimit: 2
|
|
activeDeadlineSeconds: 3600
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: backup
|
|
type: grafana
|
|
backend: restic
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
nodeSelector:
|
|
kubernetes.io/hostname: k3s-worker-2
|
|
containers:
|
|
- name: restic
|
|
image: restic/restic:0.17.3
|
|
env:
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: access-key
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: secret-key
|
|
- name: RESTIC_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: restic-password
|
|
- name: S3_ENDPOINT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: endpoint
|
|
- name: S3_BUCKET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: monitoring-s3-backup
|
|
key: bucket
|
|
- name: RESTIC_REPOSITORY
|
|
value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/grafana"
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -eu
|
|
echo "=== backup-grafana-restic started at $(date -u +%FT%TZ) ==="
|
|
echo "Repository: ${RESTIC_REPOSITORY}"
|
|
|
|
# First-run tolerance: init if the repo isn't there yet.
|
|
if restic snapshots >/dev/null 2>&1; then
|
|
echo "Repo exists, skipping init."
|
|
else
|
|
echo "Repo missing, initialising..."
|
|
restic init
|
|
fi
|
|
|
|
echo "--- restic backup /source ---"
|
|
restic backup /source \
|
|
--tag grafana \
|
|
--host k3s \
|
|
--exclude '*.tmp'
|
|
|
|
echo "--- restic forget/prune ---"
|
|
restic forget --tag grafana \
|
|
--keep-daily 7 \
|
|
--keep-weekly 4 \
|
|
--keep-monthly 6 \
|
|
--prune
|
|
|
|
echo "--- restic check --read-data-subset=5% ---"
|
|
CHECK_STATUS=0
|
|
restic check --read-data-subset=5% || CHECK_STATUS=$?
|
|
echo "restic check exit: ${CHECK_STATUS}"
|
|
|
|
# Textfile-collector metrics; identical wiring to the loki
|
|
# sibling. Scrapeable once node-exporter's textfile
|
|
# collector path is enabled — tracked in DEV-482.
|
|
{
|
|
echo "backup_grafana_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
|
|
echo "backup_grafana_timestamp_seconds $(date +%s)"
|
|
echo "backup_grafana_check_status ${CHECK_STATUS}"
|
|
} > /metrics/backup_grafana.prom
|
|
|
|
echo "=== backup-grafana-restic finished at $(date -u +%FT%TZ) ==="
|
|
exit ${CHECK_STATUS}
|
|
volumeMounts:
|
|
- name: grafana-data
|
|
mountPath: /source
|
|
readOnly: true
|
|
- name: metrics
|
|
mountPath: /metrics
|
|
- name: cache
|
|
mountPath: /root/.cache/restic
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 1500m
|
|
memory: 1Gi
|
|
volumes:
|
|
- name: grafana-data
|
|
persistentVolumeClaim:
|
|
claimName: grafana-storage
|
|
- name: metrics
|
|
emptyDir: {}
|
|
- name: cache
|
|
emptyDir: {}
|