ac9f36b1e7
- CronJob at 0 4 1 * * restores the latest openwebui-manifests backup into a transient restore-test-<ts> namespace, diffs core resource-kind counts against the live openwebui namespace, and deletes the target namespace. - ServiceAccount + ClusterRole scoped to velero.io + namespace + configmap ops. Script enforces the 'restore-test-' name prefix before deleting a namespace. Cluster-wide 'view' bound for the diff. - Failure surfaces via kube_job_status_failed (DEV-574 alert stack) and via ConfigMap velero-restore-drill-last-result written on every run. - Optional Paperclip webhook Secret is sealed but unreachable from Hetzner right now (home Paperclip is LAN-only); wired as optional secretKeyRef so the drill runs regardless. Connectivity is a follow-up. - Argo Application velero-restore-drill points at velero/restore-drill/. Co-Authored-By: Paperclip <noreply@paperclip.ing>
217 lines
8.6 KiB
YAML
217 lines
8.6 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: restore-drill-script
|
|
namespace: velero
|
|
labels:
|
|
app.kubernetes.io/part-of: velero-restore-drill
|
|
data:
|
|
drill.sh: |
|
|
#!/bin/bash
|
|
# Monthly Velero restore drill for openwebui (DEV-575).
|
|
# Picks the latest openwebui-manifests backup (or ${DRILL_BACKUP_OVERRIDE}),
|
|
# restores it into restore-test-<ts>, diffs counts vs source, cleans up.
|
|
# Writes result to ConfigMap velero-restore-drill-last-result and job stdout.
|
|
# Exits non-zero on any failure (surfaces via kube_job_status_failed alert).
|
|
set -uo pipefail
|
|
|
|
SRC_NS="${DRILL_SOURCE_NS:-openwebui}"
|
|
SCHEDULE_NAME="${DRILL_SCHEDULE_NAME:-openwebui-manifests}"
|
|
RESULT_CM="${DRILL_RESULT_CM:-velero-restore-drill-last-result}"
|
|
RESULT_NS="${DRILL_RESULT_NS:-velero}"
|
|
RESTORE_TIMEOUT="${DRILL_RESTORE_TIMEOUT:-1800}"
|
|
NS_DELETE_TIMEOUT="${DRILL_NS_DELETE_TIMEOUT:-600}"
|
|
|
|
TS="$(date -u +%Y%m%d-%H%M%S)"
|
|
TGT_NS="restore-test-${TS}"
|
|
RESTORE_NAME="drill-${TS}"
|
|
|
|
LOG=""
|
|
log() { local line; line="[$(date -u +%FT%TZ)] $*"; echo "$line"; LOG+="${line}"$'\n'; }
|
|
|
|
DRILL_STATUS="fail"
|
|
BACKUP_NAME=""
|
|
BACKUP_START=""
|
|
BACKUP_COMPLETION=""
|
|
RESTORE_PHASE=""
|
|
RESTORE_DURATION_S=""
|
|
DIFF_SUMMARY=""
|
|
|
|
write_result_cm() {
|
|
local report="$1"
|
|
kubectl -n "${RESULT_NS}" create configmap "${RESULT_CM}" \
|
|
--from-literal=result="${report}" \
|
|
--from-literal=log="${LOG}" \
|
|
--dry-run=client -o yaml \
|
|
| kubectl -n "${RESULT_NS}" apply -f - >/dev/null 2>&1 \
|
|
|| log "WARN: failed to write result ConfigMap"
|
|
}
|
|
|
|
post_webhook() {
|
|
local status="$1" backup="$2" restore="$3" restore_phase="$4" \
|
|
restore_duration_s="$5" target_ns="$6" source_ns="$7" diff_summary="$8"
|
|
if [[ -z "${PAPERCLIP_WEBHOOK_URL:-}" || -z "${PAPERCLIP_WEBHOOK_SECRET:-}" ]]; then
|
|
log "webhook: no PAPERCLIP_WEBHOOK_URL/SECRET, skipping"
|
|
return
|
|
fi
|
|
local body_file=/tmp/webhook.json
|
|
printf '{"source":"velero-restore-drill","status":"%s","timestamp":"%s","backup":"%s","restore":"%s","restore_phase":"%s","restore_duration_s":"%s","target_namespace":"%s","source_namespace":"%s","diff_summary":"%s"}\n' \
|
|
"${status}" "${TS}" "${backup}" "${restore}" "${restore_phase}" \
|
|
"${restore_duration_s}" "${target_ns}" "${source_ns}" "${diff_summary}" > "${body_file}"
|
|
local hc
|
|
hc=$(curl -sS -o /tmp/webhook.out -w '%{http_code}' -m 30 \
|
|
-X POST -H "Authorization: Bearer ${PAPERCLIP_WEBHOOK_SECRET}" \
|
|
-H "Content-Type: application/json" \
|
|
--data-binary "@${body_file}" \
|
|
"${PAPERCLIP_WEBHOOK_URL}" 2>&1) || hc="000"
|
|
log "webhook: HTTP ${hc}"
|
|
}
|
|
|
|
finalize() {
|
|
local end
|
|
end="$(date -u +%FT%TZ)"
|
|
|
|
if [[ "${TGT_NS}" == restore-test-* ]] && kubectl get ns "${TGT_NS}" >/dev/null 2>&1; then
|
|
log "cleanup: deleting namespace ${TGT_NS}"
|
|
if kubectl delete ns "${TGT_NS}" --timeout="${NS_DELETE_TIMEOUT}s" >/tmp/nsdelete.log 2>&1; then
|
|
log "cleanup: namespace ${TGT_NS} deleted"
|
|
else
|
|
log "cleanup: WARN failed to fully delete ${TGT_NS}, may be stuck terminating"
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "${RESTORE_NAME}" ]] && kubectl -n velero get restore.velero.io "${RESTORE_NAME}" >/dev/null 2>&1; then
|
|
kubectl -n velero delete restore.velero.io "${RESTORE_NAME}" --wait=false >/dev/null 2>&1 || true
|
|
fi
|
|
|
|
local report
|
|
report=$(printf 'drill=%s\ntimestamp=%s\nbackup=%s\nbackup_started=%s\nbackup_completed=%s\nrestore=%s\nrestore_phase=%s\nrestore_duration_s=%s\ntarget_namespace=%s\nsource_namespace=%s\ndiff_summary=%s\nfinished_at=%s\n' \
|
|
"${DRILL_STATUS}" "${TS}" "${BACKUP_NAME}" "${BACKUP_START}" \
|
|
"${BACKUP_COMPLETION}" "${RESTORE_NAME}" "${RESTORE_PHASE}" \
|
|
"${RESTORE_DURATION_S}" "${TGT_NS}" "${SRC_NS}" "${DIFF_SUMMARY}" "${end}")
|
|
|
|
log "----- FINAL REPORT -----"
|
|
printf '%s\n' "${report}"
|
|
|
|
write_result_cm "${report}"
|
|
post_webhook "${DRILL_STATUS}" "${BACKUP_NAME}" "${RESTORE_NAME}" \
|
|
"${RESTORE_PHASE}" "${RESTORE_DURATION_S}" "${TGT_NS}" "${SRC_NS}" \
|
|
"${DIFF_SUMMARY}"
|
|
|
|
if [[ "${DRILL_STATUS}" == "pass" ]]; then
|
|
exit 0
|
|
else
|
|
exit 1
|
|
fi
|
|
}
|
|
trap finalize EXIT
|
|
|
|
log "drill start ts=${TS} src_ns=${SRC_NS} schedule=${SCHEDULE_NAME} target=${TGT_NS}"
|
|
|
|
if [[ -n "${DRILL_BACKUP_OVERRIDE:-}" ]]; then
|
|
BACKUP_NAME="${DRILL_BACKUP_OVERRIDE}"
|
|
log "using override backup name: ${BACKUP_NAME}"
|
|
else
|
|
BACKUP_NAME=$(kubectl -n velero get backups.velero.io \
|
|
-l "velero.io/schedule-name=${SCHEDULE_NAME}" \
|
|
-o go-template='{{range .items}}{{if eq .status.phase "Completed"}}{{.status.completionTimestamp}}|{{.metadata.name}}{{"\n"}}{{end}}{{end}}' 2>/dev/null \
|
|
| sort -r | head -n1 | cut -d'|' -f2)
|
|
if [[ -z "${BACKUP_NAME}" ]]; then
|
|
log "ERROR: no Completed backup found for schedule=${SCHEDULE_NAME}"
|
|
exit 1
|
|
fi
|
|
log "latest Completed backup: ${BACKUP_NAME}"
|
|
fi
|
|
|
|
BACKUP_START=$(kubectl -n velero get backup.velero.io "${BACKUP_NAME}" -o jsonpath='{.status.startTimestamp}' 2>/dev/null || true)
|
|
BACKUP_COMPLETION=$(kubectl -n velero get backup.velero.io "${BACKUP_NAME}" -o jsonpath='{.status.completionTimestamp}' 2>/dev/null || true)
|
|
|
|
log "creating restore ${RESTORE_NAME} from backup ${BACKUP_NAME} -> ns ${TGT_NS}"
|
|
RESTORE_YAML=/tmp/restore.yaml
|
|
{
|
|
printf 'apiVersion: velero.io/v1\n'
|
|
printf 'kind: Restore\n'
|
|
printf 'metadata:\n'
|
|
printf ' name: %s\n' "${RESTORE_NAME}"
|
|
printf ' namespace: velero\n'
|
|
printf ' labels:\n'
|
|
printf ' app.kubernetes.io/part-of: velero-restore-drill\n'
|
|
printf 'spec:\n'
|
|
printf ' backupName: %s\n' "${BACKUP_NAME}"
|
|
printf ' includedNamespaces:\n'
|
|
printf ' - %s\n' "${SRC_NS}"
|
|
printf ' namespaceMapping:\n'
|
|
printf ' %s: %s\n' "${SRC_NS}" "${TGT_NS}"
|
|
printf ' includeClusterResources: false\n'
|
|
printf ' restorePVs: false\n'
|
|
printf ' existingResourcePolicy: none\n'
|
|
printf ' hooks: {}\n'
|
|
} > "${RESTORE_YAML}"
|
|
if ! kubectl apply -f "${RESTORE_YAML}" >/tmp/apply.log 2>&1; then
|
|
log "ERROR: failed to create Restore CR"
|
|
cat /tmp/apply.log | while IFS= read -r l; do log " $l"; done
|
|
exit 1
|
|
fi
|
|
|
|
log "waiting up to ${RESTORE_TIMEOUT}s for restore to finish"
|
|
start_epoch=$(date +%s)
|
|
while :; do
|
|
RESTORE_PHASE=$(kubectl -n velero get restore.velero.io "${RESTORE_NAME}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
|
|
case "${RESTORE_PHASE}" in
|
|
Completed|PartiallyFailed|Failed|FailedValidation)
|
|
break
|
|
;;
|
|
esac
|
|
now=$(date +%s)
|
|
if (( now - start_epoch > RESTORE_TIMEOUT )); then
|
|
log "ERROR: restore did not reach terminal phase within ${RESTORE_TIMEOUT}s (last=${RESTORE_PHASE})"
|
|
exit 1
|
|
fi
|
|
sleep 10
|
|
done
|
|
RESTORE_DURATION_S=$(( $(date +%s) - start_epoch ))
|
|
log "restore phase=${RESTORE_PHASE} duration=${RESTORE_DURATION_S}s"
|
|
|
|
if [[ "${RESTORE_PHASE}" != "Completed" && "${RESTORE_PHASE}" != "PartiallyFailed" ]]; then
|
|
log "ERROR: restore did not complete (phase=${RESTORE_PHASE})"
|
|
kubectl -n velero describe restore.velero.io "${RESTORE_NAME}" 2>&1 | tail -40 | while IFS= read -r l; do log " $l"; done
|
|
exit 1
|
|
fi
|
|
|
|
if ! kubectl get ns "${TGT_NS}" >/dev/null 2>&1; then
|
|
log "ERROR: restore reported ${RESTORE_PHASE} but namespace ${TGT_NS} does not exist"
|
|
exit 1
|
|
fi
|
|
|
|
count_kind() {
|
|
local ns="$1" kind="$2"
|
|
kubectl -n "${ns}" get "${kind}" --no-headers 2>/dev/null | wc -l | tr -d ' '
|
|
}
|
|
|
|
KINDS=(deployments statefulsets daemonsets replicasets services configmaps secrets serviceaccounts persistentvolumeclaims cronjobs jobs)
|
|
DIFF_SUMMARY=""
|
|
for k in "${KINDS[@]}"; do
|
|
sc=$(count_kind "${SRC_NS}" "${k}")
|
|
tc=$(count_kind "${TGT_NS}" "${k}")
|
|
DIFF_SUMMARY+="${k}=${sc}/${tc};"
|
|
done
|
|
log "diff (src/tgt): ${DIFF_SUMMARY}"
|
|
|
|
STRICT_KINDS=(deployments statefulsets services configmaps persistentvolumeclaims)
|
|
strict_ok=1
|
|
for k in "${STRICT_KINDS[@]}"; do
|
|
sc=$(count_kind "${SRC_NS}" "${k}")
|
|
tc=$(count_kind "${TGT_NS}" "${k}")
|
|
if [[ "${sc}" != "${tc}" ]]; then
|
|
log "STRICT MISMATCH kind=${k} src=${sc} tgt=${tc}"
|
|
strict_ok=0
|
|
fi
|
|
done
|
|
|
|
if (( strict_ok == 1 )); then
|
|
DRILL_STATUS="pass"
|
|
log "drill result: PASS"
|
|
else
|
|
log "drill result: FAIL (strict kind mismatch)"
|
|
exit 1
|
|
fi
|