From ac9f36b1e7b8a546225be4e2508d0a3d7ca3e8f6 Mon Sep 17 00:00:00 2001
From: "CTO (Paperclip Agent)" <cto@basicstack.de>
Date: Sat, 12 Sep 2026 18:03:37 +0000
Subject: [PATCH] feat(velero): monthly restore-drill CronJob for openwebui
 (DEV-575)

- CronJob at 0 4 1 * * restores the latest openwebui-manifests backup into
  a transient restore-test-<ts> namespace, diffs core resource-kind counts
  against the live openwebui namespace, and deletes the target namespace.
- ServiceAccount + ClusterRole scoped to velero.io + namespace + configmap
  ops. Script enforces the 'restore-test-' name prefix before deleting a
  namespace. Cluster-wide 'view' bound for the diff.
- Failure surfaces via kube_job_status_failed (DEV-574 alert stack) and via
  ConfigMap velero-restore-drill-last-result written on every run.
- Optional Paperclip webhook Secret is sealed but unreachable from Hetzner
  right now (home Paperclip is LAN-only); wired as optional secretKeyRef so
  the drill runs regardless. Connectivity is a follow-up.
- Argo Application velero-restore-drill points at velero/restore-drill/.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 apps/app-velero-restore-drill.yaml            |  23 ++
 .../restore-drill/00-serviceaccount-rbac.yaml |  60 +++++
 .../10-drill-script-configmap.yaml            | 217 ++++++++++++++++++
 .../20-restore-drill-cronjob.yaml             |  97 ++++++++
 velero/restore-drill/30-webhook-sealed.yaml   |  17 ++
 velero/restore-drill/README.md                |  72 ++++++
 6 files changed, 486 insertions(+)
 create mode 100644 apps/app-velero-restore-drill.yaml
 create mode 100644 velero/restore-drill/00-serviceaccount-rbac.yaml
 create mode 100644 velero/restore-drill/10-drill-script-configmap.yaml
 create mode 100644 velero/restore-drill/20-restore-drill-cronjob.yaml
 create mode 100644 velero/restore-drill/30-webhook-sealed.yaml
 create mode 100644 velero/restore-drill/README.md

diff --git a/apps/app-velero-restore-drill.yaml b/apps/app-velero-restore-drill.yaml
new file mode 100644
index 0000000..87b782d
--- /dev/null
+++ b/apps/app-velero-restore-drill.yaml
@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: velero-restore-drill
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: velero/restore-drill
+    directory:
+      recurse: false
+      include: '*.yaml'
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: velero
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - ServerSideApply=true
diff --git a/velero/restore-drill/00-serviceaccount-rbac.yaml b/velero/restore-drill/00-serviceaccount-rbac.yaml
new file mode 100644
index 0000000..7a61adb
--- /dev/null
+++ b/velero/restore-drill/00-serviceaccount-rbac.yaml
@@ -0,0 +1,60 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: restore-drill
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+---
+# Ops permissions: create/list/get/delete velero Backups+Restores, create+delete
+# restore-test-* namespaces. Namespace verbs cannot be name-scoped in RBAC, so we
+# accept cluster-wide namespace create/delete. Drill script enforces the
+# 'restore-test-' name prefix before deleting.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: velero-restore-drill-ops
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+rules:
+  - apiGroups: ["velero.io"]
+    resources: ["backups", "restores"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: velero-restore-drill-ops
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+subjects:
+  - kind: ServiceAccount
+    name: restore-drill
+    namespace: velero
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: velero-restore-drill-ops
+---
+# Read-only diff permissions: cluster-scoped read of the workload resources we
+# compare between the source and restored namespaces.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: velero-restore-drill-view
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+subjects:
+  - kind: ServiceAccount
+    name: restore-drill
+    namespace: velero
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: view
diff --git a/velero/restore-drill/10-drill-script-configmap.yaml b/velero/restore-drill/10-drill-script-configmap.yaml
new file mode 100644
index 0000000..6998cb7
--- /dev/null
+++ b/velero/restore-drill/10-drill-script-configmap.yaml
@@ -0,0 +1,217 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: restore-drill-script
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+data:
+  drill.sh: |
+    #!/bin/bash
+    # Monthly Velero restore drill for openwebui (DEV-575).
+    # Picks the latest openwebui-manifests backup (or ${DRILL_BACKUP_OVERRIDE}),
+    # restores it into restore-test-<ts>, diffs counts vs source, cleans up.
+    # Writes result to ConfigMap velero-restore-drill-last-result and job stdout.
+    # Exits non-zero on any failure (surfaces via kube_job_status_failed alert).
+    set -uo pipefail
+
+    SRC_NS="${DRILL_SOURCE_NS:-openwebui}"
+    SCHEDULE_NAME="${DRILL_SCHEDULE_NAME:-openwebui-manifests}"
+    RESULT_CM="${DRILL_RESULT_CM:-velero-restore-drill-last-result}"
+    RESULT_NS="${DRILL_RESULT_NS:-velero}"
+    RESTORE_TIMEOUT="${DRILL_RESTORE_TIMEOUT:-1800}"
+    NS_DELETE_TIMEOUT="${DRILL_NS_DELETE_TIMEOUT:-600}"
+
+    TS="$(date -u +%Y%m%d-%H%M%S)"
+    TGT_NS="restore-test-${TS}"
+    RESTORE_NAME="drill-${TS}"
+
+    LOG=""
+    log() { local line; line="[$(date -u +%FT%TZ)] $*"; echo "$line"; LOG+="${line}"$'\n'; }
+
+    DRILL_STATUS="fail"
+    BACKUP_NAME=""
+    BACKUP_START=""
+    BACKUP_COMPLETION=""
+    RESTORE_PHASE=""
+    RESTORE_DURATION_S=""
+    DIFF_SUMMARY=""
+
+    write_result_cm() {
+      local report="$1"
+      kubectl -n "${RESULT_NS}" create configmap "${RESULT_CM}" \
+        --from-literal=result="${report}" \
+        --from-literal=log="${LOG}" \
+        --dry-run=client -o yaml \
+        | kubectl -n "${RESULT_NS}" apply -f - >/dev/null 2>&1 \
+        || log "WARN: failed to write result ConfigMap"
+    }
+
+    post_webhook() {
+      local status="$1" backup="$2" restore="$3" restore_phase="$4" \
+            restore_duration_s="$5" target_ns="$6" source_ns="$7" diff_summary="$8"
+      if [[ -z "${PAPERCLIP_WEBHOOK_URL:-}" || -z "${PAPERCLIP_WEBHOOK_SECRET:-}" ]]; then
+        log "webhook: no PAPERCLIP_WEBHOOK_URL/SECRET, skipping"
+        return
+      fi
+      local body_file=/tmp/webhook.json
+      printf '{"source":"velero-restore-drill","status":"%s","timestamp":"%s","backup":"%s","restore":"%s","restore_phase":"%s","restore_duration_s":"%s","target_namespace":"%s","source_namespace":"%s","diff_summary":"%s"}\n' \
+        "${status}" "${TS}" "${backup}" "${restore}" "${restore_phase}" \
+        "${restore_duration_s}" "${target_ns}" "${source_ns}" "${diff_summary}" > "${body_file}"
+      local hc
+      hc=$(curl -sS -o /tmp/webhook.out -w '%{http_code}' -m 30 \
+             -X POST -H "Authorization: Bearer ${PAPERCLIP_WEBHOOK_SECRET}" \
+             -H "Content-Type: application/json" \
+             --data-binary "@${body_file}" \
+             "${PAPERCLIP_WEBHOOK_URL}" 2>&1) || hc="000"
+      log "webhook: HTTP ${hc}"
+    }
+
+    finalize() {
+      local end
+      end="$(date -u +%FT%TZ)"
+
+      if [[ "${TGT_NS}" == restore-test-* ]] && kubectl get ns "${TGT_NS}" >/dev/null 2>&1; then
+        log "cleanup: deleting namespace ${TGT_NS}"
+        if kubectl delete ns "${TGT_NS}" --timeout="${NS_DELETE_TIMEOUT}s" >/tmp/nsdelete.log 2>&1; then
+          log "cleanup: namespace ${TGT_NS} deleted"
+        else
+          log "cleanup: WARN failed to fully delete ${TGT_NS}, may be stuck terminating"
+        fi
+      fi
+
+      if [[ -n "${RESTORE_NAME}" ]] && kubectl -n velero get restore.velero.io "${RESTORE_NAME}" >/dev/null 2>&1; then
+        kubectl -n velero delete restore.velero.io "${RESTORE_NAME}" --wait=false >/dev/null 2>&1 || true
+      fi
+
+      local report
+      report=$(printf 'drill=%s\ntimestamp=%s\nbackup=%s\nbackup_started=%s\nbackup_completed=%s\nrestore=%s\nrestore_phase=%s\nrestore_duration_s=%s\ntarget_namespace=%s\nsource_namespace=%s\ndiff_summary=%s\nfinished_at=%s\n' \
+        "${DRILL_STATUS}" "${TS}" "${BACKUP_NAME}" "${BACKUP_START}" \
+        "${BACKUP_COMPLETION}" "${RESTORE_NAME}" "${RESTORE_PHASE}" \
+        "${RESTORE_DURATION_S}" "${TGT_NS}" "${SRC_NS}" "${DIFF_SUMMARY}" "${end}")
+
+      log "----- FINAL REPORT -----"
+      printf '%s\n' "${report}"
+
+      write_result_cm "${report}"
+      post_webhook "${DRILL_STATUS}" "${BACKUP_NAME}" "${RESTORE_NAME}" \
+        "${RESTORE_PHASE}" "${RESTORE_DURATION_S}" "${TGT_NS}" "${SRC_NS}" \
+        "${DIFF_SUMMARY}"
+
+      if [[ "${DRILL_STATUS}" == "pass" ]]; then
+        exit 0
+      else
+        exit 1
+      fi
+    }
+    trap finalize EXIT
+
+    log "drill start ts=${TS} src_ns=${SRC_NS} schedule=${SCHEDULE_NAME} target=${TGT_NS}"
+
+    if [[ -n "${DRILL_BACKUP_OVERRIDE:-}" ]]; then
+      BACKUP_NAME="${DRILL_BACKUP_OVERRIDE}"
+      log "using override backup name: ${BACKUP_NAME}"
+    else
+      BACKUP_NAME=$(kubectl -n velero get backups.velero.io \
+        -l "velero.io/schedule-name=${SCHEDULE_NAME}" \
+        -o go-template='{{range .items}}{{if eq .status.phase "Completed"}}{{.status.completionTimestamp}}|{{.metadata.name}}{{"\n"}}{{end}}{{end}}' 2>/dev/null \
+        | sort -r | head -n1 | cut -d'|' -f2)
+      if [[ -z "${BACKUP_NAME}" ]]; then
+        log "ERROR: no Completed backup found for schedule=${SCHEDULE_NAME}"
+        exit 1
+      fi
+      log "latest Completed backup: ${BACKUP_NAME}"
+    fi
+
+    BACKUP_START=$(kubectl -n velero get backup.velero.io "${BACKUP_NAME}" -o jsonpath='{.status.startTimestamp}' 2>/dev/null || true)
+    BACKUP_COMPLETION=$(kubectl -n velero get backup.velero.io "${BACKUP_NAME}" -o jsonpath='{.status.completionTimestamp}' 2>/dev/null || true)
+
+    log "creating restore ${RESTORE_NAME} from backup ${BACKUP_NAME} -> ns ${TGT_NS}"
+    RESTORE_YAML=/tmp/restore.yaml
+    {
+      printf 'apiVersion: velero.io/v1\n'
+      printf 'kind: Restore\n'
+      printf 'metadata:\n'
+      printf '  name: %s\n' "${RESTORE_NAME}"
+      printf '  namespace: velero\n'
+      printf '  labels:\n'
+      printf '    app.kubernetes.io/part-of: velero-restore-drill\n'
+      printf 'spec:\n'
+      printf '  backupName: %s\n' "${BACKUP_NAME}"
+      printf '  includedNamespaces:\n'
+      printf '    - %s\n' "${SRC_NS}"
+      printf '  namespaceMapping:\n'
+      printf '    %s: %s\n' "${SRC_NS}" "${TGT_NS}"
+      printf '  includeClusterResources: false\n'
+      printf '  restorePVs: false\n'
+      printf '  existingResourcePolicy: none\n'
+      printf '  hooks: {}\n'
+    } > "${RESTORE_YAML}"
+    if ! kubectl apply -f "${RESTORE_YAML}" >/tmp/apply.log 2>&1; then
+      log "ERROR: failed to create Restore CR"
+      cat /tmp/apply.log | while IFS= read -r l; do log "  $l"; done
+      exit 1
+    fi
+
+    log "waiting up to ${RESTORE_TIMEOUT}s for restore to finish"
+    start_epoch=$(date +%s)
+    while :; do
+      RESTORE_PHASE=$(kubectl -n velero get restore.velero.io "${RESTORE_NAME}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
+      case "${RESTORE_PHASE}" in
+        Completed|PartiallyFailed|Failed|FailedValidation)
+          break
+          ;;
+      esac
+      now=$(date +%s)
+      if (( now - start_epoch > RESTORE_TIMEOUT )); then
+        log "ERROR: restore did not reach terminal phase within ${RESTORE_TIMEOUT}s (last=${RESTORE_PHASE})"
+        exit 1
+      fi
+      sleep 10
+    done
+    RESTORE_DURATION_S=$(( $(date +%s) - start_epoch ))
+    log "restore phase=${RESTORE_PHASE} duration=${RESTORE_DURATION_S}s"
+
+    if [[ "${RESTORE_PHASE}" != "Completed" && "${RESTORE_PHASE}" != "PartiallyFailed" ]]; then
+      log "ERROR: restore did not complete (phase=${RESTORE_PHASE})"
+      kubectl -n velero describe restore.velero.io "${RESTORE_NAME}" 2>&1 | tail -40 | while IFS= read -r l; do log "  $l"; done
+      exit 1
+    fi
+
+    if ! kubectl get ns "${TGT_NS}" >/dev/null 2>&1; then
+      log "ERROR: restore reported ${RESTORE_PHASE} but namespace ${TGT_NS} does not exist"
+      exit 1
+    fi
+
+    count_kind() {
+      local ns="$1" kind="$2"
+      kubectl -n "${ns}" get "${kind}" --no-headers 2>/dev/null | wc -l | tr -d ' '
+    }
+
+    KINDS=(deployments statefulsets daemonsets replicasets services configmaps secrets serviceaccounts persistentvolumeclaims cronjobs jobs)
+    DIFF_SUMMARY=""
+    for k in "${KINDS[@]}"; do
+      sc=$(count_kind "${SRC_NS}" "${k}")
+      tc=$(count_kind "${TGT_NS}" "${k}")
+      DIFF_SUMMARY+="${k}=${sc}/${tc};"
+    done
+    log "diff (src/tgt): ${DIFF_SUMMARY}"
+
+    STRICT_KINDS=(deployments statefulsets services configmaps persistentvolumeclaims)
+    strict_ok=1
+    for k in "${STRICT_KINDS[@]}"; do
+      sc=$(count_kind "${SRC_NS}" "${k}")
+      tc=$(count_kind "${TGT_NS}" "${k}")
+      if [[ "${sc}" != "${tc}" ]]; then
+        log "STRICT MISMATCH kind=${k} src=${sc} tgt=${tc}"
+        strict_ok=0
+      fi
+    done
+
+    if (( strict_ok == 1 )); then
+      DRILL_STATUS="pass"
+      log "drill result: PASS"
+    else
+      log "drill result: FAIL (strict kind mismatch)"
+      exit 1
+    fi
diff --git a/velero/restore-drill/20-restore-drill-cronjob.yaml b/velero/restore-drill/20-restore-drill-cronjob.yaml
new file mode 100644
index 0000000..e7a91e3
--- /dev/null
+++ b/velero/restore-drill/20-restore-drill-cronjob.yaml
@@ -0,0 +1,97 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: velero-restore-drill
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+spec:
+  # Monthly at 04:00 UTC on the 1st (per DEV-575 acceptance).
+  schedule: "0 4 1 * *"
+  timeZone: "UTC"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  startingDeadlineSeconds: 3600
+  jobTemplate:
+    metadata:
+      labels:
+        app.kubernetes.io/part-of: velero-restore-drill
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 3600
+      ttlSecondsAfterFinished: 604800   # keep completed jobs for 7d for debug
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/part-of: velero-restore-drill
+        spec:
+          restartPolicy: Never
+          serviceAccountName: restore-drill
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            runAsGroup: 65532
+            fsGroup: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: drill
+              image: alpine/k8s:1.30.5
+              imagePullPolicy: IfNotPresent
+              command: ["/bin/bash", "/scripts/drill.sh"]
+              env:
+                - name: DRILL_SOURCE_NS
+                  value: "openwebui"
+                - name: DRILL_SCHEDULE_NAME
+                  value: "openwebui-manifests"
+                - name: DRILL_RESULT_NS
+                  value: "velero"
+                - name: DRILL_RESULT_CM
+                  value: "velero-restore-drill-last-result"
+                - name: DRILL_RESTORE_TIMEOUT
+                  value: "1800"
+                - name: DRILL_NS_DELETE_TIMEOUT
+                  value: "600"
+                # Optional Paperclip webhook (mounted from SealedSecret if wired).
+                - name: PAPERCLIP_WEBHOOK_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: velero-restore-drill-webhook
+                      key: webhookUrl
+                      optional: true
+                - name: PAPERCLIP_WEBHOOK_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: velero-restore-drill-webhook
+                      key: webhookSecret
+                      optional: true
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 128Mi
+                limits:
+                  cpu: 500m
+                  memory: 512Mi
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: ["ALL"]
+              volumeMounts:
+                - name: script
+                  mountPath: /scripts
+                  readOnly: true
+                - name: tmp
+                  mountPath: /tmp
+                - name: kube
+                  mountPath: /.kube
+          volumes:
+            - name: script
+              configMap:
+                name: restore-drill-script
+                defaultMode: 0555
+            - name: tmp
+              emptyDir: {}
+            - name: kube
+              emptyDir: {}
diff --git a/velero/restore-drill/30-webhook-sealed.yaml b/velero/restore-drill/30-webhook-sealed.yaml
new file mode 100644
index 0000000..fb25770
--- /dev/null
+++ b/velero/restore-drill/30-webhook-sealed.yaml
@@ -0,0 +1,17 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: velero-restore-drill-webhook
+  namespace: velero
+spec:
+  encryptedData:
+    webhookSecret: AgB4J8oWfPjZDar6zOp9ezv7M5DrtQFCvN6JgJmT3f6LJBngssanDW0N2nFg76a8L/Pht2RZK8aQq0GyGVbYjxtiQ0lagZGR/7j2yoKgkUyRie99YL7SMBPZ2SM9yBDRzsctHKeLZtmhsqyrmKLFnVs49Au3v2GtpknwOePr/BDZgdTGhaR/NftFuWMak6yPeoXNkGpYK7spJwoWUDN1LyQY/ZoEcgz2iQJ7odKqVE7hmt21an1xv7a/nmIDS8uNWWfYlPspTTxfQvTX4Ti28e5Z7S9kj+fqAVX6zH7tZ/jKODE72+uOya2FnjZ/SjEJ7Lz0+4eZTMWo0Kk0rK+WFY5fQvwOFlOZiCQsL3S2oxIiPjy11ZJh8gBFCXUR1KJNsPNveifsXvOC+kwWA/YaYQRHUArKZ50QOXhkwbEoS7XB1LklJil0IRzScvDSNSwtrpxg80rWVTVZEC5XJVpnRaNOSvfQI7NBYPA/rYRU3OcT0cT7vkBUIK4kgFV/4uIIoalaKcAGtPoHTEciA0avkCOgDSzlE18YZ///1hdtgJWx1OQ2ewpdawmsK1XJPjSreuf8zqxiVQosxI3Q0pTjv47aznjnxV2EDuvhj/gXYfrvO9B47chi2PdesAZt2IeOEOphl94EkVANHAuNgwnsymbK8mdFyybnCcIrWpgaIRMZYy3cQv+tqRwHJrk1cqPnul7fWKu4V7t6SfVY/ML3vsGQOevE4dce/rDquMuAbnWYqmV8YoD3jxPwe8bkhKUcrZM=
+    webhookUrl: AgBVLLkdlaxiqWyQQXcLj5uewljVQhBQpJw1OG9IMNw+MjQoBss6gqZhHgK2xAddsbnzOEAr/BJ6kJgR8DXldWsdQ9MnQZDc7VLOXs+h399KrIYeDTbuRHwWzp2BaCBSwOxmnd7S6v6h10Qq9rC02F8uRQ5gQ0B/JOVb1kwh+h6whCL3cSC511Wp6dnON2u0QJb3NoIVqGJd24ZcFgOs8w53fSgQgakvbMPYe34/RoHtT/SZmoaIUUzzAC+C1FzRKaJPKVDoPoYy5IhnIMNkecPmyg1hjJzmRQR64grIhO02WBm+lbjXnRqcauAyNeE5NNglYKaQmSr3Ya8zaB1pxm2L5DiUgCyR4O4nlIPH/37zvqa35fVnELmuwYprBCVmCxjulkPlqrh3tmRcR6EEPdd2ECND4E6XUenof57W/VsPg+iiRn/vcqAM3e0NidHhf5GtSxW5VDBmorrzqyWsEZbnNxIA01rNX1pupVTC1YssTdGF1k5/ccWUlwCpZJMeMyR0fHfn6kCsFHbF8lZ6w3VILObiKJnDU7Kkn3RXZ1k9plN21ZipffoSSXDLStQitJhX3a6ADhRazRw/xZjdqgO5iYNfSXih6GI1/6LgbE5UTZA7nqvAX9OqnH1B/7qMGCcwt18l+lF26uStN0OiDOaPdsgdzSpdv0BPibSvc4RdtvVEHiarV11geaMdxS8wZnS+C7GO2Nvx+qcGS6v9G9czIEvvrogcuBd8S3F7z0oiLmNNEeVW/CJwknO+5OkJm/wmFhQTZbRozgte8v9pjsaPZ6k/CzUdAilb+aEdkZ8DesXh5zYlGTMedclc
+  template:
+    metadata:
+      creationTimestamp: null
+      name: velero-restore-drill-webhook
+      namespace: velero
+    type: Opaque
+---
diff --git a/velero/restore-drill/README.md b/velero/restore-drill/README.md
new file mode 100644
index 0000000..d63a34a
--- /dev/null
+++ b/velero/restore-drill/README.md
@@ -0,0 +1,72 @@
+# Velero restore drill
+
+Monthly Kubernetes CronJob that exercises the Velero backup chain by restoring
+the `openwebui` namespace into a transient `restore-test-<ts>` namespace,
+diffing resource counts against the live namespace, and cleaning up.
+
+Wired by [DEV-575](https://forgejo.basicstack.de/basicstack/stack.basicstack.de/wiki/DEV-575)
+under the [DEV-569](https://forgejo.basicstack.de/basicstack/stack.basicstack.de/wiki/DEV-569)
+Velero cluster-reconstruction backup layer.
+
+## What it does
+
+1. Picks the latest `Completed` backup for schedule `openwebui-manifests`
+   (env `DRILL_BACKUP_OVERRIDE` short-circuits this — used by the failure
+   drill).
+2. Creates a `Restore` CR with namespace mapping
+   `openwebui: restore-test-<ts>`.
+3. Waits for the `Restore` to reach a terminal phase (`Completed`,
+   `PartiallyFailed`, `Failed`, `FailedValidation`) or times out at 30 min.
+4. Counts core kinds (deployments, statefulsets, services, configmaps, PVCs, …)
+   in source vs target and requires strict equality on the manifest-carrying
+   kinds to pass.
+5. Deletes the `restore-test-<ts>` namespace and the `Restore` CR.
+6. Writes the final report to ConfigMap `velero/velero-restore-drill-last-result`
+   and logs it to stdout.
+7. Optionally POSTs a JSON payload to the Paperclip routine webhook if
+   the `velero-restore-drill-webhook` Secret is present (currently unreachable
+   from the cluster — see follow-up ticket).
+
+## Failure signalling
+
+The Job exits non-zero on any failure. The alerting stack (DEV-574) will fire
+`KubeJobFailed` on the `velero-restore-drill-*` job name and route the alert
+through existing Alertmanager receivers. Ops can inspect the last result via:
+
+```
+kubectl -n velero get cm velero-restore-drill-last-result -o jsonpath='{.data.result}'
+kubectl -n velero get cm velero-restore-drill-last-result -o jsonpath='{.data.log}'
+```
+
+## Manual runs
+
+Kick a drill on demand:
+
+```
+kubectl -n velero create job --from=cronjob/velero-restore-drill drill-adhoc-$(date -u +%Y%m%d-%H%M%S)
+```
+
+Force a failure to test the alert wiring (uses a bogus backup name):
+
+```
+kubectl -n velero create job drill-adhoc-fail-$(date -u +%s) \
+  --image=alpine/k8s:1.30.5 \
+  --dry-run=client -o yaml \
+  | kubectl -n velero apply -f -   # only if you need a scratch, prefer:
+kubectl -n velero create job --from=cronjob/velero-restore-drill drill-fail-$(date -u +%Y%m%d-%H%M%S) --dry-run=client -o yaml \
+  | sed 's/value: "openwebui-manifests"/value: "openwebui-manifests"\n            - name: DRILL_BACKUP_OVERRIDE\n              value: "no-such-backup"/' \
+  | kubectl apply -f -
+```
+
+For a simpler failure drill, apply the CronJob-derived Job then patch env before scheduling — see the parent DEV-575 comment for the exact runbook.
+
+## RBAC scope
+
+- `velero.io/backups,restores`: get/list/create/delete in-cluster (namespace
+  scope is enforced by resource type — Backups/Restores live in `velero`).
+- `namespaces`: cluster-wide get/list/create/delete. Cannot be name-scoped in
+  RBAC; the drill script enforces the `restore-test-` prefix before ever
+  deleting a namespace.
+- `configmaps`: get/list/create/update/patch (only used to write the result
+  ConfigMap in `velero` — pod-level namespace scoping via ServiceAccount).
+- Cluster-wide `view` ClusterRole for the counting diff.