From 6a66924595a668e4710a68b50bddb394ac781378 Mon Sep 17 00:00:00 2001 From: "CTO (Paperclip Agent)" Date: Sat, 12 Sep 2026 17:58:45 +0000 Subject: [PATCH] feat(velero): ServiceMonitor + PrometheusRule for backup alerts (DEV-574) Adds Prometheus scrape of Velero's :8085 metrics and four alerts (VeleroBackupFailed, VeleroBackupPartial, VeleroBackupMissed, VeleroStorageLocationUnavailable) labeled team=infra severity=warning. VeleroStorageLocationUnavailable substitutes for the spec's VeleroRepoUnhealthy since Velero v1.18.1 does not expose velero_backup_repository_available (see comment in the rule). Wired via a new ArgoCD Application (velero-monitoring) that syncs only servicemonitor.yaml + prometheusrule.yaml from basicstack-repo/velero/. Co-Authored-By: Paperclip --- apps/app-velero-monitoring.yaml | 23 +++++++ velero/prometheusrule.yaml | 108 ++++++++++++++++++++++++++++++++ velero/servicemonitor.yaml | 28 +++++++++ 3 files changed, 159 insertions(+) create mode 100644 apps/app-velero-monitoring.yaml create mode 100644 velero/prometheusrule.yaml create mode 100644 velero/servicemonitor.yaml diff --git a/apps/app-velero-monitoring.yaml b/apps/app-velero-monitoring.yaml new file mode 100644 index 0000000..b3d3835 --- /dev/null +++ b/apps/app-velero-monitoring.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: velero-monitoring + namespace: argocd +spec: + project: default + source: + repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git + targetRevision: main + path: velero + directory: + recurse: false + include: '{servicemonitor.yaml,prometheusrule.yaml}' + destination: + server: https://kubernetes.default.svc + namespace: velero + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/velero/prometheusrule.yaml b/velero/prometheusrule.yaml new file mode 100644 index 0000000..dfea3f6 --- /dev/null +++ b/velero/prometheusrule.yaml @@ -0,0 +1,108 @@ +--- +# Velero alerting for cluster-reconstruction backups (DEV-569 / DEV-574). +# +# Metrics come from the Velero server on :8085/metrics, scraped via +# velero/servicemonitor.yaml. +# +# All our Velero Schedules are daily (see velero/schedules/*), so the +# freshness alert uses a 26h window (24h cadence + 2h grace). +# +# Notes: +# - `velero_backup_last_successful_timestamp` is emitted only after a +# schedule has had at least one successful run. Freshly created +# schedules produce no series until their first success, so the +# "missed" alert doesn't fire on day 0 — it fires once a schedule +# has succeeded at least once and then goes >26h without another. +# - `schedule=""` is Velero's rollup for one-off (non-Schedule) +# backups; we filter it out so ad-hoc backups don't page. +# - Velero v1.18.1 does not expose `velero_backup_repository_available` +# (mentioned in the DEV-574 spec), so the repo-health alert here +# watches `velero_backup_location_status_gauge` instead — that +# covers the same operational failure mode ("backups can't land +# because the target storage is unreachable"). +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: velero-alerts + namespace: velero + labels: + app.kubernetes.io/name: velero + app.kubernetes.io/part-of: velero + release: kube-prometheus-stack +spec: + groups: + - name: velero.backups + interval: 60s + rules: + - alert: VeleroBackupFailed + expr: sum by (schedule) (increase(velero_backup_failure_total{schedule!=""}[30m])) > 0 + for: 0m + labels: + team: infra + severity: warning + service: velero + component: backup + annotations: + summary: "Velero backup failed ({{ $labels.schedule }})" + description: | + Velero recorded {{ $value | printf "%.0f" }} new backup + failure(s) for schedule `{{ $labels.schedule }}` in the + last 30 minutes. + Investigate: `velero backup get -n velero` and + `velero backup describe --details`. + - alert: VeleroBackupPartial + expr: sum by (schedule) (increase(velero_backup_partial_failure_total{schedule!=""}[30m])) > 0 + for: 0m + labels: + team: infra + severity: warning + service: velero + component: backup + annotations: + summary: "Velero backup completed with partial failures ({{ $labels.schedule }})" + description: | + Velero recorded {{ $value | printf "%.0f" }} partial + failure(s) for schedule `{{ $labels.schedule }}` in the + last 30 minutes. Some items were skipped or errored even + though the backup finished. + Investigate: `velero backup describe --details`. + - alert: VeleroBackupMissed + # All schedules are daily → alert if the last success is + # older than 26h (24h cadence + 2h grace). + expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (26 * 3600) + for: 15m + labels: + team: infra + severity: warning + service: velero + component: backup + annotations: + summary: "Velero schedule missed daily window ({{ $labels.schedule }})" + description: | + Schedule `{{ $labels.schedule }}` has not produced a + successful backup in {{ $value | humanizeDuration }} + (>26h). Expected cadence: daily. + Check: `kubectl -n velero get schedule {{ $labels.schedule }}` + and Velero server logs. + - name: velero.storage + interval: 60s + rules: + - alert: VeleroStorageLocationUnavailable + # Substitutes for the spec's VeleroRepoUnhealthy — see file + # header. `velero_backup_location_status_gauge` is 1 when the + # BackupStorageLocation is Available, 0 when Unavailable. + expr: velero_backup_location_status_gauge == 0 + for: 15m + labels: + team: infra + severity: warning + service: velero + component: backup + annotations: + summary: "Velero BackupStorageLocation unavailable ({{ $labels.backup_location_name }})" + description: | + BackupStorageLocation `{{ $labels.backup_location_name }}` + has reported Unavailable for >15m. New backups will fail + until it is restored. + Check: `kubectl -n velero get backupstoragelocation` and + `velero backup-location get`. diff --git a/velero/servicemonitor.yaml b/velero/servicemonitor.yaml new file mode 100644 index 0000000..056a7d1 --- /dev/null +++ b/velero/servicemonitor.yaml @@ -0,0 +1,28 @@ +--- +# Scrape Velero's built-in Prometheus endpoint (http-monitoring / :8085) +# so kube-prometheus-stack can drive the accompanying PrometheusRule +# (velero/prometheusrule.yaml). The `release: kube-prometheus-stack` +# label is what the operator's serviceMonitorSelector matches. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: velero + namespace: velero + labels: + app.kubernetes.io/name: velero + app.kubernetes.io/part-of: velero + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - velero + selector: + matchLabels: + app.kubernetes.io/instance: velero + app.kubernetes.io/name: velero + endpoints: + - port: http-monitoring + path: /metrics + interval: 30s + scrapeTimeout: 10s + scheme: http