---
# Velero alerting for cluster-reconstruction backups (DEV-569 / DEV-574).
#
# Metrics come from the Velero server on :8085/metrics, scraped via
# velero/servicemonitor.yaml.
#
# All our Velero Schedules are daily (see velero/schedules/*), so the
# freshness alert uses a 26h window (24h cadence + 2h grace).
#
# Notes:
#   - `velero_backup_last_successful_timestamp` is emitted only after a
#     schedule has had at least one successful run. Freshly created
#     schedules produce no series until their first success, so the
#     "missed" alert doesn't fire on day 0 — it fires once a schedule
#     has succeeded at least once and then goes >26h without another.
#   - `schedule=""` is Velero's rollup for one-off (non-Schedule)
#     backups; we filter it out so ad-hoc backups don't page.
#   - Velero v1.18.1 does not expose `velero_backup_repository_available`
#     (mentioned in the DEV-574 spec), so the repo-health alert here
#     watches `velero_backup_location_status_gauge` instead — that
#     covers the same operational failure mode ("backups can't land
#     because the target storage is unreachable").
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: velero-alerts
  namespace: velero
  labels:
    app.kubernetes.io/name: velero
    app.kubernetes.io/part-of: velero
    release: kube-prometheus-stack
spec:
  groups:
    - name: velero.backups
      interval: 60s
      rules:
        - alert: VeleroBackupFailed
          expr: sum by (schedule) (increase(velero_backup_failure_total{schedule!=""}[30m])) > 0
          for: 0m
          labels:
            team: infra
            severity: warning
            service: velero
            component: backup
          annotations:
            summary: "Velero backup failed ({{ $labels.schedule }})"
            description: |
              Velero recorded {{ $value | printf "%.0f" }} new backup
              failure(s) for schedule `{{ $labels.schedule }}` in the
              last 30 minutes.
              Investigate: `velero backup get -n velero` and
              `velero backup describe <name> --details`.
        - alert: VeleroBackupPartial
          expr: sum by (schedule) (increase(velero_backup_partial_failure_total{schedule!=""}[30m])) > 0
          for: 0m
          labels:
            team: infra
            severity: warning
            service: velero
            component: backup
          annotations:
            summary: "Velero backup completed with partial failures ({{ $labels.schedule }})"
            description: |
              Velero recorded {{ $value | printf "%.0f" }} partial
              failure(s) for schedule `{{ $labels.schedule }}` in the
              last 30 minutes. Some items were skipped or errored even
              though the backup finished.
              Investigate: `velero backup describe <name> --details`.
        - alert: VeleroBackupMissed
          # All schedules are daily → alert if the last success is
          # older than 26h (24h cadence + 2h grace).
          expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (26 * 3600)
          for: 15m
          labels:
            team: infra
            severity: warning
            service: velero
            component: backup
          annotations:
            summary: "Velero schedule missed daily window ({{ $labels.schedule }})"
            description: |
              Schedule `{{ $labels.schedule }}` has not produced a
              successful backup in {{ $value | humanizeDuration }}
              (>26h). Expected cadence: daily.
              Check: `kubectl -n velero get schedule {{ $labels.schedule }}`
              and Velero server logs.
    - name: velero.storage
      interval: 60s
      rules:
        - alert: VeleroStorageLocationUnavailable
          # Substitutes for the spec's VeleroRepoUnhealthy — see file
          # header. `velero_backup_location_status_gauge` is 1 when the
          # BackupStorageLocation is Available, 0 when Unavailable.
          expr: velero_backup_location_status_gauge == 0
          for: 15m
          labels:
            team: infra
            severity: warning
            service: velero
            component: backup
          annotations:
            summary: "Velero BackupStorageLocation unavailable ({{ $labels.backup_location_name }})"
            description: |
              BackupStorageLocation `{{ $labels.backup_location_name }}`
              has reported Unavailable for >15m. New backups will fail
              until it is restored.
              Check: `kubectl -n velero get backupstoragelocation` and
              `velero backup-location get`.