stack.basicstack.de/apps/monitoring/backup-restic-alerts.yaml

---
# Prometheus alerting rules for the restic-based monitoring backups
# (DEV-490 / DEV-482 Option 4). Freshness alerts fire when a
# CronJob has not emitted its `backup_<kind>_timestamp_seconds`
# metric within 28h (schedules are daily; 28h gives one missed
# run + 4h grace before we page).
#
# Check-status alerts fire on the first non-zero exit from
# `restic check --read-data-subset=5%`.
#
# Repo-size alerts fire when a repository exceeds 20 GiB — the
# baseline is expected < 5 GiB per repo. The alert catches
# retention/prune bugs and runaway growth. The metric is emitted
# by the same CronJob step (see backup-*-restic-cronjob.yaml,
# `restic stats --json --mode raw-data`).
#
# The metrics are written to the node's textfile-collector directory
# (`/var/lib/node_exporter/textfile_collector`) and scraped by the
# kube-prometheus-stack node-exporter DaemonSet (DEV-494).
#
# Because a CronJob may run on a different worker across days (loki
# backup follows the loki pod; k8s-resources is unpinned), stale
# `.prom` files can linger on nodes the job has since left. That
# would leave a per-node series with an old timestamp/check-status
# indefinitely. The alerts below aggregate across instances so a
# single fresh sample from the node where the job currently runs is
# enough to keep the freshness alert quiet, and check/size alerts
# fire when *any* node reports a bad value (which is the correct
# behaviour — a recent failure is still a signal).
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: backup-restic-alerts
  namespace: monitoring
  labels:
    app: backup
    backend: restic
    release: kube-prometheus-stack
spec:
  groups:
    - name: backup-restic.freshness
      interval: 60s
      rules:
        - alert: BackupLokiStale
          # `max()` collapses per-node samples so a stale `.prom` file
          # left behind on a node the loki backup no longer runs on
          # does not fire this alert; the freshest sample wins.
          expr: time() - max(backup_loki_timestamp_seconds) > 28 * 3600
          for: 15m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: loki
          annotations:
            summary: "Loki restic backup is stale (>28h)"
            description: |
              `backup-loki-restic` has not written a fresh
              `backup_loki_timestamp_seconds` sample in more than
              28 hours. Expected daily at 03:00 UTC.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupGrafanaStale
          expr: time() - max(backup_grafana_timestamp_seconds) > 28 * 3600
          for: 15m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: grafana
          annotations:
            summary: "Grafana restic backup is stale (>28h)"
            description: |
              `backup-grafana-restic` has not written a fresh
              `backup_grafana_timestamp_seconds` sample in more
              than 28 hours. Expected daily at 03:15 UTC.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupK8sResourcesStale
          expr: time() - max(backup_k8s_resources_timestamp_seconds) > 28 * 3600
          for: 15m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: k8s-resources
          annotations:
            summary: "K8s-resources restic backup is stale (>28h)"
            description: |
              `backup-k8s-resources` has not written a fresh
              `backup_k8s_resources_timestamp_seconds` sample in
              more than 28 hours. Expected daily at 02:00 UTC.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupPrometheusStale
          expr: time() - max(backup_prometheus_timestamp_seconds) > 28 * 3600
          for: 15m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: prometheus
          annotations:
            summary: "Prometheus restic backup is stale (>28h)"
            description: |
              `prometheus-backup` has not written a fresh
              `backup_prometheus_timestamp_seconds` sample in more
              than 28 hours. Expected daily at 03:30 UTC.
              Runbook: docs/monitoring/restic-restore.md
    - name: backup-restic.integrity
      interval: 60s
      rules:
        - alert: BackupLokiCheckFailed
          expr: backup_loki_check_status != 0
          for: 5m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: loki
          annotations:
            summary: "restic check failed on loki repo"
            description: |
              `restic check --read-data-subset=5%` returned
              exit code {{ $value }} on the loki repository.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupGrafanaCheckFailed
          expr: backup_grafana_check_status != 0
          for: 5m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: grafana
          annotations:
            summary: "restic check failed on grafana repo"
            description: |
              `restic check --read-data-subset=5%` returned
              exit code {{ $value }} on the grafana repository.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupK8sResourcesCheckFailed
          expr: backup_k8s_resources_check_status != 0
          for: 5m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: k8s-resources
          annotations:
            summary: "restic check failed on k8s-resources repo"
            description: |
              `restic check --read-data-subset=5%` returned
              exit code {{ $value }} on the k8s-resources
              repository.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupPrometheusCheckFailed
          expr: backup_prometheus_check_status != 0
          for: 5m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: prometheus
          annotations:
            summary: "restic check failed on prometheus repo"
            description: |
              `restic check --read-data-subset=5%` returned
              exit code {{ $value }} on the prometheus
              repository.
              Runbook: docs/monitoring/restic-restore.md
    - name: backup-restic.size
      interval: 60s
      rules:
        - alert: ResticRepoOversize
          # 20 GiB = 20 * 1024^3 = 21474836480 bytes.
          expr: restic_repo_size_bytes > 21474836480
          for: 30m
          labels:
            severity: warning
            service: monitoring
            component: backup
          annotations:
            summary: "restic repository {{ $labels.repo }} exceeds 20 GiB"
            description: |
              Repository {{ $labels.repo }} is
              {{ $value | humanize1024 }}B, above the 20 GiB
              guard. Baseline is <5 GiB per repo. Investigate
              retention/prune (see forget flags in the CronJob)
              and dedup effectiveness.