---
# Prometheus alerting rules for the restic-based monitoring backups
# (DEV-490 / DEV-482 Option 4). Freshness alerts fire when a
# CronJob has not emitted its `backup_<kind>_timestamp_seconds`
# metric within 28h (schedules are daily; 28h gives one missed
# run + 4h grace before we page).
#
# Check-status alerts fire on the first non-zero exit from
# `restic check --read-data-subset=5%`.
#
# Repo-size alerts fire when a repository exceeds 20 GiB — the
# baseline is expected < 5 GiB per repo. The alert catches
# retention/prune bugs and runaway growth. The metric is emitted
# by the same CronJob step (see backup-*-restic-cronjob.yaml,
# `restic stats --json --mode raw-data`).
#
# The metrics are written to an emptyDir at /metrics per job.
# They become scrapeable once node-exporter's textfile collector
# path is wired up — tracked as a follow-up. `promtool test rules`
# against apps/monitoring/backup-restic-alerts.test.yaml verifies
# the rules with synthetic samples in the meantime.
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: backup-restic-alerts
  namespace: monitoring
  labels:
    app: backup
    backend: restic
    release: kube-prometheus-stack
spec:
  groups:
    - name: backup-restic.freshness
      interval: 60s
      rules:
        - alert: BackupLokiStale
          expr: time() - backup_loki_timestamp_seconds > 28 * 3600
          for: 15m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: loki
          annotations:
            summary: "Loki restic backup is stale (>28h)"
            description: |
              `backup-loki-restic` has not written a fresh
              `backup_loki_timestamp_seconds` sample in more than
              28 hours. Expected daily at 03:00 UTC.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupGrafanaStale
          expr: time() - backup_grafana_timestamp_seconds > 28 * 3600
          for: 15m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: grafana
          annotations:
            summary: "Grafana restic backup is stale (>28h)"
            description: |
              `backup-grafana-restic` has not written a fresh
              `backup_grafana_timestamp_seconds` sample in more
              than 28 hours. Expected daily at 03:15 UTC.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupK8sResourcesStale
          expr: time() - backup_k8s_resources_timestamp_seconds > 28 * 3600
          for: 15m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: k8s-resources
          annotations:
            summary: "K8s-resources restic backup is stale (>28h)"
            description: |
              `backup-k8s-resources` has not written a fresh
              `backup_k8s_resources_timestamp_seconds` sample in
              more than 28 hours. Expected daily at 02:00 UTC.
              Runbook: docs/monitoring/restic-restore.md
    - name: backup-restic.integrity
      interval: 60s
      rules:
        - alert: BackupLokiCheckFailed
          expr: backup_loki_check_status != 0
          for: 5m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: loki
          annotations:
            summary: "restic check failed on loki repo"
            description: |
              `restic check --read-data-subset=5%` returned
              exit code {{ $value }} on the loki repository.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupGrafanaCheckFailed
          expr: backup_grafana_check_status != 0
          for: 5m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: grafana
          annotations:
            summary: "restic check failed on grafana repo"
            description: |
              `restic check --read-data-subset=5%` returned
              exit code {{ $value }} on the grafana repository.
              Runbook: docs/monitoring/restic-restore.md
        - alert: BackupK8sResourcesCheckFailed
          expr: backup_k8s_resources_check_status != 0
          for: 5m
          labels:
            severity: warning
            service: monitoring
            component: backup
            repo: k8s-resources
          annotations:
            summary: "restic check failed on k8s-resources repo"
            description: |
              `restic check --read-data-subset=5%` returned
              exit code {{ $value }} on the k8s-resources
              repository.
              Runbook: docs/monitoring/restic-restore.md
    - name: backup-restic.size
      interval: 60s
      rules:
        - alert: ResticRepoOversize
          # 20 GiB = 20 * 1024^3 = 21474836480 bytes.
          expr: restic_repo_size_bytes > 21474836480
          for: 30m
          labels:
            severity: warning
            service: monitoring
            component: backup
          annotations:
            summary: "restic repository {{ $labels.repo }} exceeds 20 GiB"
            description: |
              Repository {{ $labels.repo }} is
              {{ $value | humanize1024 }}B, above the 20 GiB
              guard. Baseline is <5 GiB per repo. Investigate
              retention/prune (see forget flags in the CronJob)
              and dedup effectiveness.