146 lines
5.4 KiB
YAML
146 lines
5.4 KiB
YAML
|
---
|
||
|
|
# Prometheus alerting rules for the restic-based monitoring backups
|
||
|
|
# (DEV-490 / DEV-482 Option 4). Freshness alerts fire when a
|
||
|
|
# CronJob has not emitted its `backup_<kind>_timestamp_seconds`
|
||
|
|
# metric within 28h (schedules are daily; 28h gives one missed
|
||
|
|
# run + 4h grace before we page).
|
||
|
|
#
|
||
|
|
# Check-status alerts fire on the first non-zero exit from
|
||
|
|
# `restic check --read-data-subset=5%`.
|
||
|
|
#
|
||
|
|
# Repo-size alerts fire when a repository exceeds 20 GiB — the
|
||
|
|
# baseline is expected < 5 GiB per repo. The alert catches
|
||
|
|
# retention/prune bugs and runaway growth. The metric is emitted
|
||
|
|
# by the same CronJob step (see backup-*-restic-cronjob.yaml,
|
||
|
|
# `restic stats --json --mode raw-data`).
|
||
|
|
#
|
||
|
|
# The metrics are written to an emptyDir at /metrics per job.
|
||
|
|
# They become scrapeable once node-exporter's textfile collector
|
||
|
|
# path is wired up — tracked as a follow-up. `promtool test rules`
|
||
|
|
# against apps/monitoring/backup-restic-alerts.test.yaml verifies
|
||
|
|
# the rules with synthetic samples in the meantime.
|
||
|
|
apiVersion: monitoring.coreos.com/v1
|
||
|
|
kind: PrometheusRule
|
||
|
|
metadata:
|
||
|
|
name: backup-restic-alerts
|
||
|
|
namespace: monitoring
|
||
|
|
labels:
|
||
|
|
app: backup
|
||
|
|
backend: restic
|
||
|
|
release: kube-prometheus-stack
|
||
|
|
spec:
|
||
|
|
groups:
|
||
|
|
- name: backup-restic.freshness
|
||
|
|
interval: 60s
|
||
|
|
rules:
|
||
|
|
- alert: BackupLokiStale
|
||
|
|
expr: time() - backup_loki_timestamp_seconds > 28 * 3600
|
||
|
|
for: 15m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
service: monitoring
|
||
|
|
component: backup
|
||
|
|
repo: loki
|
||
|
|
annotations:
|
||
|
|
summary: "Loki restic backup is stale (>28h)"
|
||
|
|
description: |
|
||
|
|
`backup-loki-restic` has not written a fresh
|
||
|
|
`backup_loki_timestamp_seconds` sample in more than
|
||
|
|
28 hours. Expected daily at 03:00 UTC.
|
||
|
|
Runbook: docs/monitoring/restic-restore.md
|
||
|
|
- alert: BackupGrafanaStale
|
||
|
|
expr: time() - backup_grafana_timestamp_seconds > 28 * 3600
|
||
|
|
for: 15m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
service: monitoring
|
||
|
|
component: backup
|
||
|
|
repo: grafana
|
||
|
|
annotations:
|
||
|
|
summary: "Grafana restic backup is stale (>28h)"
|
||
|
|
description: |
|
||
|
|
`backup-grafana-restic` has not written a fresh
|
||
|
|
`backup_grafana_timestamp_seconds` sample in more
|
||
|
|
than 28 hours. Expected daily at 03:15 UTC.
|
||
|
|
Runbook: docs/monitoring/restic-restore.md
|
||
|
|
- alert: BackupK8sResourcesStale
|
||
|
|
expr: time() - backup_k8s_resources_timestamp_seconds > 28 * 3600
|
||
|
|
for: 15m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
service: monitoring
|
||
|
|
component: backup
|
||
|
|
repo: k8s-resources
|
||
|
|
annotations:
|
||
|
|
summary: "K8s-resources restic backup is stale (>28h)"
|
||
|
|
description: |
|
||
|
|
`backup-k8s-resources` has not written a fresh
|
||
|
|
`backup_k8s_resources_timestamp_seconds` sample in
|
||
|
|
more than 28 hours. Expected daily at 02:00 UTC.
|
||
|
|
Runbook: docs/monitoring/restic-restore.md
|
||
|
|
- name: backup-restic.integrity
|
||
|
|
interval: 60s
|
||
|
|
rules:
|
||
|
|
- alert: BackupLokiCheckFailed
|
||
|
|
expr: backup_loki_check_status != 0
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
service: monitoring
|
||
|
|
component: backup
|
||
|
|
repo: loki
|
||
|
|
annotations:
|
||
|
|
summary: "restic check failed on loki repo"
|
||
|
|
description: |
|
||
|
|
`restic check --read-data-subset=5%` returned
|
||
|
|
exit code {{ $value }} on the loki repository.
|
||
|
|
Runbook: docs/monitoring/restic-restore.md
|
||
|
|
- alert: BackupGrafanaCheckFailed
|
||
|
|
expr: backup_grafana_check_status != 0
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
service: monitoring
|
||
|
|
component: backup
|
||
|
|
repo: grafana
|
||
|
|
annotations:
|
||
|
|
summary: "restic check failed on grafana repo"
|
||
|
|
description: |
|
||
|
|
`restic check --read-data-subset=5%` returned
|
||
|
|
exit code {{ $value }} on the grafana repository.
|
||
|
|
Runbook: docs/monitoring/restic-restore.md
|
||
|
|
- alert: BackupK8sResourcesCheckFailed
|
||
|
|
expr: backup_k8s_resources_check_status != 0
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
service: monitoring
|
||
|
|
component: backup
|
||
|
|
repo: k8s-resources
|
||
|
|
annotations:
|
||
|
|
summary: "restic check failed on k8s-resources repo"
|
||
|
|
description: |
|
||
|
|
`restic check --read-data-subset=5%` returned
|
||
|
|
exit code {{ $value }} on the k8s-resources
|
||
|
|
repository.
|
||
|
|
Runbook: docs/monitoring/restic-restore.md
|
||
|
|
- name: backup-restic.size
|
||
|
|
interval: 60s
|
||
|
|
rules:
|
||
|
|
- alert: ResticRepoOversize
|
||
|
|
# 20 GiB = 20 * 1024^3 = 21474836480 bytes.
|
||
|
|
expr: restic_repo_size_bytes > 21474836480
|
||
|
|
for: 30m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
service: monitoring
|
||
|
|
component: backup
|
||
|
|
annotations:
|
||
|
|
summary: "restic repository {{ $labels.repo }} exceeds 20 GiB"
|
||
|
|
description: |
|
||
|
|
Repository {{ $labels.repo }} is
|
||
|
|
{{ $value | humanize1024 }}B, above the 20 GiB
|
||
|
|
guard. Baseline is <5 GiB per repo. Investigate
|
||
|
|
retention/prune (see forget flags in the CronJob)
|
||
|
|
and dedup effectiveness.
|