chore(monitoring): add restic alerts for forgejo repo (DEV-514)

New freshness (BackupForgejoStale, 28h) and integrity
(BackupForgejoCheckFailed) rules matching the sibling
loki/grafana/k8s-resources/prometheus alerts.

Metrics come from backup_forgejo_{timestamp_seconds,check_status,success}
emitted by apps/forgejo/forgejo-backup-cronjob.yaml.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

apps/forgejo/forgejo-backup-cronjob.yaml

@@ -1,3 +1,18 @@
+---
+# Forgejo pg_dump → restic → Hetzner Object Storage (DEV-514).
+#
+# Replaces the legacy volume-based backup that wrote pg_dump files to
+# an RWO PVC (`platform-backup-data`). The hcloud volume backing that
+# PVC was deleted externally on 2026-08-17; rather than re-provision
+# the same legacy pattern, this migrates to the restic→S3 pipeline
+# already used for loki/grafana/prometheus (DEV-485…DEV-492).
+#
+# initContainer runs pg_dump -F c into an emptyDir; main container
+# runs `restic backup --stdin-from-command` isn't used here because
+# the dump has to complete before restic starts (need a proper exit
+# code, and the custom-format dump is not resumable). Instead we
+# stage the dump on emptyDir and let restic dedup it into
+# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/forgejo`.
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -5,76 +20,185 @@ metadata:
   namespace: forgejo
   labels:
     app: forgejo-backup
+    backend: restic
 spec:
   schedule: "0 3 * * *"
   concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 7
+  successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 3
   jobTemplate:
+    metadata:
+      labels:
+        app: forgejo-backup
+        backend: restic
     spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 3600
       template:
+        metadata:
+          labels:
+            app: forgejo-backup
+            backend: restic
         spec:
           restartPolicy: OnFailure
+          initContainers:
+            - name: pgdump
+              image: postgres:16-alpine
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -eu
+                  echo "=== forgejo pg_dump started at $(date -u +%FT%TZ) ==="
+                  mkdir -p /dump
+                  PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \
+                    -h forgejo-postgres.forgejo.svc.cluster.local \
+                    -U "${POSTGRES_USER}" \
+                    -d "${POSTGRES_DB}" \
+                    -F c \
+                    -f /dump/forgejo-db.dump
+                  echo "pg_dump size: $(wc -c < /dump/forgejo-db.dump) bytes"
+                  # Manifest lets a restic-restore consumer identify the
+                  # dump without touching the binary.
+                  {
+                    echo "backup_timestamp=$(date -u +%Y%m%dT%H%M%SZ)"
+                    echo "type=postgresql_custom_dump"
+                    echo "database=${POSTGRES_DB}"
+                    echo "restore_cmd=PGPASSWORD=<pass> pg_restore -h <host> -U ${POSTGRES_USER} -d ${POSTGRES_DB} -F c forgejo-db.dump"
+                  } > /dump/manifest.txt
+                  echo "=== forgejo pg_dump finished at $(date -u +%FT%TZ) ==="
+              env:
+                - name: POSTGRES_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-postgres-secret
+                      key: POSTGRES_USER
+                - name: POSTGRES_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-postgres-secret
+                      key: POSTGRES_PASSWORD
+                - name: POSTGRES_DB
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-postgres-secret
+                      key: POSTGRES_DB
+              volumeMounts:
+                - name: dump
+                  mountPath: /dump
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 128Mi
+                limits:
+                  cpu: 1000m
+                  memory: 512Mi
           containers:
-          - name: forgejo-backup
-            image: postgres:16-alpine
-            imagePullPolicy: IfNotPresent
-            command:
-            - /bin/sh
-            - -lc
-            - |
-              set -euo pipefail
-              ts=$(date -u +%Y%m%dT%H%M%SZ)
-              BACKUP_DIR="/data/forgejo-${ts}"
-              mkdir -p "${BACKUP_DIR}"
+            - name: restic
+              image: harbor.basicstack.de/library/restic:0.17.3
+              env:
+                - name: AWS_ACCESS_KEY_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-s3-backup
+                      key: access-key
+                - name: AWS_SECRET_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-s3-backup
+                      key: secret-key
+                - name: RESTIC_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-s3-backup
+                      key: restic-password
+                - name: S3_ENDPOINT
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-s3-backup
+                      key: endpoint
+                - name: S3_BUCKET
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-s3-backup
+                      key: bucket
+                - name: RESTIC_REPOSITORY
+                  value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/forgejo"
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -eu
+                  echo "=== backup-forgejo-restic started at $(date -u +%FT%TZ) ==="
+                  echo "Repository: ${RESTIC_REPOSITORY}"
 
-              echo "[${ts}] Starting Forgejo backup..."
+                  if restic snapshots >/dev/null 2>&1; then
+                    echo "Repo exists, skipping init."
+                  else
+                    echo "Repo missing, initialising..."
+                    restic init
+                  fi
 
-              # Backup PostgreSQL database
-              echo "[$(date -u +%H:%M:%S)] Dumping PostgreSQL database..."
-              PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \
-                -h forgejo-postgres.forgejo.svc.cluster.local \
-                -U "${POSTGRES_USER}" \
-                -d "${POSTGRES_DB}" \
-                -F c \
-                -f "${BACKUP_DIR}/forgejo-db.dump"
+                  echo "--- restic backup /source ---"
+                  restic backup /source \
+                    --tag forgejo \
+                    --host k3s
 
-              echo "[$(date -u +%H:%M:%S)] Database backup complete: $(ls -lh ${BACKUP_DIR}/forgejo-db.dump | awk '{print $5}')"
+                  echo "--- restic forget/prune ---"
+                  restic forget --tag forgejo \
+                    --keep-daily 7 \
+                    --keep-weekly 4 \
+                    --keep-monthly 6 \
+                    --prune
 
-              # Create backup manifest
-              echo "backup_timestamp=${ts}" > "${BACKUP_DIR}/manifest.txt"
-              echo "type=postgresql_custom_dump" >> "${BACKUP_DIR}/manifest.txt"
-              echo "database=forgejo" >> "${BACKUP_DIR}/manifest.txt"
-              echo "restore_cmd=PGPASSWORD=<pass> pg_restore -h <host> -U forgejo -d forgejo -F c forgejo-db.dump" >> "${BACKUP_DIR}/manifest.txt"
+                  echo "--- restic check --read-data-subset=5% ---"
+                  CHECK_STATUS=0
+                  restic check --read-data-subset=5% || CHECK_STATUS=$?
+                  echo "restic check exit: ${CHECK_STATUS}"
 
-              # Cleanup old backups (keep 14 days)
-              find /data -maxdepth 1 -type d -name "forgejo-*" -mtime +14 -exec rm -rf {} + 2>/dev/null || true
+                  echo "--- restic stats (repo size) ---"
+                  REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
+                    | grep -oE '"total_size":[0-9]+' \
+                    | head -1 \
+                    | cut -d: -f2)
+                  REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
+                  echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
 
-              echo "[$(date -u +%H:%M:%S)] Backup complete. Files:"
-              ls -lh "${BACKUP_DIR}/"
+                  # Textfile-collector metrics, same wiring as
+                  # loki/grafana siblings (DEV-494). Atomic write.
+                  {
+                    echo "backup_forgejo_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
+                    echo "backup_forgejo_timestamp_seconds $(date +%s)"
+                    echo "backup_forgejo_check_status ${CHECK_STATUS}"
+                    echo "restic_repo_size_bytes{repo=\"forgejo\"} ${REPO_SIZE_BYTES}"
+                  } > /metrics/backup_forgejo.prom.tmp
+                  mv /metrics/backup_forgejo.prom.tmp /metrics/backup_forgejo.prom
 
-              echo "[$(date -u +%H:%M:%S)] All backups in storage:"
-              ls -1 /data | grep "forgejo-"
-            env:
-            - name: POSTGRES_USER
-              valueFrom:
-                secretKeyRef:
-                  name: forgejo-postgres-secret
-                  key: POSTGRES_USER
-            - name: POSTGRES_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: forgejo-postgres-secret
-                  key: POSTGRES_PASSWORD
-            - name: POSTGRES_DB
-              valueFrom:
-                secretKeyRef:
-                  name: forgejo-postgres-secret
-                  key: POSTGRES_DB
-            volumeMounts:
-            - name: backup-data
-              mountPath: /data
+                  echo "=== backup-forgejo-restic finished at $(date -u +%FT%TZ) ==="
+                  exit ${CHECK_STATUS}
+              volumeMounts:
+                - name: dump
+                  mountPath: /source
+                  readOnly: true
+                - name: metrics
+                  mountPath: /metrics
+                - name: cache
+                  mountPath: /root/.cache/restic
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 128Mi
+                limits:
+                  cpu: 1500m
+                  memory: 1Gi
           volumes:
-          - name: backup-data
-            persistentVolumeClaim:
-              claimName: platform-backup-data
+            - name: dump
+              emptyDir:
+                sizeLimit: 5Gi
+            - name: metrics
+              hostPath:
+                path: /var/lib/node_exporter/textfile_collector
+                type: DirectoryOrCreate
+            - name: cache
+              emptyDir: {}

apps/forgejo/forgejo-s3-backup-sealed.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: forgejo-s3-backup
+  namespace: forgejo
+spec:
+  encryptedData:
+    access-key: AgAu/XawJy0uhi3l8vwtd+loShP2ITnbOEZ/9eu6dkypz7l1MmFJhnEGq8Sy1lsg7BKj6epYRI40kDJIAG6KMBD5i7FvLYCvbFuPJaS3yehA8b1ECreHXvx/VYRUZL4K3WtiPO0fXmUNkxAqIcfHOcHtOZ7YcgSSyTw4DaXzngtek4xcTQlisyIVeBgc1dLPY+vHGn10BGkU5R/LnsUQqgGdp6u24Y+m7OrQGqMH8xwtZR7siEEkasu4IWG0QcY+okcMeiCi8mgAF64rdeZnIeEWqRnKHXxwEurtm20NmmCeRR8OgZfWDesgasThsnNSlRySd4EKlP3DMz8Yyluz/LlSkPt24brnP4qKuEoRpCHHOWgOizX8hkkS+Al7Rnpmxv2pt5CGlRw9s/5uPY81PAN9WLIgjs+XIylvaJBSW06nw3v+MjCUr5jfimW9riTLN32cBSurdpNMzlgb5+DJzEvaGNEG+jSjjFj2eSLci23UWySh7oFdmXeBp3kDWCrrNlJOYqJEwAAunTwvtM/1XknW2qJyyJrD7hbWN7SX4Ymb5Q5q18fMGH+IzB/Si1jdBli8pi8B2Gd/jo6RSmhXUheO3xsB2sXYc09GuSBBBBNzr9MsRL3mvUVr22pay+YPcCbwbwtdR0qmrdhEHAychYSJpNGuwGvs0AthRsNGN6hr1/IJqnnSRkGdJPr6k+T3v7XJzouYy+8Dh8r6oPW3hOhHo4HpJw==
+    bucket: AgAyQVCK+wE0wWxwVjx5DVnI2falTGNzsUrolv1YyoG2Bx6msYe2G00ovxCMJ6/dWws1Q/Sf6yoiFT0IByYflQFhY/ABzXRi7/gFxCWyYU/7yTo7w1Ue0+eEnBKFtkP/ZhIvtfubo6GDDtbmshpp4UtzPq7kB6qvE6oCFyf6awQD2pe/GKX8RZaDx5TCbxPxhedLkYdc7srR/8rFZBnQiXm/2ohtDOwvLA+RVOpnCW/bsVLvLYQmXyhszMv2M3zqXwM9ODYzQ4Y7lIw6T3Hsmel/1hHWJtN8/mlq+Q3Z/NbzhVeEWs8/xbK/ShC9RwZUgBWxAiW04HvwcxVYs60KaJ4oOMLtO4Iko4+iPTNonjra2wmziEpUtsotvtReRjwHdvk1bBn3q6uqzNaJFv9wLc/RVCvKSpqE5KOxmdbke9EDUs4wQHVQCAfFfwnzzpcF/Rf6ML3TUWc+U0W6UjCfVTWwROznpi0E18tA07apx+xm1y+MyuVYnSpZv++vXetnCjA+KFvqZSon/WnKQrmSO2RFR07FcMs6gQ+EOSaRXBPCnEcnF2IJUbJLdFsgpQbzYG6zo3m2KmxN7gaGrTUdhNg/N7LklnNbJML9KRa7e2Rv8t4NGoigOpZG6+soIijZOSfEJF6owN6MP3oecgUgNsnQ5ftKK/RznNaDtIeZ5KByHlQ7CsZ0MIkHBmoEkDt64xWNEoW4NrWPSxas6pOQGZMPMA==
+    endpoint: AgBhXqp3TdUU6GxsexRIoeHNwjvxnd2v4Any9WGs7gtyZvfdGIuWHIQ+JpsuRDrEq+BvTaX49D0QYoqnoQ6ZgmmGUI7kJxc+Ombcqo/8PAsOmzfdUaz1NcLAeR/UTGu8BDxAQcySSFax2dyK/zkY3jcsmuUCJNJLB57EZwtwx4fyz+ota+zNBE7pY+sDpWkXKHHtWrvNkD7tSLYHQuS3uzATidRixngL+iaH8rQX3ZK6KaK4H4ZdFG3/1Fw0wjLt8+yVBntBnh/bCL9/JLWQWIiHtooQnKXDdEwYQHWhrktti1KDzy/S++o4XVcRG1alzsQ0vV7uzSZAh4xCmnzximJxeJKawzFXRgFUcoEPrOI/nVedYrAU0DzkuwPdqtQDi/fGoqzZPx3PYhB9wv2PJsLg20ZlFcfHYUCad3O6OmO+VlrpYYcUe4oStkOygrmK5bv89fQ4Tov7BB14+RQIo9ogKmLzZu3xStqIVK+/Y8OUC64zyo7FpPMoYKtRW5oawu8pxo7Gtb+27BMMXQBiMEPNf38C4aj3GGJOvm4CFxVHV510/7o2hSW2ITt5X2RzNOEzULJOHD3DvO8wdWzvpBykRFnZ8JTYv77Kbx7bd9Ylx62VXm2a+Bd0K1Nc/sNJE63pglVhdl05oLYsUTrcSsNoJ6Dz3pMKeue3pYrn9KNurG0s+BFfO6I2Aa4/L8iAggXBzf9G1ALSkn7A30C9rMeyV+/1n+q8GpdPqq4ltqLOHVwIZw==
+    restic-password: AgB1JGFnSDe3N5ZaiT9xShfzUQiyDtGwH2wotYO5Ekk9nqAbS4xTNBXu/1Z1+cB8zVUP80hrPHHbVXjNu47AcLI+Bn8Ozn9fKIZFsnVHXqicG0Y+WHRMpQxuMijVEMl48oUNE+vw+xRhYTzibVWoxur+VEzyPCn8rLUX07tcNhY+jYptNVBYlVVXDrHcOs0s9wpvIlSWmc9RsINMu+6FaedXAs3nKY4gUpWxmNLEpLG8V7D2mNJZHk1adghTN9p8RMGcLtNylDY9W1Sdnziga/NG0WulZf+4IWV6e7wc2oBCSekTofCZuCva8HazJ1EIeqBvt2GuXoG1CjjbPNb2UwZeH21Iol1W0C5So1VFguxHZq5WL44mqD79NLlnSAMmQMSvxf7gXQVXEdtT5UlF9nYCkuQ+K+mT+8iRMy1YSCQSQ9kkD36unkrlYcLXKoPLYxHcoo7fGZ0MPIW8J+P77Hp9Z8jNPCyCaeSJYApCxybO8ujbUGdIr2FCP1z/JDP4/zZat9MkTA6G3H9kDNso7AJBHL/NfkWImvQnveLGj0CuhPpf2zwOylCqTeXrgoN8zztO2/qVDa3VRQbznap0njr0qzVrvOfTxYoiRmkGh3PXZ2mb/NqbeIvCFhj2VyVgavdMchlIbM3fCzXGEdCa2RN+ZYVtcFe/b6wR/qf7lrMaPHsTZiMfYuMJoC9F9wdbkOttT9c36Sob4x+aQYLs7/+XGCJdbw672qot6x6P7GHjY8zVKlMYLSZjKcoD48c=
+    secret-key: AgAHz7lfqTLG0Kldt/IZGW/o8KKazICHGbdVklXs3Gzs6Mc2KaG3nggkZDRWwY6noochg4Wc9mDDMGY3XIQnEy3PXs/rjQgwym0Dbvc54CvjWUcfmxCJyIqY1uCUxoXp/WAEV18lpWFSkVx+5U9wSccYVmkOJXNJ/MgH0vzJPaNW6xpYDg1XRC4nj72vwqyvZ/gM/LEX41NOpPjBROLGql3uBk5UfzNZG4dBun0aaPiYx5KKAbz8JAbK0umvOOOctTPSv1hIhyv9xJRRB67MW/J8T6mB2YfezsisXSSOAVNuE2JWrafHFxKq/hSNQHSrQRKjt18k6vfxmfkICbgQUZl/7AnoKi/f+76Feu+3wuMT+XPsaJN+iWb3jNzcb1T+Aiv4rGyIA7uCB6FNOc9Ngf5ngDQJw0nDNHDSnUoowgcE96kuPefU1rmn2V+JJcwoYdzYE3Ds48C0rNT+WSIit4551SiyWuj7iCRLhyEf5vhojxkuoyCN+EC1a50XwIkt223EBmy6FFPeBxql6HyJbOE313BPsQU6enPrTKNFd7iZYqJXq1k+SRuGwJhMeo1BBNhmVqxFGOpyxkSVs3VPybeXd82Zk/xF0SWKyBp3XQnLB39ZmT83Z5jwrYfkZmWkkdRvCME6NmlQtftRDDk3efGbgw7bz4Eq0d08KPCezsaDSAGNewonYy3UOiv/CNQg5hQ5G6R55y7+kEMV4nNkjGj0qF1hrZURS0mV4I/7H0OwXpDYiVetfaYI
+  template:
+    metadata:
+      creationTimestamp: null
+      name: forgejo-s3-backup
+      namespace: forgejo

apps/forgejo/platform-backup-data-pvc.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: platform-backup-data
-  namespace: forgejo
-  labels:
-    app: forgejo-backup
-spec:
-  accessModes:
-  - ReadWriteOnce
-  storageClassName: hcloud-volumes-encrypted
-  resources:
-    requests:
-      storage: 20Gi

apps/monitoring/backup-restic-alerts.yaml

@@ -104,6 +104,21 @@ spec:
               `backup_prometheus_timestamp_seconds` sample in more
               than 28 hours. Expected daily at 03:30 UTC.
               Runbook: docs/monitoring/restic-restore.md
+        - alert: BackupForgejoStale
+          expr: time() - max(backup_forgejo_timestamp_seconds) > 28 * 3600
+          for: 15m
+          labels:
+            severity: warning
+            service: monitoring
+            component: backup
+            repo: forgejo
+          annotations:
+            summary: "Forgejo restic backup is stale (>28h)"
+            description: |
+              `forgejo-backup` has not written a fresh
+              `backup_forgejo_timestamp_seconds` sample in more
+              than 28 hours. Expected daily at 03:00 UTC.
+              Runbook: docs/monitoring/restic-restore.md
     - name: backup-restic.integrity
       interval: 60s
       rules:
@@ -165,6 +180,21 @@ spec:
               exit code {{ $value }} on the prometheus
               repository.
               Runbook: docs/monitoring/restic-restore.md
+        - alert: BackupForgejoCheckFailed
+          expr: backup_forgejo_check_status != 0
+          for: 5m
+          labels:
+            severity: warning
+            service: monitoring
+            component: backup
+            repo: forgejo
+          annotations:
+            summary: "restic check failed on forgejo repo"
+            description: |
+              `restic check --read-data-subset=5%` returned
+              exit code {{ $value }} on the forgejo
+              repository.
+              Runbook: docs/monitoring/restic-restore.md
     - name: backup-restic.size
       interval: 60s
       rules: