apps/forgejo/forgejo-backup-cronjob.yaml

@@ -1,18 +1,3 @@
----
-# Forgejo pg_dump → restic → Hetzner Object Storage (DEV-514).
-#
-# Replaces the legacy volume-based backup that wrote pg_dump files to
-# an RWO PVC (`platform-backup-data`). The hcloud volume backing that
-# PVC was deleted externally on 2026-08-17; rather than re-provision
-# the same legacy pattern, this migrates to the restic→S3 pipeline
-# already used for loki/grafana/prometheus (DEV-485…DEV-492).
-#
-# initContainer runs pg_dump -F c into an emptyDir; main container
-# runs `restic backup --stdin-from-command` isn't used here because
-# the dump has to complete before restic starts (need a proper exit
-# code, and the custom-format dump is not resumable). Instead we
-# stage the dump on emptyDir and let restic dedup it into
-# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/forgejo`.
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -20,54 +5,56 @@ metadata:
   namespace: forgejo
   labels:
     app: forgejo-backup
-    backend: restic
 spec:
   schedule: "0 3 * * *"
   concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 3
+  successfulJobsHistoryLimit: 7
   failedJobsHistoryLimit: 3
   jobTemplate:
-    metadata:
-      labels:
-        app: forgejo-backup
-        backend: restic
     spec:
-      backoffLimit: 2
-      activeDeadlineSeconds: 3600
       template:
-        metadata:
-          labels:
-            app: forgejo-backup
-            backend: restic
         spec:
           restartPolicy: OnFailure
-          initContainers:
+          containers:
-            - name: pgdump
+          - name: forgejo-backup
             image: postgres:16-alpine
             imagePullPolicy: IfNotPresent
             command:
             - /bin/sh
-                - -c
+            - -lc
             - |
-                  set -eu
+              set -euo pipefail
-                  echo "=== forgejo pg_dump started at $(date -u +%FT%TZ) ==="
+              ts=$(date -u +%Y%m%dT%H%M%SZ)
-                  mkdir -p /dump
+              BACKUP_DIR="/data/forgejo-${ts}"
+              mkdir -p "${BACKUP_DIR}"
+
+              echo "[${ts}] Starting Forgejo backup..."
+
+              # Backup PostgreSQL database
+              echo "[$(date -u +%H:%M:%S)] Dumping PostgreSQL database..."
               PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \
                 -h forgejo-postgres.forgejo.svc.cluster.local \
                 -U "${POSTGRES_USER}" \
                 -d "${POSTGRES_DB}" \
                 -F c \
-                    -f /dump/forgejo-db.dump
+                -f "${BACKUP_DIR}/forgejo-db.dump"
-                  echo "pg_dump size: $(wc -c < /dump/forgejo-db.dump) bytes"
+
-                  # Manifest lets a restic-restore consumer identify the
+              echo "[$(date -u +%H:%M:%S)] Database backup complete: $(ls -lh ${BACKUP_DIR}/forgejo-db.dump | awk '{print $5}')"
-                  # dump without touching the binary.
+
-                  {
+              # Create backup manifest
-                    echo "backup_timestamp=$(date -u +%Y%m%dT%H%M%SZ)"
+              echo "backup_timestamp=${ts}" > "${BACKUP_DIR}/manifest.txt"
-                    echo "type=postgresql_custom_dump"
+              echo "type=postgresql_custom_dump" >> "${BACKUP_DIR}/manifest.txt"
-                    echo "database=${POSTGRES_DB}"
+              echo "database=forgejo" >> "${BACKUP_DIR}/manifest.txt"
-                    echo "restore_cmd=PGPASSWORD=<pass> pg_restore -h <host> -U ${POSTGRES_USER} -d ${POSTGRES_DB} -F c forgejo-db.dump"
+              echo "restore_cmd=PGPASSWORD=<pass> pg_restore -h <host> -U forgejo -d forgejo -F c forgejo-db.dump" >> "${BACKUP_DIR}/manifest.txt"
-                  } > /dump/manifest.txt
+
-                  echo "=== forgejo pg_dump finished at $(date -u +%FT%TZ) ==="
+              # Cleanup old backups (keep 14 days)
+              find /data -maxdepth 1 -type d -name "forgejo-*" -mtime +14 -exec rm -rf {} + 2>/dev/null || true
+
+              echo "[$(date -u +%H:%M:%S)] Backup complete. Files:"
+              ls -lh "${BACKUP_DIR}/"
+
+              echo "[$(date -u +%H:%M:%S)] All backups in storage:"
+              ls -1 /data | grep "forgejo-"
             env:
             - name: POSTGRES_USER
               valueFrom:
@@ -85,120 +72,9 @@ spec:
                   name: forgejo-postgres-secret
                   key: POSTGRES_DB
             volumeMounts:
-                - name: dump
+            - name: backup-data
-                  mountPath: /dump
+              mountPath: /data
-              resources:
-                requests:
-                  cpu: 100m
-                  memory: 128Mi
-                limits:
-                  cpu: 1000m
-                  memory: 512Mi
-          containers:
-            - name: restic
-              image: harbor.basicstack.de/library/restic:0.17.3
-              env:
-                - name: AWS_ACCESS_KEY_ID
-                  valueFrom:
-                    secretKeyRef:
-                      name: forgejo-s3-backup
-                      key: access-key
-                - name: AWS_SECRET_ACCESS_KEY
-                  valueFrom:
-                    secretKeyRef:
-                      name: forgejo-s3-backup
-                      key: secret-key
-                - name: RESTIC_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: forgejo-s3-backup
-                      key: restic-password
-                - name: S3_ENDPOINT
-                  valueFrom:
-                    secretKeyRef:
-                      name: forgejo-s3-backup
-                      key: endpoint
-                - name: S3_BUCKET
-                  valueFrom:
-                    secretKeyRef:
-                      name: forgejo-s3-backup
-                      key: bucket
-                - name: RESTIC_REPOSITORY
-                  value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/forgejo"
-              command:
-                - /bin/sh
-                - -c
-                - |
-                  set -eu
-                  echo "=== backup-forgejo-restic started at $(date -u +%FT%TZ) ==="
-                  echo "Repository: ${RESTIC_REPOSITORY}"
-
-                  if restic snapshots >/dev/null 2>&1; then
-                    echo "Repo exists, skipping init."
-                  else
-                    echo "Repo missing, initialising..."
-                    restic init
-                  fi
-
-                  echo "--- restic backup /source ---"
-                  restic backup /source \
-                    --tag forgejo \
-                    --host k3s
-
-                  echo "--- restic forget/prune ---"
-                  restic forget --tag forgejo \
-                    --keep-daily 7 \
-                    --keep-weekly 4 \
-                    --keep-monthly 6 \
-                    --prune
-
-                  echo "--- restic check --read-data-subset=5% ---"
-                  CHECK_STATUS=0
-                  restic check --read-data-subset=5% || CHECK_STATUS=$?
-                  echo "restic check exit: ${CHECK_STATUS}"
-
-                  echo "--- restic stats (repo size) ---"
-                  REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
-                    | grep -oE '"total_size":[0-9]+' \
-                    | head -1 \
-                    | cut -d: -f2)
-                  REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
-                  echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
-
-                  # Textfile-collector metrics, same wiring as
-                  # loki/grafana siblings (DEV-494). Atomic write.
-                  {
-                    echo "backup_forgejo_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
-                    echo "backup_forgejo_timestamp_seconds $(date +%s)"
-                    echo "backup_forgejo_check_status ${CHECK_STATUS}"
-                    echo "restic_repo_size_bytes{repo=\"forgejo\"} ${REPO_SIZE_BYTES}"
-                  } > /metrics/backup_forgejo.prom.tmp
-                  mv /metrics/backup_forgejo.prom.tmp /metrics/backup_forgejo.prom
-
-                  echo "=== backup-forgejo-restic finished at $(date -u +%FT%TZ) ==="
-                  exit ${CHECK_STATUS}
-              volumeMounts:
-                - name: dump
-                  mountPath: /source
-                  readOnly: true
-                - name: metrics
-                  mountPath: /metrics
-                - name: cache
-                  mountPath: /root/.cache/restic
-              resources:
-                requests:
-                  cpu: 100m
-                  memory: 128Mi
-                limits:
-                  cpu: 1500m
-                  memory: 1Gi
           volumes:
-            - name: dump
+          - name: backup-data
-              emptyDir:
+            persistentVolumeClaim:
-                sizeLimit: 5Gi
+              claimName: platform-backup-data
-            - name: metrics
-              hostPath:
-                path: /var/lib/node_exporter/textfile_collector
-                type: DirectoryOrCreate
-            - name: cache
-              emptyDir: {}

apps/forgejo/forgejo-s3-backup-sealed.yaml

@@ -1,19 +0,0 @@
----
-apiVersion: bitnami.com/v1alpha1
-kind: SealedSecret
-metadata:
-  creationTimestamp: null
-  name: forgejo-s3-backup
-  namespace: forgejo
-spec:
-  encryptedData:
-    access-key: AgAu/XawJy0uhi3l8vwtd+loShP2ITnbOEZ/9eu6dkypz7l1MmFJhnEGq8Sy1lsg7BKj6epYRI40kDJIAG6KMBD5i7FvLYCvbFuPJaS3yehA8b1ECreHXvx/VYRUZL4K3WtiPO0fXmUNkxAqIcfHOcHtOZ7YcgSSyTw4DaXzngtek4xcTQlisyIVeBgc1dLPY+vHGn10BGkU5R/LnsUQqgGdp6u24Y+m7OrQGqMH8xwtZR7siEEkasu4IWG0QcY+okcMeiCi8mgAF64rdeZnIeEWqRnKHXxwEurtm20NmmCeRR8OgZfWDesgasThsnNSlRySd4EKlP3DMz8Yyluz/LlSkPt24brnP4qKuEoRpCHHOWgOizX8hkkS+Al7Rnpmxv2pt5CGlRw9s/5uPY81PAN9WLIgjs+XIylvaJBSW06nw3v+MjCUr5jfimW9riTLN32cBSurdpNMzlgb5+DJzEvaGNEG+jSjjFj2eSLci23UWySh7oFdmXeBp3kDWCrrNlJOYqJEwAAunTwvtM/1XknW2qJyyJrD7hbWN7SX4Ymb5Q5q18fMGH+IzB/Si1jdBli8pi8B2Gd/jo6RSmhXUheO3xsB2sXYc09GuSBBBBNzr9MsRL3mvUVr22pay+YPcCbwbwtdR0qmrdhEHAychYSJpNGuwGvs0AthRsNGN6hr1/IJqnnSRkGdJPr6k+T3v7XJzouYy+8Dh8r6oPW3hOhHo4HpJw==
-    bucket: AgAyQVCK+wE0wWxwVjx5DVnI2falTGNzsUrolv1YyoG2Bx6msYe2G00ovxCMJ6/dWws1Q/Sf6yoiFT0IByYflQFhY/ABzXRi7/gFxCWyYU/7yTo7w1Ue0+eEnBKFtkP/ZhIvtfubo6GDDtbmshpp4UtzPq7kB6qvE6oCFyf6awQD2pe/GKX8RZaDx5TCbxPxhedLkYdc7srR/8rFZBnQiXm/2ohtDOwvLA+RVOpnCW/bsVLvLYQmXyhszMv2M3zqXwM9ODYzQ4Y7lIw6T3Hsmel/1hHWJtN8/mlq+Q3Z/NbzhVeEWs8/xbK/ShC9RwZUgBWxAiW04HvwcxVYs60KaJ4oOMLtO4Iko4+iPTNonjra2wmziEpUtsotvtReRjwHdvk1bBn3q6uqzNaJFv9wLc/RVCvKSpqE5KOxmdbke9EDUs4wQHVQCAfFfwnzzpcF/Rf6ML3TUWc+U0W6UjCfVTWwROznpi0E18tA07apx+xm1y+MyuVYnSpZv++vXetnCjA+KFvqZSon/WnKQrmSO2RFR07FcMs6gQ+EOSaRXBPCnEcnF2IJUbJLdFsgpQbzYG6zo3m2KmxN7gaGrTUdhNg/N7LklnNbJML9KRa7e2Rv8t4NGoigOpZG6+soIijZOSfEJF6owN6MP3oecgUgNsnQ5ftKK/RznNaDtIeZ5KByHlQ7CsZ0MIkHBmoEkDt64xWNEoW4NrWPSxas6pOQGZMPMA==
-    endpoint: AgBhXqp3TdUU6GxsexRIoeHNwjvxnd2v4Any9WGs7gtyZvfdGIuWHIQ+JpsuRDrEq+BvTaX49D0QYoqnoQ6ZgmmGUI7kJxc+Ombcqo/8PAsOmzfdUaz1NcLAeR/UTGu8BDxAQcySSFax2dyK/zkY3jcsmuUCJNJLB57EZwtwx4fyz+ota+zNBE7pY+sDpWkXKHHtWrvNkD7tSLYHQuS3uzATidRixngL+iaH8rQX3ZK6KaK4H4ZdFG3/1Fw0wjLt8+yVBntBnh/bCL9/JLWQWIiHtooQnKXDdEwYQHWhrktti1KDzy/S++o4XVcRG1alzsQ0vV7uzSZAh4xCmnzximJxeJKawzFXRgFUcoEPrOI/nVedYrAU0DzkuwPdqtQDi/fGoqzZPx3PYhB9wv2PJsLg20ZlFcfHYUCad3O6OmO+VlrpYYcUe4oStkOygrmK5bv89fQ4Tov7BB14+RQIo9ogKmLzZu3xStqIVK+/Y8OUC64zyo7FpPMoYKtRW5oawu8pxo7Gtb+27BMMXQBiMEPNf38C4aj3GGJOvm4CFxVHV510/7o2hSW2ITt5X2RzNOEzULJOHD3DvO8wdWzvpBykRFnZ8JTYv77Kbx7bd9Ylx62VXm2a+Bd0K1Nc/sNJE63pglVhdl05oLYsUTrcSsNoJ6Dz3pMKeue3pYrn9KNurG0s+BFfO6I2Aa4/L8iAggXBzf9G1ALSkn7A30C9rMeyV+/1n+q8GpdPqq4ltqLOHVwIZw==
-    restic-password: AgB1JGFnSDe3N5ZaiT9xShfzUQiyDtGwH2wotYO5Ekk9nqAbS4xTNBXu/1Z1+cB8zVUP80hrPHHbVXjNu47AcLI+Bn8Ozn9fKIZFsnVHXqicG0Y+WHRMpQxuMijVEMl48oUNE+vw+xRhYTzibVWoxur+VEzyPCn8rLUX07tcNhY+jYptNVBYlVVXDrHcOs0s9wpvIlSWmc9RsINMu+6FaedXAs3nKY4gUpWxmNLEpLG8V7D2mNJZHk1adghTN9p8RMGcLtNylDY9W1Sdnziga/NG0WulZf+4IWV6e7wc2oBCSekTofCZuCva8HazJ1EIeqBvt2GuXoG1CjjbPNb2UwZeH21Iol1W0C5So1VFguxHZq5WL44mqD79NLlnSAMmQMSvxf7gXQVXEdtT5UlF9nYCkuQ+K+mT+8iRMy1YSCQSQ9kkD36unkrlYcLXKoPLYxHcoo7fGZ0MPIW8J+P77Hp9Z8jNPCyCaeSJYApCxybO8ujbUGdIr2FCP1z/JDP4/zZat9MkTA6G3H9kDNso7AJBHL/NfkWImvQnveLGj0CuhPpf2zwOylCqTeXrgoN8zztO2/qVDa3VRQbznap0njr0qzVrvOfTxYoiRmkGh3PXZ2mb/NqbeIvCFhj2VyVgavdMchlIbM3fCzXGEdCa2RN+ZYVtcFe/b6wR/qf7lrMaPHsTZiMfYuMJoC9F9wdbkOttT9c36Sob4x+aQYLs7/+XGCJdbw672qot6x6P7GHjY8zVKlMYLSZjKcoD48c=
-    secret-key: AgAHz7lfqTLG0Kldt/IZGW/o8KKazICHGbdVklXs3Gzs6Mc2KaG3nggkZDRWwY6noochg4Wc9mDDMGY3XIQnEy3PXs/rjQgwym0Dbvc54CvjWUcfmxCJyIqY1uCUxoXp/WAEV18lpWFSkVx+5U9wSccYVmkOJXNJ/MgH0vzJPaNW6xpYDg1XRC4nj72vwqyvZ/gM/LEX41NOpPjBROLGql3uBk5UfzNZG4dBun0aaPiYx5KKAbz8JAbK0umvOOOctTPSv1hIhyv9xJRRB67MW/J8T6mB2YfezsisXSSOAVNuE2JWrafHFxKq/hSNQHSrQRKjt18k6vfxmfkICbgQUZl/7AnoKi/f+76Feu+3wuMT+XPsaJN+iWb3jNzcb1T+Aiv4rGyIA7uCB6FNOc9Ngf5ngDQJw0nDNHDSnUoowgcE96kuPefU1rmn2V+JJcwoYdzYE3Ds48C0rNT+WSIit4551SiyWuj7iCRLhyEf5vhojxkuoyCN+EC1a50XwIkt223EBmy6FFPeBxql6HyJbOE313BPsQU6enPrTKNFd7iZYqJXq1k+SRuGwJhMeo1BBNhmVqxFGOpyxkSVs3VPybeXd82Zk/xF0SWKyBp3XQnLB39ZmT83Z5jwrYfkZmWkkdRvCME6NmlQtftRDDk3efGbgw7bz4Eq0d08KPCezsaDSAGNewonYy3UOiv/CNQg5hQ5G6R55y7+kEMV4nNkjGj0qF1hrZURS0mV4I/7H0OwXpDYiVetfaYI
-  template:
-    metadata:
-      creationTimestamp: null
-      name: forgejo-s3-backup
-      namespace: forgejo

apps/forgejo/platform-backup-data-pvc.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: platform-backup-data
+  namespace: forgejo
+  labels:
+    app: forgejo-backup
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: hcloud-volumes-encrypted
+  resources:
+    requests:
+      storage: 20Gi

apps/monitoring/backup-restic-alerts.yaml

@@ -104,21 +104,6 @@ spec:
               `backup_prometheus_timestamp_seconds` sample in more
               than 28 hours. Expected daily at 03:30 UTC.
               Runbook: docs/monitoring/restic-restore.md
-        - alert: BackupForgejoStale
-          expr: time() - max(backup_forgejo_timestamp_seconds) > 28 * 3600
-          for: 15m
-          labels:
-            severity: warning
-            service: monitoring
-            component: backup
-            repo: forgejo
-          annotations:
-            summary: "Forgejo restic backup is stale (>28h)"
-            description: |
-              `forgejo-backup` has not written a fresh
-              `backup_forgejo_timestamp_seconds` sample in more
-              than 28 hours. Expected daily at 03:00 UTC.
-              Runbook: docs/monitoring/restic-restore.md
     - name: backup-restic.integrity
       interval: 60s
       rules:
@@ -180,21 +165,6 @@ spec:
               exit code {{ $value }} on the prometheus
               repository.
               Runbook: docs/monitoring/restic-restore.md
-        - alert: BackupForgejoCheckFailed
-          expr: backup_forgejo_check_status != 0
-          for: 5m
-          labels:
-            severity: warning
-            service: monitoring
-            component: backup
-            repo: forgejo
-          annotations:
-            summary: "restic check failed on forgejo repo"
-            description: |
-              `restic check --read-data-subset=5%` returned
-              exit code {{ $value }} on the forgejo
-              repository.
-              Runbook: docs/monitoring/restic-restore.md
     - name: backup-restic.size
       interval: 60s
       rules: