From 585d6ee1140a40b5ff51f1f2cfb4a31d9bbbe36c Mon Sep 17 00:00:00 2001 From: CTO Agent Date: Sun, 23 Aug 2026 01:13:28 +0000 Subject: [PATCH] feat(forgejo-backup): migrate to pg_dump -> restic -> Hetzner S3 (DEV-514) Replaces the legacy volume-based backup that wrote pg_dump files to `platform-backup-data` (RWO PVC on hcloud volume 106575948). That volume was deleted externally on 2026-08-17; instead of re-provisioning the legacy pattern, this migrates forgejo to the restic->S3 pipeline already in use for loki/grafana/prometheus (DEV-485..DEV-492). - initContainer runs pg_dump -F c into emptyDir (5 GiB cap) - main container: restic backup /source, tag forgejo, repo s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/forgejo - forget/prune (7d/4w/6m), check --read-data-subset=5% - textfile-collector metrics -> backup_forgejo.prom (DEV-494 wiring) - SealedSecret forgejo-s3-backup mirrors monitoring-s3-backup fields - retire platform-backup-data PVC manifest Verified manually 2026-08-23 01:11 UTC: forgejo-backup-manual-1787447497 -> snapshot a961a473, repo 192 KiB. Orphan PV/PVC (`platform-backup-data`, `pvc-de59ae9c-...`) deleted. Co-Authored-By: Paperclip --- apps/forgejo/forgejo-backup-cronjob.yaml | 234 ++++++++++++++++----- apps/forgejo/forgejo-s3-backup-sealed.yaml | 19 ++ apps/forgejo/platform-backup-data-pvc.yaml | 14 -- 3 files changed, 198 insertions(+), 69 deletions(-) create mode 100644 apps/forgejo/forgejo-s3-backup-sealed.yaml delete mode 100644 apps/forgejo/platform-backup-data-pvc.yaml diff --git a/apps/forgejo/forgejo-backup-cronjob.yaml b/apps/forgejo/forgejo-backup-cronjob.yaml index bcd11fc..4979b6a 100644 --- a/apps/forgejo/forgejo-backup-cronjob.yaml +++ b/apps/forgejo/forgejo-backup-cronjob.yaml @@ -1,3 +1,18 @@ +--- +# Forgejo pg_dump → restic → Hetzner Object Storage (DEV-514). +# +# Replaces the legacy volume-based backup that wrote pg_dump files to +# an RWO PVC (`platform-backup-data`). The hcloud volume backing that +# PVC was deleted externally on 2026-08-17; rather than re-provision +# the same legacy pattern, this migrates to the restic→S3 pipeline +# already used for loki/grafana/prometheus (DEV-485…DEV-492). +# +# initContainer runs pg_dump -F c into an emptyDir; main container +# runs `restic backup --stdin-from-command` isn't used here because +# the dump has to complete before restic starts (need a proper exit +# code, and the custom-format dump is not resumable). Instead we +# stage the dump on emptyDir and let restic dedup it into +# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/forgejo`. apiVersion: batch/v1 kind: CronJob metadata: @@ -5,76 +20,185 @@ metadata: namespace: forgejo labels: app: forgejo-backup + backend: restic spec: schedule: "0 3 * * *" concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 7 + successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 jobTemplate: + metadata: + labels: + app: forgejo-backup + backend: restic spec: + backoffLimit: 2 + activeDeadlineSeconds: 3600 template: + metadata: + labels: + app: forgejo-backup + backend: restic spec: restartPolicy: OnFailure + initContainers: + - name: pgdump + image: postgres:16-alpine + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + set -eu + echo "=== forgejo pg_dump started at $(date -u +%FT%TZ) ===" + mkdir -p /dump + PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \ + -h forgejo-postgres.forgejo.svc.cluster.local \ + -U "${POSTGRES_USER}" \ + -d "${POSTGRES_DB}" \ + -F c \ + -f /dump/forgejo-db.dump + echo "pg_dump size: $(wc -c < /dump/forgejo-db.dump) bytes" + # Manifest lets a restic-restore consumer identify the + # dump without touching the binary. + { + echo "backup_timestamp=$(date -u +%Y%m%dT%H%M%SZ)" + echo "type=postgresql_custom_dump" + echo "database=${POSTGRES_DB}" + echo "restore_cmd=PGPASSWORD= pg_restore -h -U ${POSTGRES_USER} -d ${POSTGRES_DB} -F c forgejo-db.dump" + } > /dump/manifest.txt + echo "=== forgejo pg_dump finished at $(date -u +%FT%TZ) ===" + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: forgejo-postgres-secret + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: forgejo-postgres-secret + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: forgejo-postgres-secret + key: POSTGRES_DB + volumeMounts: + - name: dump + mountPath: /dump + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1000m + memory: 512Mi containers: - - name: forgejo-backup - image: postgres:16-alpine - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -lc - - | - set -euo pipefail - ts=$(date -u +%Y%m%dT%H%M%SZ) - BACKUP_DIR="/data/forgejo-${ts}" - mkdir -p "${BACKUP_DIR}" + - name: restic + image: harbor.basicstack.de/library/restic:0.17.3 + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: forgejo-s3-backup + key: access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: forgejo-s3-backup + key: secret-key + - name: RESTIC_PASSWORD + valueFrom: + secretKeyRef: + name: forgejo-s3-backup + key: restic-password + - name: S3_ENDPOINT + valueFrom: + secretKeyRef: + name: forgejo-s3-backup + key: endpoint + - name: S3_BUCKET + valueFrom: + secretKeyRef: + name: forgejo-s3-backup + key: bucket + - name: RESTIC_REPOSITORY + value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/forgejo" + command: + - /bin/sh + - -c + - | + set -eu + echo "=== backup-forgejo-restic started at $(date -u +%FT%TZ) ===" + echo "Repository: ${RESTIC_REPOSITORY}" - echo "[${ts}] Starting Forgejo backup..." + if restic snapshots >/dev/null 2>&1; then + echo "Repo exists, skipping init." + else + echo "Repo missing, initialising..." + restic init + fi - # Backup PostgreSQL database - echo "[$(date -u +%H:%M:%S)] Dumping PostgreSQL database..." - PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \ - -h forgejo-postgres.forgejo.svc.cluster.local \ - -U "${POSTGRES_USER}" \ - -d "${POSTGRES_DB}" \ - -F c \ - -f "${BACKUP_DIR}/forgejo-db.dump" + echo "--- restic backup /source ---" + restic backup /source \ + --tag forgejo \ + --host k3s - echo "[$(date -u +%H:%M:%S)] Database backup complete: $(ls -lh ${BACKUP_DIR}/forgejo-db.dump | awk '{print $5}')" + echo "--- restic forget/prune ---" + restic forget --tag forgejo \ + --keep-daily 7 \ + --keep-weekly 4 \ + --keep-monthly 6 \ + --prune - # Create backup manifest - echo "backup_timestamp=${ts}" > "${BACKUP_DIR}/manifest.txt" - echo "type=postgresql_custom_dump" >> "${BACKUP_DIR}/manifest.txt" - echo "database=forgejo" >> "${BACKUP_DIR}/manifest.txt" - echo "restore_cmd=PGPASSWORD= pg_restore -h -U forgejo -d forgejo -F c forgejo-db.dump" >> "${BACKUP_DIR}/manifest.txt" + echo "--- restic check --read-data-subset=5% ---" + CHECK_STATUS=0 + restic check --read-data-subset=5% || CHECK_STATUS=$? + echo "restic check exit: ${CHECK_STATUS}" - # Cleanup old backups (keep 14 days) - find /data -maxdepth 1 -type d -name "forgejo-*" -mtime +14 -exec rm -rf {} + 2>/dev/null || true + echo "--- restic stats (repo size) ---" + REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \ + | grep -oE '"total_size":[0-9]+' \ + | head -1 \ + | cut -d: -f2) + REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0} + echo "restic repo size: ${REPO_SIZE_BYTES} bytes" - echo "[$(date -u +%H:%M:%S)] Backup complete. Files:" - ls -lh "${BACKUP_DIR}/" + # Textfile-collector metrics, same wiring as + # loki/grafana siblings (DEV-494). Atomic write. + { + echo "backup_forgejo_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)" + echo "backup_forgejo_timestamp_seconds $(date +%s)" + echo "backup_forgejo_check_status ${CHECK_STATUS}" + echo "restic_repo_size_bytes{repo=\"forgejo\"} ${REPO_SIZE_BYTES}" + } > /metrics/backup_forgejo.prom.tmp + mv /metrics/backup_forgejo.prom.tmp /metrics/backup_forgejo.prom - echo "[$(date -u +%H:%M:%S)] All backups in storage:" - ls -1 /data | grep "forgejo-" - env: - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: forgejo-postgres-secret - key: POSTGRES_USER - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: forgejo-postgres-secret - key: POSTGRES_PASSWORD - - name: POSTGRES_DB - valueFrom: - secretKeyRef: - name: forgejo-postgres-secret - key: POSTGRES_DB - volumeMounts: - - name: backup-data - mountPath: /data + echo "=== backup-forgejo-restic finished at $(date -u +%FT%TZ) ===" + exit ${CHECK_STATUS} + volumeMounts: + - name: dump + mountPath: /source + readOnly: true + - name: metrics + mountPath: /metrics + - name: cache + mountPath: /root/.cache/restic + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1500m + memory: 1Gi volumes: - - name: backup-data - persistentVolumeClaim: - claimName: platform-backup-data + - name: dump + emptyDir: + sizeLimit: 5Gi + - name: metrics + hostPath: + path: /var/lib/node_exporter/textfile_collector + type: DirectoryOrCreate + - name: cache + emptyDir: {} diff --git a/apps/forgejo/forgejo-s3-backup-sealed.yaml b/apps/forgejo/forgejo-s3-backup-sealed.yaml new file mode 100644 index 0000000..319b3fc --- /dev/null +++ b/apps/forgejo/forgejo-s3-backup-sealed.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: forgejo-s3-backup + namespace: forgejo +spec: + encryptedData: + access-key: AgAu/XawJy0uhi3l8vwtd+loShP2ITnbOEZ/9eu6dkypz7l1MmFJhnEGq8Sy1lsg7BKj6epYRI40kDJIAG6KMBD5i7FvLYCvbFuPJaS3yehA8b1ECreHXvx/VYRUZL4K3WtiPO0fXmUNkxAqIcfHOcHtOZ7YcgSSyTw4DaXzngtek4xcTQlisyIVeBgc1dLPY+vHGn10BGkU5R/LnsUQqgGdp6u24Y+m7OrQGqMH8xwtZR7siEEkasu4IWG0QcY+okcMeiCi8mgAF64rdeZnIeEWqRnKHXxwEurtm20NmmCeRR8OgZfWDesgasThsnNSlRySd4EKlP3DMz8Yyluz/LlSkPt24brnP4qKuEoRpCHHOWgOizX8hkkS+Al7Rnpmxv2pt5CGlRw9s/5uPY81PAN9WLIgjs+XIylvaJBSW06nw3v+MjCUr5jfimW9riTLN32cBSurdpNMzlgb5+DJzEvaGNEG+jSjjFj2eSLci23UWySh7oFdmXeBp3kDWCrrNlJOYqJEwAAunTwvtM/1XknW2qJyyJrD7hbWN7SX4Ymb5Q5q18fMGH+IzB/Si1jdBli8pi8B2Gd/jo6RSmhXUheO3xsB2sXYc09GuSBBBBNzr9MsRL3mvUVr22pay+YPcCbwbwtdR0qmrdhEHAychYSJpNGuwGvs0AthRsNGN6hr1/IJqnnSRkGdJPr6k+T3v7XJzouYy+8Dh8r6oPW3hOhHo4HpJw== + bucket: AgAyQVCK+wE0wWxwVjx5DVnI2falTGNzsUrolv1YyoG2Bx6msYe2G00ovxCMJ6/dWws1Q/Sf6yoiFT0IByYflQFhY/ABzXRi7/gFxCWyYU/7yTo7w1Ue0+eEnBKFtkP/ZhIvtfubo6GDDtbmshpp4UtzPq7kB6qvE6oCFyf6awQD2pe/GKX8RZaDx5TCbxPxhedLkYdc7srR/8rFZBnQiXm/2ohtDOwvLA+RVOpnCW/bsVLvLYQmXyhszMv2M3zqXwM9ODYzQ4Y7lIw6T3Hsmel/1hHWJtN8/mlq+Q3Z/NbzhVeEWs8/xbK/ShC9RwZUgBWxAiW04HvwcxVYs60KaJ4oOMLtO4Iko4+iPTNonjra2wmziEpUtsotvtReRjwHdvk1bBn3q6uqzNaJFv9wLc/RVCvKSpqE5KOxmdbke9EDUs4wQHVQCAfFfwnzzpcF/Rf6ML3TUWc+U0W6UjCfVTWwROznpi0E18tA07apx+xm1y+MyuVYnSpZv++vXetnCjA+KFvqZSon/WnKQrmSO2RFR07FcMs6gQ+EOSaRXBPCnEcnF2IJUbJLdFsgpQbzYG6zo3m2KmxN7gaGrTUdhNg/N7LklnNbJML9KRa7e2Rv8t4NGoigOpZG6+soIijZOSfEJF6owN6MP3oecgUgNsnQ5ftKK/RznNaDtIeZ5KByHlQ7CsZ0MIkHBmoEkDt64xWNEoW4NrWPSxas6pOQGZMPMA== + endpoint: AgBhXqp3TdUU6GxsexRIoeHNwjvxnd2v4Any9WGs7gtyZvfdGIuWHIQ+JpsuRDrEq+BvTaX49D0QYoqnoQ6ZgmmGUI7kJxc+Ombcqo/8PAsOmzfdUaz1NcLAeR/UTGu8BDxAQcySSFax2dyK/zkY3jcsmuUCJNJLB57EZwtwx4fyz+ota+zNBE7pY+sDpWkXKHHtWrvNkD7tSLYHQuS3uzATidRixngL+iaH8rQX3ZK6KaK4H4ZdFG3/1Fw0wjLt8+yVBntBnh/bCL9/JLWQWIiHtooQnKXDdEwYQHWhrktti1KDzy/S++o4XVcRG1alzsQ0vV7uzSZAh4xCmnzximJxeJKawzFXRgFUcoEPrOI/nVedYrAU0DzkuwPdqtQDi/fGoqzZPx3PYhB9wv2PJsLg20ZlFcfHYUCad3O6OmO+VlrpYYcUe4oStkOygrmK5bv89fQ4Tov7BB14+RQIo9ogKmLzZu3xStqIVK+/Y8OUC64zyo7FpPMoYKtRW5oawu8pxo7Gtb+27BMMXQBiMEPNf38C4aj3GGJOvm4CFxVHV510/7o2hSW2ITt5X2RzNOEzULJOHD3DvO8wdWzvpBykRFnZ8JTYv77Kbx7bd9Ylx62VXm2a+Bd0K1Nc/sNJE63pglVhdl05oLYsUTrcSsNoJ6Dz3pMKeue3pYrn9KNurG0s+BFfO6I2Aa4/L8iAggXBzf9G1ALSkn7A30C9rMeyV+/1n+q8GpdPqq4ltqLOHVwIZw== + restic-password: AgB1JGFnSDe3N5ZaiT9xShfzUQiyDtGwH2wotYO5Ekk9nqAbS4xTNBXu/1Z1+cB8zVUP80hrPHHbVXjNu47AcLI+Bn8Ozn9fKIZFsnVHXqicG0Y+WHRMpQxuMijVEMl48oUNE+vw+xRhYTzibVWoxur+VEzyPCn8rLUX07tcNhY+jYptNVBYlVVXDrHcOs0s9wpvIlSWmc9RsINMu+6FaedXAs3nKY4gUpWxmNLEpLG8V7D2mNJZHk1adghTN9p8RMGcLtNylDY9W1Sdnziga/NG0WulZf+4IWV6e7wc2oBCSekTofCZuCva8HazJ1EIeqBvt2GuXoG1CjjbPNb2UwZeH21Iol1W0C5So1VFguxHZq5WL44mqD79NLlnSAMmQMSvxf7gXQVXEdtT5UlF9nYCkuQ+K+mT+8iRMy1YSCQSQ9kkD36unkrlYcLXKoPLYxHcoo7fGZ0MPIW8J+P77Hp9Z8jNPCyCaeSJYApCxybO8ujbUGdIr2FCP1z/JDP4/zZat9MkTA6G3H9kDNso7AJBHL/NfkWImvQnveLGj0CuhPpf2zwOylCqTeXrgoN8zztO2/qVDa3VRQbznap0njr0qzVrvOfTxYoiRmkGh3PXZ2mb/NqbeIvCFhj2VyVgavdMchlIbM3fCzXGEdCa2RN+ZYVtcFe/b6wR/qf7lrMaPHsTZiMfYuMJoC9F9wdbkOttT9c36Sob4x+aQYLs7/+XGCJdbw672qot6x6P7GHjY8zVKlMYLSZjKcoD48c= + secret-key: AgAHz7lfqTLG0Kldt/IZGW/o8KKazICHGbdVklXs3Gzs6Mc2KaG3nggkZDRWwY6noochg4Wc9mDDMGY3XIQnEy3PXs/rjQgwym0Dbvc54CvjWUcfmxCJyIqY1uCUxoXp/WAEV18lpWFSkVx+5U9wSccYVmkOJXNJ/MgH0vzJPaNW6xpYDg1XRC4nj72vwqyvZ/gM/LEX41NOpPjBROLGql3uBk5UfzNZG4dBun0aaPiYx5KKAbz8JAbK0umvOOOctTPSv1hIhyv9xJRRB67MW/J8T6mB2YfezsisXSSOAVNuE2JWrafHFxKq/hSNQHSrQRKjt18k6vfxmfkICbgQUZl/7AnoKi/f+76Feu+3wuMT+XPsaJN+iWb3jNzcb1T+Aiv4rGyIA7uCB6FNOc9Ngf5ngDQJw0nDNHDSnUoowgcE96kuPefU1rmn2V+JJcwoYdzYE3Ds48C0rNT+WSIit4551SiyWuj7iCRLhyEf5vhojxkuoyCN+EC1a50XwIkt223EBmy6FFPeBxql6HyJbOE313BPsQU6enPrTKNFd7iZYqJXq1k+SRuGwJhMeo1BBNhmVqxFGOpyxkSVs3VPybeXd82Zk/xF0SWKyBp3XQnLB39ZmT83Z5jwrYfkZmWkkdRvCME6NmlQtftRDDk3efGbgw7bz4Eq0d08KPCezsaDSAGNewonYy3UOiv/CNQg5hQ5G6R55y7+kEMV4nNkjGj0qF1hrZURS0mV4I/7H0OwXpDYiVetfaYI + template: + metadata: + creationTimestamp: null + name: forgejo-s3-backup + namespace: forgejo diff --git a/apps/forgejo/platform-backup-data-pvc.yaml b/apps/forgejo/platform-backup-data-pvc.yaml deleted file mode 100644 index 6a4b630..0000000 --- a/apps/forgejo/platform-backup-data-pvc.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: platform-backup-data - namespace: forgejo - labels: - app: forgejo-backup -spec: - accessModes: - - ReadWriteOnce - storageClassName: hcloud-volumes-encrypted - resources: - requests: - storage: 20Gi