feat(stalwart): deploy stalwart-postgres StatefulSet + backup (DEV-469)

Phase 2 of the Stalwart RocksDB -> PostgreSQL migration (DEV-467).
Purely additive - does not touch the running stalwart-0 pod, its
ConfigMap, PVC, or bootstrap-config.

- stalwart-postgres.yaml: single-replica StatefulSet with a 10Gi
  hcloud-volumes-encrypted PVC (subPath pgdata), ClusterIP Service
  on 5432, nodeAffinity csi.hetzner.cloud/location=fsn1 so the DB
  co-locates with stalwart-0 (which is fsn1-pinned by its PVC).
  Resources match forgejo-postgres (250m/512Mi req, 500m/1Gi lim).
- stalwart-postgres-credentials-sealed.yaml: sealed secret with
  POSTGRES_USER=stalwart, POSTGRES_DB=stalwart, POSTGRES_PASSWORD,
  plus a copy of the password under stalwart-db-password for
  Stalwart's [store.postgres] config in the Phase 4 cutover.
- stalwart-postgres-backup.yaml: daily CronJob at 02:30 UTC that
  streams pg_dump | gzip into the existing stalwart-s3-backup restic
  repo tagged stalwart-postgres, with independent retention keys
  (14d/8w/6m) so it doesn't collide with the RocksDB snapshots.
  activeDeadlineSeconds=1800 mirrors the DEV-464 sibling fix.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

apps/stalwart/stalwart-postgres-backup.yaml

@@ -0,0 +1,120 @@
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: stalwart-postgres-backup
+  namespace: stalwart
+  labels:
+    app: stalwart-postgres-backup
+spec:
+  schedule: "30 2 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 1800
+      template:
+        metadata:
+          labels:
+            app: stalwart-postgres-backup
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: backup
+            image: postgres:16-alpine
+            imagePullPolicy: IfNotPresent
+            command:
+            - /bin/sh
+            - -c
+            - |
+              set -euo pipefail
+              BACKUP_START=$(date +%s)
+              echo "=== Starting Stalwart Postgres backup at $(date -u) ==="
+
+              # pg_dump already in image; restic is small and only needed here.
+              apk add --no-cache restic
+
+              export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
+              echo "Repository: ${RESTIC_REPOSITORY}"
+
+              # Repo is shared with the RocksDB backup, so it already exists;
+              # init only if the RocksDB side has not created it yet.
+              restic snapshots > /dev/null 2>&1 || restic init
+
+              echo "Streaming pg_dump | gzip -> restic (tag stalwart-postgres)..."
+              PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \
+                -h stalwart-postgres.stalwart.svc.cluster.local \
+                -U "${POSTGRES_USER}" \
+                -d "${POSTGRES_DB}" \
+                --no-owner --no-acl \
+                | gzip -c \
+                | restic backup \
+                    --stdin \
+                    --stdin-filename stalwart-postgres.sql.gz \
+                    --tag stalwart-postgres \
+                    --tag daily \
+                    --host stalwart-postgres-k8s
+
+              echo "Forget/prune (tag=stalwart-postgres, independent retention)..."
+              restic forget \
+                --tag stalwart-postgres \
+                --keep-daily 14 \
+                --keep-weekly 8 \
+                --keep-monthly 6 \
+                --prune
+
+              echo "Latest stalwart-postgres snapshots:"
+              restic snapshots --tag stalwart-postgres --compact || true
+
+              BACKUP_END=$(date +%s)
+              echo "=== Postgres backup complete in $((BACKUP_END - BACKUP_START))s at $(date -u) ==="
+            env:
+            - name: POSTGRES_USER
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-postgres-credentials
+                  key: POSTGRES_USER
+            - name: POSTGRES_DB
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-postgres-credentials
+                  key: POSTGRES_DB
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-postgres-credentials
+                  key: POSTGRES_PASSWORD
+            - name: RESTIC_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-s3-backup
+                  key: restic-password
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-s3-backup
+                  key: access-key
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-s3-backup
+                  key: secret-key
+            - name: S3_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-s3-backup
+                  key: endpoint
+            - name: S3_BUCKET
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-s3-backup
+                  key: bucket
+            resources:
+              requests:
+                cpu: 100m
+                memory: 128Mi
+              limits:
+                cpu: 500m
+                memory: 512Mi

apps/stalwart/stalwart-postgres-credentials-sealed.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: stalwart-postgres-credentials
+  namespace: stalwart
+spec:
+  encryptedData:
+    POSTGRES_DB: AgBl0xkIxrdRjAlv0JP/rPXwV4sbfFcLc1TwPKZ0VC7+MXd9K1s+FMEottjxe3jUMeVk1ddI0OelZ7PlZcVeocvC4fq02djtagLcjSBn5AFqnn0vJO+sAKsnqxuyQv7IIuNhqL5a88obOeuKJPB325IsXLEhFUrCjXDz4KTjn1RW4p9sWcbn1jBbGYHSRpp7GMn6LgkJOjhbpiQeptr270dTiOjHAxI3Ky507v5qAA8m5VwjHK9pXN8QkkYqIq3mXp4WWRt9LVAfnaQ+tnxPpeZBgzf3pV2Hj4yDpFuans8xGft9e5Er2ygEwpUD+AOEJdjSntzvkx4K5Lprv3ud1PXY+f5LZb7w41Ex2fwQtGKWepQyAD1LSqtapkOVirgWOfmGxR0NcyWTpqu6VuVYPgUq+I37CCmTdVKbBmqhSIMRll0E7Cnzse1wiLIugyHtidg5whWpYiHiByEKCxwmd0LKAGZ2gTrIOctRmpLb5UYvDxr+T/mrG20p4SymFucaT7MrvMXRnewB5HSvhooXgFyTq7zNL6mlVgK5oQI1RRJ71dNtrjs2Z/RLCzffbV8K3oSM8lFuN0BBea5M8QorJoraqwa7cHPyCCCbuJuXBMo8ZDC96etvoICiBFwt4hFcxuOS43XX6CLZiVq516/TN4VLm558ai9KfIZvEkaIivJeFqdDuR2oVUV9nAQc6nqMHHnRiFmPJpUpXg==
+    POSTGRES_PASSWORD: AgASUNVrGocpv2dDca4E/cRK8IISrCS5z40ObiAzxsZql/+NdRfMzk1fUl5QF94KAgm3xe1V901SWNJvNVG/qKfHt83K8ARlWyHxFH8etc9diMQgXD4EuW21wVyWbI1lRaDSP56yqMMlOpgrgfLhN45BToiD0Dqe6s1lAdO7ozyBcNO9kEkHzM/bFmGgbDX/u8/c4bXXV3VSQueBPVDPMToEzIazCJPDQKn7PlmOzmANQ8xWa5avRpQfpsPcoatevDMnYDGyVEAlxBIAbaf8muyVD7yce6rFCYAQCUhqwyiu3YPVNBJT1PdmtNwZ2WHu42nuLcOkUoEWLtX/Ex/fnBhlvKvFBKCLBR2XPLjY/GkfoYXFN/+RbwCdvJnj77v11AxBLPrNpLx7sQmS0Gxz2ivzxexjuSVibyVO2pYqsi4idOOc8OtUEwbxN1WjncKMRz0WB5k2bOE4/DIjYIZ5KYz/+RVi6fR0CMKPuCgq76j3DHazMEKCicCdqw0iZaqyXKTIf3jzaAkdC0v6c/1by4d5z2NppDiGKwvLcwHYHlq2IKPPCjSdpu/efGI2CxijcHLtYpa5N2gg40T9oJRsf6NWu9lrvtdsoKiaWCo4hnkqmhGsNXiAqVerd7BslEkD8h5bnZYZPZ6IVZCZw86rwDOmICOj79lqtIcHDJyJra+n7wC3FlokBwR1bR3QcVJcqL2V2OBqlHQ2SCHv+uxL8d8IkHibnj23FJHlwYRBFNvw
+    POSTGRES_USER: AgA6zhC4W9S/+X+o2Gcj+0WfapqtUjZEmxpnMxZwQpYk5qaR1TrfFezKgKuGbnFWZ1HeUJM1gTlsOvg2wIlbKLHd8on8KeK4JhtR5GUd75icaW4/SSwWdODlHP6UlY1rxbc2uYxwLkbauRcjY1DKl633n/BXNYzAKFd1MbN0XsD3cMHgwDHy2MD6qW8VS80BEqj6ljNUPC/wJTR1CISH+rAiWmuuX+mf8whjqZxulqFFWaPuscC9ceHZT2bdhkC5uSBRXc5t6ZTiroRW50cT08J+u0W2Xm56ygJ6uuIxBWgTWKcAE1UNRIIXLOlK19j5yR6e216TaDTj06Spu6l7thfdPiRAx1kXMJhCK3Rhh5umWVrXxyomfA6IweUeI/kfLC3q3I5UGezZUMTou4QUsRVYwxnuyVsiDQJ0MWd0p5RVN690PbLxFebsg3YFW1R33UAvzhukO/Ct+QvWG+FfVUnaVrMh3nM+UnJau7w+9hdToG9wwvwBwCJRNRpCDo3ev1d+0AuBURXWnOSCUcifDsF6rKfCVxRMEVB5dLj5IYN3j0XRRMkvsE5UAHCFC5QFaUQD/5V6LSRvX1vV/aD88Ver57P0P4bYnNHLj02GP9OIV7HTv2jDJz3ZycfchXjRw9cIM7WmmF2HWlyCT608HD8NKf+dZlF7FJr+REYGCmNCvA4PIB6g6kwIOZzQ0KjVqFEJ0z+G7xXXjQ==
+    stalwart-db-password: AgBhrofehsfXWbdixnkOqdsNu8y9eDNIBW1g5ZgHcNAX0uq5Qa/84KguJZcNm4nzIuTs5KZTjTLvuync4cypKVKoqzthRwGWL01uAc3I8+gzjAsSLIzfWa1Dq9BZTEA0Q7oO3dBmdooAy+e5WR0MRhsF9VQWWwxM/UzQad4ihM/PBlkfvx+6Ql0pTp8m/K+E5YqXzNnla5eCra+zlzG3JWF7SOF82AnM6sLKzsHQzTv31z4I6pJTyoEtBNxbjsmmuCIh+fGPBusqTEY3vjEwVJJEaviq2B6rR9uz/sNF5bc6XtX0nAeyeQR4WURrTmGBDgwI85Z0/vDBsD9D+NTuoTitw4kTO8JcE2H4FUY8afurxpODvx6mmLaYnV8c7hLz4l33k9zCpjb8gHfNqQR/sKRu+iSeej12E0MgxUXnSL9nhLBltwA/6KpGHz4TxTG58/AopF9yDQIKOsZF7Lj4mJC3AQ7qlpwF8j/5wiMEr8OqyhAiOm6wmzIArZzb91SVlKitLmQEPBdOzx2kboZmeSJi/P52R6qFYm/bamGPTI5m6x+MnEXgur37eubnSlY/v65sfhdJb+Gm4B1QyS06CSG4VCgom1T5Fpc1zTWxgYN85OaHQVFR1T5jhOCgZPS01/rN3R1D/wryve8n2aGLZhf+ad8vKyXbcg25lIchzwWOXYk5nM0NOshw3Dk2/IcFfwja3QGak0ps7syZUHPY+wCPL9mrd7F4OqaLrZmbZHrv
+  template:
+    metadata:
+      creationTimestamp: null
+      name: stalwart-postgres-credentials
+      namespace: stalwart
+    type: Opaque

apps/stalwart/stalwart-postgres.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart-postgres
+  namespace: stalwart
+  labels:
+    app: stalwart-postgres
+spec:
+  type: ClusterIP
+  selector:
+    app: stalwart-postgres
+  ports:
+  - name: postgres
+    port: 5432
+    targetPort: 5432
+    protocol: TCP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: stalwart-postgres
+  namespace: stalwart
+  labels:
+    app: stalwart-postgres
+spec:
+  serviceName: stalwart-postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      app: stalwart-postgres
+  template:
+    metadata:
+      labels:
+        app: stalwart-postgres
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      containers:
+      - name: postgres
+        image: postgres:16-alpine
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: postgres
+          containerPort: 5432
+          protocol: TCP
+        envFrom:
+        - secretRef:
+            name: stalwart-postgres-credentials
+        env:
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        resources:
+          requests:
+            cpu: 250m
+            memory: 512Mi
+          limits:
+            cpu: 500m
+            memory: 1Gi
+        readinessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 6
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 5
+          failureThreshold: 6
+        volumeMounts:
+        - name: postgres-data
+          mountPath: /var/lib/postgresql/data
+          subPath: pgdata
+  volumeClaimTemplates:
+  - metadata:
+      name: postgres-data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: hcloud-volumes-encrypted
+      resources:
+        requests:
+          storage: 10Gi