From d8cd3638fa5fe694a1a364e011538653bbfcd457 Mon Sep 17 00:00:00 2001
From: CTO Agent <cto@basicstack.de>
Date: Sat, 11 Jul 2026 11:21:30 +0000
Subject: [PATCH] Fix Stalwart backup CronJob to handle ReadWriteOnce PVC

Implement scale-down/backup/scale-up pattern to work around PVC access mode limitation.

Changes:
- Add RBAC (ServiceAccount, Role, RoleBinding) with statefulsets/scale and pods permissions
- Switch to alpine:3.19 base image with kubectl and restic
- Scale down StatefulSet to 0 replicas before backup
- Run restic backup while pod is stopped
- Scale back up to 1 replica with error handling
- Add cleanup trap to ensure scale-up even on failure
- Set 10-minute timeout and backoff limit

Tested successfully: backup completes in ~32 seconds with minimal downtime.
Resolves DEV-236.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 apps/stalwart/stalwart-fresh-deployment.yaml | 99 +++++++++++++++++++-
 1 file changed, 95 insertions(+), 4 deletions(-)

diff --git a/apps/stalwart/stalwart-fresh-deployment.yaml b/apps/stalwart/stalwart-fresh-deployment.yaml
index 7f817e4..de86340 100644
--- a/apps/stalwart/stalwart-fresh-deployment.yaml
+++ b/apps/stalwart/stalwart-fresh-deployment.yaml
@@ -206,6 +206,42 @@ spec:
             port:
               number: 8080
 ---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stalwart-backup
+  namespace: stalwart
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: stalwart-backup
+  namespace: stalwart
+rules:
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get", "list", "patch"]
+- apiGroups: ["apps"]
+  resources: ["statefulsets/scale"]
+  verbs: ["get", "update", "patch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: stalwart-backup
+  namespace: stalwart
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: stalwart-backup
+subjects:
+- kind: ServiceAccount
+  name: stalwart-backup
+  namespace: stalwart
+---
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -217,43 +253,98 @@ spec:
   failedJobsHistoryLimit: 3
   jobTemplate:
     spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 600  # 10 minute timeout
       template:
         metadata:
           labels:
             app: stalwart-backup
         spec:
+          serviceAccountName: stalwart-backup
           restartPolicy: OnFailure
           containers:
           - name: backup
-            image: restic/restic:latest
+            image: alpine:3.19
             command:
             - /bin/sh
             - -c
             - |
               set -e
-              echo "Starting Stalwart backup at $(date)"
+              BACKUP_START=$(date +%s)
+              echo "=== Starting Stalwart backup at $(date) ==="
+
+              # Install required tools
+              echo "Installing kubectl and restic..."
+              apk add --no-cache kubectl restic curl bash
+
+              # Build restic repository URL from env vars
+              export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
+              echo "Using restic repository: $RESTIC_REPOSITORY"
+
+              # Function to scale StatefulSet
+              scale_statefulset() {
+                local replicas=$1
+                echo "Scaling stalwart StatefulSet to $replicas replicas..."
+                kubectl scale statefulset stalwart -n stalwart --replicas=$replicas
+
+                if [ "$replicas" -eq 0 ]; then
+                  echo "Waiting for pod to terminate..."
+                  kubectl wait --for=delete pod/stalwart-0 -n stalwart --timeout=120s || true
+                  sleep 5
+                else
+                  echo "Waiting for pod to be ready..."
+                  kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=120s || echo "Warning: Pod not ready after 120s"
+                fi
+              }
+
+              # Trap to ensure we scale back up on any exit
+              cleanup() {
+                EXIT_CODE=$?
+                echo "Cleanup: scaling stalwart back to 1 replica..."
+                scale_statefulset 1 || echo "ERROR: Failed to scale back up!"
+
+                BACKUP_END=$(date +%s)
+                DURATION=$((BACKUP_END - BACKUP_START))
+
+                if [ $EXIT_CODE -eq 0 ]; then
+                  echo "=== Backup completed successfully in ${DURATION}s at $(date) ==="
+                else
+                  echo "=== Backup FAILED after ${DURATION}s at $(date) ==="
+                fi
+
+                exit $EXIT_CODE
+              }
+              trap cleanup EXIT INT TERM
+
+              # Scale down Stalwart
+              scale_statefulset 0
 
               # Build restic repository URL from env vars (K8s doesn't expand $(VAR) in value fields)
               export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
               echo "Using repository: $RESTIC_REPOSITORY"
 
               # Initialize restic repo if needed
+              echo "Initializing restic repository..."
               restic snapshots || restic init
 
               # Backup the data directory
+              echo "Running backup..."
               restic backup /var/lib/stalwart \
                 --tag stalwart \
                 --tag daily \
-                --host stalwart-k8s
+                --host stalwart-k8s \
+                --verbose
 
               # Prune old backups (keep 7 daily, 4 weekly, 6 monthly)
+              echo "Pruning old backups..."
               restic forget \
+                --tag stalwart \
                 --keep-daily 7 \
                 --keep-weekly 4 \
                 --keep-monthly 6 \
                 --prune
 
-              echo "Backup completed successfully at $(date)"
+              echo "Backup operations completed"
             env:
             - name: RESTIC_PASSWORD
               valueFrom: