stack.basicstack.de/apps/stalwart/stalwart-fresh-deployment.yaml
CTO Agent b06c994c9d Fix Stalwart pod crash-loop by changing health probes to exec
The pod was crash-looping because Stalwart's security configuration
blocks the kubelet's IP (10.244.4.1) from accessing the HTTP health
endpoints. The kubelet's health checks were failing, causing the
startup probe to fail after 6 attempts, leading to pod restarts.

Changed all three health probes (startup, liveness, readiness) from
httpGet to exec with curl localhost. This bypasses the IP blocking
since the health check runs from inside the container using localhost,
which is not subject to Stalwart's external IP blocking rules.

This fix is non-destructive to Stalwart's configuration and state.
The pod will restart once with the new probe configuration, but no
data or configuration will be lost.

Root cause: Stalwart logs showed "Blocked IP address (security.ip-blocked)
listenerId=http, remoteIp=10.244.4.1" followed by "Shutting down Stalwart
Server (server.shutdown) causedBy=SIGTERM" in a repeating pattern.

Fixes: DEV-420

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-08-01 08:14:41 +00:00

387 lines
10 KiB
YAML

---
apiVersion: v1
kind: Namespace
metadata:
name: stalwart
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: stalwart-data
namespace: stalwart
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 20Gi
---
apiVersion: v1
kind: Service
metadata:
name: stalwart-smtp
namespace: stalwart
spec:
type: LoadBalancer
externalTrafficPolicy: Cluster
selector:
app: stalwart
ports:
- name: smtp
port: 25
targetPort: 25
protocol: TCP
- name: submission
port: 587
targetPort: 587
protocol: TCP
# Port 465 (SMTPS) removed per DEV-359 approval - enforce secure protocols
---
apiVersion: v1
kind: Service
metadata:
name: stalwart-imap
namespace: stalwart
spec:
type: LoadBalancer
externalTrafficPolicy: Cluster
selector:
app: stalwart
ports:
# Port 143 (IMAP) removed per DEV-359 approval - enforce secure protocols
- name: imaps
port: 993
targetPort: 993
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: stalwart-http
namespace: stalwart
spec:
type: ClusterIP
selector:
app: stalwart
ports:
- name: http
port: 8080
targetPort: 8080
protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: stalwart
namespace: stalwart
annotations:
# Automatically restart this StatefulSet when the TLS certificate secret is updated
# This ensures the pod reloads new certificates after cert-manager renews them
secret.reloader.stakater.com/reload: "stalwart-tls"
spec:
serviceName: stalwart-http
replicas: 1
selector:
matchLabels:
app: stalwart
template:
metadata:
labels:
app: stalwart
spec:
initContainers:
- name: fix-permissions
image: busybox:latest
command: ["sh", "-c", "chown -R 2000:2000 /var/lib/stalwart && chmod -R 755 /var/lib/stalwart"]
volumeMounts:
- name: data
mountPath: /var/lib/stalwart
containers:
- name: stalwart
image: stalwartlabs/stalwart:v0.16.11
ports:
- containerPort: 25
name: smtp
- containerPort: 587
name: submission
# Port 465 (SMTPS) removed - enforce STARTTLS on 587
# Port 143 (IMAP) removed - enforce TLS on 993
- containerPort: 993
name: imaps
- containerPort: 8080
name: http
env:
#- name: STALWART_RECOVERY_ADMIN
# valueFrom:
# secretKeyRef:
# name: stalwart-admin-credentials
# key: admin-password
# optional: false
- name: TLS_CERTIFICATE
value: "/etc/stalwart/certs/tls.crt"
- name: TLS_PRIVATE_KEY
value: "/etc/stalwart/certs/tls.key"
volumeMounts:
- name: data
mountPath: /var/lib/stalwart
- name: bootstrap-config
mountPath: /etc/stalwart/config.json
subPath: config.json
- name: tls-certs
mountPath: /etc/stalwart/certs
readOnly: true
resources:
requests:
memory: "512Mi"
cpu: "250m"
limits:
memory: "2Gi"
cpu: "2000m"
startupProbe:
exec:
command:
- curl
- -f
- --max-time
- "3"
- http://localhost:8080/healthz/live
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
livenessProbe:
exec:
command:
- curl
- -f
- --max-time
- "3"
- http://localhost:8080/healthz/live
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
exec:
command:
- curl
- -f
- --max-time
- "3"
- http://localhost:8080/healthz/ready
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3
volumes:
- name: data
persistentVolumeClaim:
claimName: stalwart-data
- name: bootstrap-config
configMap:
name: stalwart-bootstrap-config
- name: tls-certs
secret:
secretName: stalwart-tls
defaultMode: 0444
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: stalwart-web
namespace: stalwart
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
traefik.ingress.kubernetes.io/router.entrypoints: websecure
spec:
ingressClassName: traefik
tls:
- hosts:
- mail.basicstack.de
secretName: stalwart-tls
rules:
- host: mail.basicstack.de
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: stalwart-http
port:
number: 8080
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: stalwart-backup
namespace: stalwart
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: stalwart-backup
namespace: stalwart
rules:
- apiGroups: ["apps"]
resources: ["statefulsets"]
verbs: ["get", "list", "patch"]
- apiGroups: ["apps"]
resources: ["statefulsets/scale"]
verbs: ["get", "update", "patch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: stalwart-backup
namespace: stalwart
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: stalwart-backup
subjects:
- kind: ServiceAccount
name: stalwart-backup
namespace: stalwart
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: stalwart-backup
namespace: stalwart
spec:
schedule: "0 3 * * *" # 3 AM daily
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 2
activeDeadlineSeconds: 600 # 10 minute timeout
template:
metadata:
labels:
app: stalwart-backup
spec:
serviceAccountName: stalwart-backup
restartPolicy: OnFailure
containers:
- name: backup
image: alpine:3.19
command:
- /bin/sh
- -c
- |
set -e
BACKUP_START=$(date +%s)
echo "=== Starting Stalwart backup at $(date) ==="
# Install required tools
echo "Installing kubectl and restic..."
apk add --no-cache kubectl restic curl bash
# Build restic repository URL from env vars
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
echo "Using restic repository: $RESTIC_REPOSITORY"
# Function to scale StatefulSet
scale_statefulset() {
local replicas=$1
echo "Scaling stalwart StatefulSet to $replicas replicas..."
kubectl scale statefulset stalwart -n stalwart --replicas=$replicas
if [ "$replicas" -eq 0 ]; then
echo "Waiting for pod to terminate..."
kubectl wait --for=delete pod/stalwart-0 -n stalwart --timeout=120s || true
sleep 5
else
echo "Waiting for pod to be ready..."
kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=120s || echo "Warning: Pod not ready after 120s"
fi
}
# Trap to ensure we scale back up on any exit
cleanup() {
EXIT_CODE=$?
echo "Cleanup: scaling stalwart back to 1 replica..."
scale_statefulset 1 || echo "ERROR: Failed to scale back up!"
BACKUP_END=$(date +%s)
DURATION=$((BACKUP_END - BACKUP_START))
if [ $EXIT_CODE -eq 0 ]; then
echo "=== Backup completed successfully in ${DURATION}s at $(date) ==="
else
echo "=== Backup FAILED after ${DURATION}s at $(date) ==="
fi
exit $EXIT_CODE
}
trap cleanup EXIT INT TERM
# Scale down Stalwart
scale_statefulset 0
# Build restic repository URL from env vars (K8s doesn't expand $(VAR) in value fields)
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
echo "Using repository: $RESTIC_REPOSITORY"
# Initialize restic repo if needed
echo "Initializing restic repository..."
restic snapshots || restic init
# Backup the data directory
echo "Running backup..."
restic backup /var/lib/stalwart \
--tag stalwart \
--tag daily \
--host stalwart-k8s \
--verbose
# Prune old backups (keep 7 daily, 4 weekly, 6 monthly)
echo "Pruning old backups..."
restic forget \
--tag stalwart \
--keep-daily 7 \
--keep-weekly 4 \
--keep-monthly 6 \
--prune
echo "Backup operations completed"
env:
- name: RESTIC_PASSWORD
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: restic-password
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: access-key
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: secret-key
- name: S3_ENDPOINT
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: endpoint
- name: S3_BUCKET
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: bucket
volumeMounts:
- name: data
mountPath: /var/lib/stalwart
readOnly: true
volumes:
- name: data
persistentVolumeClaim:
claimName: stalwart-data