The pod was crash-looping because Stalwart's security configuration blocks the kubelet's IP (10.244.4.1) from accessing the HTTP health endpoints. The kubelet's health checks were failing, causing the startup probe to fail after 6 attempts, leading to pod restarts. Changed all three health probes (startup, liveness, readiness) from httpGet to exec with curl localhost. This bypasses the IP blocking since the health check runs from inside the container using localhost, which is not subject to Stalwart's external IP blocking rules. This fix is non-destructive to Stalwart's configuration and state. The pod will restart once with the new probe configuration, but no data or configuration will be lost. Root cause: Stalwart logs showed "Blocked IP address (security.ip-blocked) listenerId=http, remoteIp=10.244.4.1" followed by "Shutting down Stalwart Server (server.shutdown) causedBy=SIGTERM" in a repeating pattern. Fixes: DEV-420 Co-Authored-By: Paperclip <noreply@paperclip.ing>
387 lines
10 KiB
YAML
387 lines
10 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: stalwart
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: stalwart-data
|
|
namespace: stalwart
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: hcloud-volumes-encrypted
|
|
resources:
|
|
requests:
|
|
storage: 20Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: stalwart-smtp
|
|
namespace: stalwart
|
|
spec:
|
|
type: LoadBalancer
|
|
externalTrafficPolicy: Cluster
|
|
selector:
|
|
app: stalwart
|
|
ports:
|
|
- name: smtp
|
|
port: 25
|
|
targetPort: 25
|
|
protocol: TCP
|
|
- name: submission
|
|
port: 587
|
|
targetPort: 587
|
|
protocol: TCP
|
|
# Port 465 (SMTPS) removed per DEV-359 approval - enforce secure protocols
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: stalwart-imap
|
|
namespace: stalwart
|
|
spec:
|
|
type: LoadBalancer
|
|
externalTrafficPolicy: Cluster
|
|
selector:
|
|
app: stalwart
|
|
ports:
|
|
# Port 143 (IMAP) removed per DEV-359 approval - enforce secure protocols
|
|
- name: imaps
|
|
port: 993
|
|
targetPort: 993
|
|
protocol: TCP
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: stalwart-http
|
|
namespace: stalwart
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
app: stalwart
|
|
ports:
|
|
- name: http
|
|
port: 8080
|
|
targetPort: 8080
|
|
protocol: TCP
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: stalwart
|
|
namespace: stalwart
|
|
annotations:
|
|
# Automatically restart this StatefulSet when the TLS certificate secret is updated
|
|
# This ensures the pod reloads new certificates after cert-manager renews them
|
|
secret.reloader.stakater.com/reload: "stalwart-tls"
|
|
spec:
|
|
serviceName: stalwart-http
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: stalwart
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: stalwart
|
|
spec:
|
|
initContainers:
|
|
- name: fix-permissions
|
|
image: busybox:latest
|
|
command: ["sh", "-c", "chown -R 2000:2000 /var/lib/stalwart && chmod -R 755 /var/lib/stalwart"]
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /var/lib/stalwart
|
|
containers:
|
|
- name: stalwart
|
|
image: stalwartlabs/stalwart:v0.16.11
|
|
ports:
|
|
- containerPort: 25
|
|
name: smtp
|
|
- containerPort: 587
|
|
name: submission
|
|
# Port 465 (SMTPS) removed - enforce STARTTLS on 587
|
|
# Port 143 (IMAP) removed - enforce TLS on 993
|
|
- containerPort: 993
|
|
name: imaps
|
|
- containerPort: 8080
|
|
name: http
|
|
env:
|
|
#- name: STALWART_RECOVERY_ADMIN
|
|
# valueFrom:
|
|
# secretKeyRef:
|
|
# name: stalwart-admin-credentials
|
|
# key: admin-password
|
|
# optional: false
|
|
- name: TLS_CERTIFICATE
|
|
value: "/etc/stalwart/certs/tls.crt"
|
|
- name: TLS_PRIVATE_KEY
|
|
value: "/etc/stalwart/certs/tls.key"
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /var/lib/stalwart
|
|
- name: bootstrap-config
|
|
mountPath: /etc/stalwart/config.json
|
|
subPath: config.json
|
|
- name: tls-certs
|
|
mountPath: /etc/stalwart/certs
|
|
readOnly: true
|
|
resources:
|
|
requests:
|
|
memory: "512Mi"
|
|
cpu: "250m"
|
|
limits:
|
|
memory: "2Gi"
|
|
cpu: "2000m"
|
|
startupProbe:
|
|
exec:
|
|
command:
|
|
- curl
|
|
- -f
|
|
- --max-time
|
|
- "3"
|
|
- http://localhost:8080/healthz/live
|
|
initialDelaySeconds: 60
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
failureThreshold: 6
|
|
livenessProbe:
|
|
exec:
|
|
command:
|
|
- curl
|
|
- -f
|
|
- --max-time
|
|
- "3"
|
|
- http://localhost:8080/healthz/live
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
failureThreshold: 3
|
|
readinessProbe:
|
|
exec:
|
|
command:
|
|
- curl
|
|
- -f
|
|
- --max-time
|
|
- "3"
|
|
- http://localhost:8080/healthz/ready
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 5
|
|
timeoutSeconds: 3
|
|
failureThreshold: 3
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: stalwart-data
|
|
- name: bootstrap-config
|
|
configMap:
|
|
name: stalwart-bootstrap-config
|
|
- name: tls-certs
|
|
secret:
|
|
secretName: stalwart-tls
|
|
defaultMode: 0444
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: stalwart-web
|
|
namespace: stalwart
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
|
spec:
|
|
ingressClassName: traefik
|
|
tls:
|
|
- hosts:
|
|
- mail.basicstack.de
|
|
secretName: stalwart-tls
|
|
rules:
|
|
- host: mail.basicstack.de
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: stalwart-http
|
|
port:
|
|
number: 8080
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
rules:
|
|
- apiGroups: ["apps"]
|
|
resources: ["statefulsets"]
|
|
verbs: ["get", "list", "patch"]
|
|
- apiGroups: ["apps"]
|
|
resources: ["statefulsets/scale"]
|
|
verbs: ["get", "update", "patch"]
|
|
- apiGroups: [""]
|
|
resources: ["pods"]
|
|
verbs: ["get", "list", "watch"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: stalwart-backup
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
spec:
|
|
schedule: "0 3 * * *" # 3 AM daily
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 2
|
|
activeDeadlineSeconds: 600 # 10 minute timeout
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: stalwart-backup
|
|
spec:
|
|
serviceAccountName: stalwart-backup
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: backup
|
|
image: alpine:3.19
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
BACKUP_START=$(date +%s)
|
|
echo "=== Starting Stalwart backup at $(date) ==="
|
|
|
|
# Install required tools
|
|
echo "Installing kubectl and restic..."
|
|
apk add --no-cache kubectl restic curl bash
|
|
|
|
# Build restic repository URL from env vars
|
|
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
|
|
echo "Using restic repository: $RESTIC_REPOSITORY"
|
|
|
|
# Function to scale StatefulSet
|
|
scale_statefulset() {
|
|
local replicas=$1
|
|
echo "Scaling stalwart StatefulSet to $replicas replicas..."
|
|
kubectl scale statefulset stalwart -n stalwart --replicas=$replicas
|
|
|
|
if [ "$replicas" -eq 0 ]; then
|
|
echo "Waiting for pod to terminate..."
|
|
kubectl wait --for=delete pod/stalwart-0 -n stalwart --timeout=120s || true
|
|
sleep 5
|
|
else
|
|
echo "Waiting for pod to be ready..."
|
|
kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=120s || echo "Warning: Pod not ready after 120s"
|
|
fi
|
|
}
|
|
|
|
# Trap to ensure we scale back up on any exit
|
|
cleanup() {
|
|
EXIT_CODE=$?
|
|
echo "Cleanup: scaling stalwart back to 1 replica..."
|
|
scale_statefulset 1 || echo "ERROR: Failed to scale back up!"
|
|
|
|
BACKUP_END=$(date +%s)
|
|
DURATION=$((BACKUP_END - BACKUP_START))
|
|
|
|
if [ $EXIT_CODE -eq 0 ]; then
|
|
echo "=== Backup completed successfully in ${DURATION}s at $(date) ==="
|
|
else
|
|
echo "=== Backup FAILED after ${DURATION}s at $(date) ==="
|
|
fi
|
|
|
|
exit $EXIT_CODE
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# Scale down Stalwart
|
|
scale_statefulset 0
|
|
|
|
# Build restic repository URL from env vars (K8s doesn't expand $(VAR) in value fields)
|
|
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
|
|
echo "Using repository: $RESTIC_REPOSITORY"
|
|
|
|
# Initialize restic repo if needed
|
|
echo "Initializing restic repository..."
|
|
restic snapshots || restic init
|
|
|
|
# Backup the data directory
|
|
echo "Running backup..."
|
|
restic backup /var/lib/stalwart \
|
|
--tag stalwart \
|
|
--tag daily \
|
|
--host stalwart-k8s \
|
|
--verbose
|
|
|
|
# Prune old backups (keep 7 daily, 4 weekly, 6 monthly)
|
|
echo "Pruning old backups..."
|
|
restic forget \
|
|
--tag stalwart \
|
|
--keep-daily 7 \
|
|
--keep-weekly 4 \
|
|
--keep-monthly 6 \
|
|
--prune
|
|
|
|
echo "Backup operations completed"
|
|
env:
|
|
- name: RESTIC_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: restic-password
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: access-key
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: secret-key
|
|
- name: S3_ENDPOINT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: endpoint
|
|
- name: S3_BUCKET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: bucket
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /var/lib/stalwart
|
|
readOnly: true
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: stalwart-data
|