stack.basicstack.de/apps/stalwart/stalwart-fresh-deployment.yaml
CTO Agent ae3f164987 fix(stalwart): Add externalTrafficPolicy: Local to LoadBalancer services
Root cause: ArgoCD continuously reverts the LoadBalancer services to use
externalTrafficPolicy: Cluster (the k8s default), causing k3s to assign
internal flannel VXLAN IPs (10.42.1.x) instead of the node public IPs.

With externalTrafficPolicy: Cluster, traffic can be routed to any node,
and k3s's service controller assigns the flannel overlay IPs. This breaks
external connectivity because those IPs are not routable from outside.

With externalTrafficPolicy: Local, traffic is only routed to pods on the
same node, and k3s assigns the node's actual public IP to the LoadBalancer.

This was the missing piece from the reliability hardening in commit b0f2acf.
Without this in git, any manual kubectl patch is reverted by ArgoCD sync.

Evidence: stalwart-smtp and stalwart-imap both showing LoadBalancer IPs:
10.42.1.1, 10.42.1.2, 10.42.1.3, 10.42.1.5 (internal flannel IPs)

Related: DEV-230, DEV-231, DEV-233, DEV-235

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-07-13 20:03:49 +00:00

383 lines
10 KiB
YAML

---
apiVersion: v1
kind: Namespace
metadata:
name: stalwart
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: stalwart-data
namespace: stalwart
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 20Gi
---
apiVersion: v1
kind: Service
metadata:
name: stalwart-smtp
namespace: stalwart
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector:
app: stalwart
ports:
- name: smtp
port: 25
targetPort: 25
protocol: TCP
- name: submission
port: 587
targetPort: 587
protocol: TCP
- name: submissions
port: 465
targetPort: 465
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: stalwart-imap
namespace: stalwart
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector:
app: stalwart
ports:
- name: imap
port: 143
targetPort: 143
protocol: TCP
- name: imaps
port: 993
targetPort: 993
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: stalwart-http
namespace: stalwart
spec:
type: ClusterIP
selector:
app: stalwart
ports:
- name: http
port: 8080
targetPort: 8080
protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: stalwart
namespace: stalwart
annotations:
# Automatically restart this StatefulSet when the TLS certificate secret is updated
# This ensures the pod reloads new certificates after cert-manager renews them
secret.reloader.stakater.com/reload: "stalwart-tls"
spec:
serviceName: stalwart-http
replicas: 1
selector:
matchLabels:
app: stalwart
template:
metadata:
labels:
app: stalwart
spec:
initContainers:
- name: fix-permissions
image: busybox:latest
command: ["sh", "-c", "chown -R 2000:2000 /var/lib/stalwart && chmod -R 755 /var/lib/stalwart"]
volumeMounts:
- name: data
mountPath: /var/lib/stalwart
containers:
- name: stalwart
image: stalwartlabs/stalwart:v0.16.11
ports:
- containerPort: 25
name: smtp
- containerPort: 587
name: submission
- containerPort: 465
name: submissions
- containerPort: 143
name: imap
- containerPort: 993
name: imaps
- containerPort: 8080
name: http
env:
- name: STALWART_RECOVERY_ADMIN
valueFrom:
secretKeyRef:
name: stalwart-admin-credentials
key: admin-password
optional: false
- name: TLS_CERTIFICATE
value: "/etc/stalwart/certs/tls.crt"
- name: TLS_PRIVATE_KEY
value: "/etc/stalwart/certs/tls.key"
volumeMounts:
- name: data
mountPath: /var/lib/stalwart
- name: bootstrap-config
mountPath: /etc/stalwart/config.json
subPath: config.json
- name: tls-certs
mountPath: /etc/stalwart/certs
readOnly: true
resources:
requests:
memory: "512Mi"
cpu: "250m"
limits:
memory: "2Gi"
cpu: "2000m"
startupProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
livenessProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3
volumes:
- name: data
persistentVolumeClaim:
claimName: stalwart-data
- name: bootstrap-config
configMap:
name: stalwart-bootstrap-config
- name: tls-certs
secret:
secretName: stalwart-tls
defaultMode: 0444
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: stalwart-web
namespace: stalwart
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
traefik.ingress.kubernetes.io/router.entrypoints: websecure
spec:
ingressClassName: traefik
tls:
- hosts:
- mail.basicstack.de
secretName: stalwart-tls
rules:
- host: mail.basicstack.de
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: stalwart-http
port:
number: 8080
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: stalwart-backup
namespace: stalwart
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: stalwart-backup
namespace: stalwart
rules:
- apiGroups: ["apps"]
resources: ["statefulsets"]
verbs: ["get", "list", "patch"]
- apiGroups: ["apps"]
resources: ["statefulsets/scale"]
verbs: ["get", "update", "patch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: stalwart-backup
namespace: stalwart
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: stalwart-backup
subjects:
- kind: ServiceAccount
name: stalwart-backup
namespace: stalwart
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: stalwart-backup
namespace: stalwart
spec:
schedule: "0 3 * * *" # 3 AM daily
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 2
activeDeadlineSeconds: 600 # 10 minute timeout
template:
metadata:
labels:
app: stalwart-backup
spec:
serviceAccountName: stalwart-backup
restartPolicy: OnFailure
containers:
- name: backup
image: alpine:3.19
command:
- /bin/sh
- -c
- |
set -e
BACKUP_START=$(date +%s)
echo "=== Starting Stalwart backup at $(date) ==="
# Install required tools
echo "Installing kubectl and restic..."
apk add --no-cache kubectl restic curl bash
# Build restic repository URL from env vars
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
echo "Using restic repository: $RESTIC_REPOSITORY"
# Function to scale StatefulSet
scale_statefulset() {
local replicas=$1
echo "Scaling stalwart StatefulSet to $replicas replicas..."
kubectl scale statefulset stalwart -n stalwart --replicas=$replicas
if [ "$replicas" -eq 0 ]; then
echo "Waiting for pod to terminate..."
kubectl wait --for=delete pod/stalwart-0 -n stalwart --timeout=120s || true
sleep 5
else
echo "Waiting for pod to be ready..."
kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=120s || echo "Warning: Pod not ready after 120s"
fi
}
# Trap to ensure we scale back up on any exit
cleanup() {
EXIT_CODE=$?
echo "Cleanup: scaling stalwart back to 1 replica..."
scale_statefulset 1 || echo "ERROR: Failed to scale back up!"
BACKUP_END=$(date +%s)
DURATION=$((BACKUP_END - BACKUP_START))
if [ $EXIT_CODE -eq 0 ]; then
echo "=== Backup completed successfully in ${DURATION}s at $(date) ==="
else
echo "=== Backup FAILED after ${DURATION}s at $(date) ==="
fi
exit $EXIT_CODE
}
trap cleanup EXIT INT TERM
# Scale down Stalwart
scale_statefulset 0
# Build restic repository URL from env vars (K8s doesn't expand $(VAR) in value fields)
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
echo "Using repository: $RESTIC_REPOSITORY"
# Initialize restic repo if needed
echo "Initializing restic repository..."
restic snapshots || restic init
# Backup the data directory
echo "Running backup..."
restic backup /var/lib/stalwart \
--tag stalwart \
--tag daily \
--host stalwart-k8s \
--verbose
# Prune old backups (keep 7 daily, 4 weekly, 6 monthly)
echo "Pruning old backups..."
restic forget \
--tag stalwart \
--keep-daily 7 \
--keep-weekly 4 \
--keep-monthly 6 \
--prune
echo "Backup operations completed"
env:
- name: RESTIC_PASSWORD
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: restic-password
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: access-key
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: secret-key
- name: S3_ENDPOINT
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: endpoint
- name: S3_BUCKET
valueFrom:
secretKeyRef:
name: stalwart-s3-backup
key: bucket
volumeMounts:
- name: data
mountPath: /var/lib/stalwart
readOnly: true
volumes:
- name: data
persistentVolumeClaim:
claimName: stalwart-data