2026-07-04 10:17:13 +00:00
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Namespace
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart
|
|
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: PersistentVolumeClaim
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-data
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
accessModes:
|
|
|
|
|
- ReadWriteOnce
|
|
|
|
|
storageClassName: hcloud-volumes-encrypted
|
|
|
|
|
resources:
|
|
|
|
|
requests:
|
|
|
|
|
storage: 20Gi
|
|
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Service
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-smtp
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
type: LoadBalancer
|
fix(stalwart): Add externalTrafficPolicy: Local to LoadBalancer services
Root cause: ArgoCD continuously reverts the LoadBalancer services to use
externalTrafficPolicy: Cluster (the k8s default), causing k3s to assign
internal flannel VXLAN IPs (10.42.1.x) instead of the node public IPs.
With externalTrafficPolicy: Cluster, traffic can be routed to any node,
and k3s's service controller assigns the flannel overlay IPs. This breaks
external connectivity because those IPs are not routable from outside.
With externalTrafficPolicy: Local, traffic is only routed to pods on the
same node, and k3s assigns the node's actual public IP to the LoadBalancer.
This was the missing piece from the reliability hardening in commit b0f2acf.
Without this in git, any manual kubectl patch is reverted by ArgoCD sync.
Evidence: stalwart-smtp and stalwart-imap both showing LoadBalancer IPs:
10.42.1.1, 10.42.1.2, 10.42.1.3, 10.42.1.5 (internal flannel IPs)
Related: DEV-230, DEV-231, DEV-233, DEV-235
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-07-13 20:03:29 +00:00
|
|
|
externalTrafficPolicy: Local
|
2026-07-04 10:17:13 +00:00
|
|
|
selector:
|
|
|
|
|
app: stalwart
|
|
|
|
|
ports:
|
|
|
|
|
- name: smtp
|
|
|
|
|
port: 25
|
|
|
|
|
targetPort: 25
|
|
|
|
|
protocol: TCP
|
|
|
|
|
- name: submission
|
|
|
|
|
port: 587
|
|
|
|
|
targetPort: 587
|
|
|
|
|
protocol: TCP
|
2026-07-22 18:04:05 +00:00
|
|
|
# Port 465 (SMTPS) removed per DEV-359 approval - enforce secure protocols
|
2026-07-04 10:17:13 +00:00
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Service
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-imap
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
type: LoadBalancer
|
fix(stalwart): Add externalTrafficPolicy: Local to LoadBalancer services
Root cause: ArgoCD continuously reverts the LoadBalancer services to use
externalTrafficPolicy: Cluster (the k8s default), causing k3s to assign
internal flannel VXLAN IPs (10.42.1.x) instead of the node public IPs.
With externalTrafficPolicy: Cluster, traffic can be routed to any node,
and k3s's service controller assigns the flannel overlay IPs. This breaks
external connectivity because those IPs are not routable from outside.
With externalTrafficPolicy: Local, traffic is only routed to pods on the
same node, and k3s assigns the node's actual public IP to the LoadBalancer.
This was the missing piece from the reliability hardening in commit b0f2acf.
Without this in git, any manual kubectl patch is reverted by ArgoCD sync.
Evidence: stalwart-smtp and stalwart-imap both showing LoadBalancer IPs:
10.42.1.1, 10.42.1.2, 10.42.1.3, 10.42.1.5 (internal flannel IPs)
Related: DEV-230, DEV-231, DEV-233, DEV-235
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-07-13 20:03:29 +00:00
|
|
|
externalTrafficPolicy: Local
|
2026-07-04 10:17:13 +00:00
|
|
|
selector:
|
|
|
|
|
app: stalwart
|
|
|
|
|
ports:
|
2026-07-22 18:04:05 +00:00
|
|
|
# Port 143 (IMAP) removed per DEV-359 approval - enforce secure protocols
|
2026-07-04 10:17:13 +00:00
|
|
|
- name: imaps
|
|
|
|
|
port: 993
|
|
|
|
|
targetPort: 993
|
|
|
|
|
protocol: TCP
|
|
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Service
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-http
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
type: ClusterIP
|
|
|
|
|
selector:
|
|
|
|
|
app: stalwart
|
|
|
|
|
ports:
|
|
|
|
|
- name: http
|
|
|
|
|
port: 8080
|
|
|
|
|
targetPort: 8080
|
|
|
|
|
protocol: TCP
|
|
|
|
|
---
|
|
|
|
|
apiVersion: apps/v1
|
|
|
|
|
kind: StatefulSet
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart
|
|
|
|
|
namespace: stalwart
|
2026-07-04 16:48:48 +00:00
|
|
|
annotations:
|
|
|
|
|
# Automatically restart this StatefulSet when the TLS certificate secret is updated
|
|
|
|
|
# This ensures the pod reloads new certificates after cert-manager renews them
|
|
|
|
|
secret.reloader.stakater.com/reload: "stalwart-tls"
|
2026-07-04 10:17:13 +00:00
|
|
|
spec:
|
|
|
|
|
serviceName: stalwart-http
|
|
|
|
|
replicas: 1
|
|
|
|
|
selector:
|
|
|
|
|
matchLabels:
|
|
|
|
|
app: stalwart
|
|
|
|
|
template:
|
|
|
|
|
metadata:
|
|
|
|
|
labels:
|
|
|
|
|
app: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
initContainers:
|
|
|
|
|
- name: fix-permissions
|
|
|
|
|
image: busybox:latest
|
2026-07-04 10:23:15 +00:00
|
|
|
command: ["sh", "-c", "chown -R 2000:2000 /var/lib/stalwart && chmod -R 755 /var/lib/stalwart"]
|
2026-07-04 10:17:13 +00:00
|
|
|
volumeMounts:
|
|
|
|
|
- name: data
|
|
|
|
|
mountPath: /var/lib/stalwart
|
|
|
|
|
containers:
|
|
|
|
|
- name: stalwart
|
|
|
|
|
image: stalwartlabs/stalwart:v0.16.11
|
|
|
|
|
ports:
|
|
|
|
|
- containerPort: 25
|
|
|
|
|
name: smtp
|
|
|
|
|
- containerPort: 587
|
|
|
|
|
name: submission
|
2026-07-22 18:04:05 +00:00
|
|
|
# Port 465 (SMTPS) removed - enforce STARTTLS on 587
|
|
|
|
|
# Port 143 (IMAP) removed - enforce TLS on 993
|
2026-07-04 10:17:13 +00:00
|
|
|
- containerPort: 993
|
|
|
|
|
name: imaps
|
|
|
|
|
- containerPort: 8080
|
|
|
|
|
name: http
|
|
|
|
|
env:
|
|
|
|
|
- name: STALWART_RECOVERY_ADMIN
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-admin-credentials
|
|
|
|
|
key: admin-password
|
|
|
|
|
optional: false
|
2026-07-04 13:22:36 +00:00
|
|
|
- name: TLS_CERTIFICATE
|
|
|
|
|
value: "/etc/stalwart/certs/tls.crt"
|
|
|
|
|
- name: TLS_PRIVATE_KEY
|
|
|
|
|
value: "/etc/stalwart/certs/tls.key"
|
2026-07-04 10:17:13 +00:00
|
|
|
volumeMounts:
|
|
|
|
|
- name: data
|
|
|
|
|
mountPath: /var/lib/stalwart
|
2026-07-04 10:34:28 +00:00
|
|
|
- name: bootstrap-config
|
|
|
|
|
mountPath: /etc/stalwart/config.json
|
|
|
|
|
subPath: config.json
|
2026-07-04 12:13:18 +00:00
|
|
|
- name: tls-certs
|
|
|
|
|
mountPath: /etc/stalwart/certs
|
|
|
|
|
readOnly: true
|
2026-07-04 10:17:13 +00:00
|
|
|
resources:
|
|
|
|
|
requests:
|
|
|
|
|
memory: "512Mi"
|
|
|
|
|
cpu: "250m"
|
|
|
|
|
limits:
|
|
|
|
|
memory: "2Gi"
|
|
|
|
|
cpu: "2000m"
|
Stalwart reliability hardening: fix k3s service networking issues
Root cause: k3s service ClusterIP routing instability causing intermittent
failures despite healthy pods. This is the 5th incident - prior fixes treated
symptoms, not the systemic networking fragility.
Changes:
- Add startup probe (60s delay, prevents premature service registration)
- Fix backup job env var substitution (use shell ${VAR}, not K8s $(VAR))
- Add comprehensive monitoring (ServiceMonitor, PrometheusRule, blackbox probes)
- Add alerting for service failures, high latency, pod restarts, backup failures
Evidence:
- Pod healthy (4d15h uptime, 0 restarts) but service ClusterIP routing broken
- Direct pod IP worked, service ClusterIP failed with "Connection reset by peer"
- Iptables rules correct, endpoints correct, but packets not flowing
- Required pod restart + Traefik restart to restore service
Monitoring now tests full service path from outside cluster, not just pod health.
Will alert immediately on failures instead of relying on reactive discovery.
Related: DEV-213, DEV-221, DEV-223, DEV-224, DEV-230, DEV-231
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-07-11 11:03:51 +00:00
|
|
|
startupProbe:
|
|
|
|
|
httpGet:
|
|
|
|
|
path: /
|
|
|
|
|
port: 8080
|
|
|
|
|
initialDelaySeconds: 60
|
|
|
|
|
periodSeconds: 10
|
|
|
|
|
timeoutSeconds: 5
|
|
|
|
|
failureThreshold: 6
|
2026-07-04 10:17:13 +00:00
|
|
|
livenessProbe:
|
|
|
|
|
httpGet:
|
|
|
|
|
path: /
|
|
|
|
|
port: 8080
|
|
|
|
|
initialDelaySeconds: 30
|
|
|
|
|
periodSeconds: 10
|
|
|
|
|
timeoutSeconds: 5
|
|
|
|
|
failureThreshold: 3
|
|
|
|
|
readinessProbe:
|
|
|
|
|
httpGet:
|
|
|
|
|
path: /
|
|
|
|
|
port: 8080
|
|
|
|
|
initialDelaySeconds: 10
|
|
|
|
|
periodSeconds: 5
|
|
|
|
|
timeoutSeconds: 3
|
|
|
|
|
failureThreshold: 3
|
|
|
|
|
volumes:
|
|
|
|
|
- name: data
|
|
|
|
|
persistentVolumeClaim:
|
|
|
|
|
claimName: stalwart-data
|
2026-07-04 10:34:28 +00:00
|
|
|
- name: bootstrap-config
|
2026-07-04 10:17:13 +00:00
|
|
|
configMap:
|
2026-07-04 10:34:28 +00:00
|
|
|
name: stalwart-bootstrap-config
|
2026-07-04 12:13:18 +00:00
|
|
|
- name: tls-certs
|
|
|
|
|
secret:
|
|
|
|
|
secretName: stalwart-tls
|
|
|
|
|
defaultMode: 0444
|
2026-07-04 10:17:13 +00:00
|
|
|
---
|
|
|
|
|
apiVersion: networking.k8s.io/v1
|
|
|
|
|
kind: Ingress
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-web
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
annotations:
|
|
|
|
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
|
|
|
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
|
|
|
|
spec:
|
|
|
|
|
ingressClassName: traefik
|
|
|
|
|
tls:
|
|
|
|
|
- hosts:
|
|
|
|
|
- mail.basicstack.de
|
|
|
|
|
secretName: stalwart-tls
|
|
|
|
|
rules:
|
|
|
|
|
- host: mail.basicstack.de
|
|
|
|
|
http:
|
|
|
|
|
paths:
|
|
|
|
|
- path: /
|
|
|
|
|
pathType: Prefix
|
|
|
|
|
backend:
|
|
|
|
|
service:
|
|
|
|
|
name: stalwart-http
|
|
|
|
|
port:
|
|
|
|
|
number: 8080
|
|
|
|
|
---
|
2026-07-11 11:21:30 +00:00
|
|
|
apiVersion: v1
|
|
|
|
|
kind: ServiceAccount
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
---
|
|
|
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
|
|
|
kind: Role
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
rules:
|
|
|
|
|
- apiGroups: ["apps"]
|
|
|
|
|
resources: ["statefulsets"]
|
|
|
|
|
verbs: ["get", "list", "patch"]
|
|
|
|
|
- apiGroups: ["apps"]
|
|
|
|
|
resources: ["statefulsets/scale"]
|
|
|
|
|
verbs: ["get", "update", "patch"]
|
|
|
|
|
- apiGroups: [""]
|
|
|
|
|
resources: ["pods"]
|
|
|
|
|
verbs: ["get", "list", "watch"]
|
|
|
|
|
---
|
|
|
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
|
|
|
kind: RoleBinding
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
roleRef:
|
|
|
|
|
apiGroup: rbac.authorization.k8s.io
|
|
|
|
|
kind: Role
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
subjects:
|
|
|
|
|
- kind: ServiceAccount
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
---
|
2026-07-04 10:17:13 +00:00
|
|
|
apiVersion: batch/v1
|
|
|
|
|
kind: CronJob
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
schedule: "0 3 * * *" # 3 AM daily
|
|
|
|
|
successfulJobsHistoryLimit: 3
|
|
|
|
|
failedJobsHistoryLimit: 3
|
|
|
|
|
jobTemplate:
|
|
|
|
|
spec:
|
2026-07-11 11:21:30 +00:00
|
|
|
backoffLimit: 2
|
|
|
|
|
activeDeadlineSeconds: 600 # 10 minute timeout
|
2026-07-04 10:17:13 +00:00
|
|
|
template:
|
|
|
|
|
metadata:
|
|
|
|
|
labels:
|
|
|
|
|
app: stalwart-backup
|
|
|
|
|
spec:
|
2026-07-11 11:21:30 +00:00
|
|
|
serviceAccountName: stalwart-backup
|
2026-07-04 10:17:13 +00:00
|
|
|
restartPolicy: OnFailure
|
|
|
|
|
containers:
|
|
|
|
|
- name: backup
|
2026-07-11 11:21:30 +00:00
|
|
|
image: alpine:3.19
|
2026-07-04 10:17:13 +00:00
|
|
|
command:
|
|
|
|
|
- /bin/sh
|
|
|
|
|
- -c
|
|
|
|
|
- |
|
|
|
|
|
set -e
|
2026-07-11 11:21:30 +00:00
|
|
|
BACKUP_START=$(date +%s)
|
|
|
|
|
echo "=== Starting Stalwart backup at $(date) ==="
|
|
|
|
|
|
|
|
|
|
# Install required tools
|
|
|
|
|
echo "Installing kubectl and restic..."
|
|
|
|
|
apk add --no-cache kubectl restic curl bash
|
|
|
|
|
|
|
|
|
|
# Build restic repository URL from env vars
|
|
|
|
|
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
|
|
|
|
|
echo "Using restic repository: $RESTIC_REPOSITORY"
|
|
|
|
|
|
|
|
|
|
# Function to scale StatefulSet
|
|
|
|
|
scale_statefulset() {
|
|
|
|
|
local replicas=$1
|
|
|
|
|
echo "Scaling stalwart StatefulSet to $replicas replicas..."
|
|
|
|
|
kubectl scale statefulset stalwart -n stalwart --replicas=$replicas
|
|
|
|
|
|
|
|
|
|
if [ "$replicas" -eq 0 ]; then
|
|
|
|
|
echo "Waiting for pod to terminate..."
|
|
|
|
|
kubectl wait --for=delete pod/stalwart-0 -n stalwart --timeout=120s || true
|
|
|
|
|
sleep 5
|
|
|
|
|
else
|
|
|
|
|
echo "Waiting for pod to be ready..."
|
|
|
|
|
kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=120s || echo "Warning: Pod not ready after 120s"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Trap to ensure we scale back up on any exit
|
|
|
|
|
cleanup() {
|
|
|
|
|
EXIT_CODE=$?
|
|
|
|
|
echo "Cleanup: scaling stalwart back to 1 replica..."
|
|
|
|
|
scale_statefulset 1 || echo "ERROR: Failed to scale back up!"
|
|
|
|
|
|
|
|
|
|
BACKUP_END=$(date +%s)
|
|
|
|
|
DURATION=$((BACKUP_END - BACKUP_START))
|
|
|
|
|
|
|
|
|
|
if [ $EXIT_CODE -eq 0 ]; then
|
|
|
|
|
echo "=== Backup completed successfully in ${DURATION}s at $(date) ==="
|
|
|
|
|
else
|
|
|
|
|
echo "=== Backup FAILED after ${DURATION}s at $(date) ==="
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
exit $EXIT_CODE
|
|
|
|
|
}
|
|
|
|
|
trap cleanup EXIT INT TERM
|
|
|
|
|
|
|
|
|
|
# Scale down Stalwart
|
|
|
|
|
scale_statefulset 0
|
2026-07-04 10:17:13 +00:00
|
|
|
|
Stalwart reliability hardening: fix k3s service networking issues
Root cause: k3s service ClusterIP routing instability causing intermittent
failures despite healthy pods. This is the 5th incident - prior fixes treated
symptoms, not the systemic networking fragility.
Changes:
- Add startup probe (60s delay, prevents premature service registration)
- Fix backup job env var substitution (use shell ${VAR}, not K8s $(VAR))
- Add comprehensive monitoring (ServiceMonitor, PrometheusRule, blackbox probes)
- Add alerting for service failures, high latency, pod restarts, backup failures
Evidence:
- Pod healthy (4d15h uptime, 0 restarts) but service ClusterIP routing broken
- Direct pod IP worked, service ClusterIP failed with "Connection reset by peer"
- Iptables rules correct, endpoints correct, but packets not flowing
- Required pod restart + Traefik restart to restore service
Monitoring now tests full service path from outside cluster, not just pod health.
Will alert immediately on failures instead of relying on reactive discovery.
Related: DEV-213, DEV-221, DEV-223, DEV-224, DEV-230, DEV-231
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-07-11 11:03:51 +00:00
|
|
|
# Build restic repository URL from env vars (K8s doesn't expand $(VAR) in value fields)
|
|
|
|
|
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
|
|
|
|
|
echo "Using repository: $RESTIC_REPOSITORY"
|
|
|
|
|
|
2026-07-04 10:17:13 +00:00
|
|
|
# Initialize restic repo if needed
|
2026-07-11 11:21:30 +00:00
|
|
|
echo "Initializing restic repository..."
|
2026-07-04 10:17:13 +00:00
|
|
|
restic snapshots || restic init
|
|
|
|
|
|
|
|
|
|
# Backup the data directory
|
2026-07-11 11:21:30 +00:00
|
|
|
echo "Running backup..."
|
2026-07-04 10:17:13 +00:00
|
|
|
restic backup /var/lib/stalwart \
|
|
|
|
|
--tag stalwart \
|
|
|
|
|
--tag daily \
|
2026-07-11 11:21:30 +00:00
|
|
|
--host stalwart-k8s \
|
|
|
|
|
--verbose
|
2026-07-04 10:17:13 +00:00
|
|
|
|
|
|
|
|
# Prune old backups (keep 7 daily, 4 weekly, 6 monthly)
|
2026-07-11 11:21:30 +00:00
|
|
|
echo "Pruning old backups..."
|
2026-07-04 10:17:13 +00:00
|
|
|
restic forget \
|
2026-07-11 11:21:30 +00:00
|
|
|
--tag stalwart \
|
2026-07-04 10:17:13 +00:00
|
|
|
--keep-daily 7 \
|
|
|
|
|
--keep-weekly 4 \
|
|
|
|
|
--keep-monthly 6 \
|
|
|
|
|
--prune
|
|
|
|
|
|
2026-07-11 11:21:30 +00:00
|
|
|
echo "Backup operations completed"
|
2026-07-04 10:17:13 +00:00
|
|
|
env:
|
|
|
|
|
- name: RESTIC_PASSWORD
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: restic-password
|
|
|
|
|
- name: AWS_ACCESS_KEY_ID
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: access-key
|
|
|
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: secret-key
|
|
|
|
|
- name: S3_ENDPOINT
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: endpoint
|
|
|
|
|
- name: S3_BUCKET
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: bucket
|
|
|
|
|
volumeMounts:
|
|
|
|
|
- name: data
|
|
|
|
|
mountPath: /var/lib/stalwart
|
|
|
|
|
readOnly: true
|
|
|
|
|
volumes:
|
|
|
|
|
- name: data
|
|
|
|
|
persistentVolumeClaim:
|
|
|
|
|
claimName: stalwart-data
|