2026-07-04 10:17:13 +00:00
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Namespace
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart
|
|
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: PersistentVolumeClaim
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-data
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
accessModes:
|
|
|
|
|
- ReadWriteOnce
|
|
|
|
|
storageClassName: hcloud-volumes-encrypted
|
|
|
|
|
resources:
|
|
|
|
|
requests:
|
|
|
|
|
storage: 20Gi
|
|
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Service
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-smtp
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
type: LoadBalancer
|
|
|
|
|
selector:
|
|
|
|
|
app: stalwart
|
|
|
|
|
ports:
|
|
|
|
|
- name: smtp
|
|
|
|
|
port: 25
|
|
|
|
|
targetPort: 25
|
|
|
|
|
protocol: TCP
|
|
|
|
|
- name: submission
|
|
|
|
|
port: 587
|
|
|
|
|
targetPort: 587
|
|
|
|
|
protocol: TCP
|
|
|
|
|
- name: submissions
|
|
|
|
|
port: 465
|
|
|
|
|
targetPort: 465
|
|
|
|
|
protocol: TCP
|
|
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Service
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-imap
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
type: LoadBalancer
|
|
|
|
|
selector:
|
|
|
|
|
app: stalwart
|
|
|
|
|
ports:
|
|
|
|
|
- name: imap
|
|
|
|
|
port: 143
|
|
|
|
|
targetPort: 143
|
|
|
|
|
protocol: TCP
|
|
|
|
|
- name: imaps
|
|
|
|
|
port: 993
|
|
|
|
|
targetPort: 993
|
|
|
|
|
protocol: TCP
|
|
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Service
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-http
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
type: ClusterIP
|
|
|
|
|
selector:
|
|
|
|
|
app: stalwart
|
|
|
|
|
ports:
|
|
|
|
|
- name: http
|
|
|
|
|
port: 8080
|
|
|
|
|
targetPort: 8080
|
|
|
|
|
protocol: TCP
|
|
|
|
|
---
|
|
|
|
|
apiVersion: apps/v1
|
|
|
|
|
kind: StatefulSet
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart
|
|
|
|
|
namespace: stalwart
|
2026-07-04 16:48:48 +00:00
|
|
|
annotations:
|
|
|
|
|
# Automatically restart this StatefulSet when the TLS certificate secret is updated
|
|
|
|
|
# This ensures the pod reloads new certificates after cert-manager renews them
|
|
|
|
|
secret.reloader.stakater.com/reload: "stalwart-tls"
|
2026-07-04 10:17:13 +00:00
|
|
|
spec:
|
|
|
|
|
serviceName: stalwart-http
|
|
|
|
|
replicas: 1
|
|
|
|
|
selector:
|
|
|
|
|
matchLabels:
|
|
|
|
|
app: stalwart
|
|
|
|
|
template:
|
|
|
|
|
metadata:
|
|
|
|
|
labels:
|
|
|
|
|
app: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
initContainers:
|
|
|
|
|
- name: fix-permissions
|
|
|
|
|
image: busybox:latest
|
2026-07-04 10:23:15 +00:00
|
|
|
command: ["sh", "-c", "chown -R 2000:2000 /var/lib/stalwart && chmod -R 755 /var/lib/stalwart"]
|
2026-07-04 10:17:13 +00:00
|
|
|
volumeMounts:
|
|
|
|
|
- name: data
|
|
|
|
|
mountPath: /var/lib/stalwart
|
|
|
|
|
containers:
|
|
|
|
|
- name: stalwart
|
|
|
|
|
image: stalwartlabs/stalwart:v0.16.11
|
|
|
|
|
ports:
|
|
|
|
|
- containerPort: 25
|
|
|
|
|
name: smtp
|
|
|
|
|
- containerPort: 587
|
|
|
|
|
name: submission
|
|
|
|
|
- containerPort: 465
|
|
|
|
|
name: submissions
|
|
|
|
|
- containerPort: 143
|
|
|
|
|
name: imap
|
|
|
|
|
- containerPort: 993
|
|
|
|
|
name: imaps
|
|
|
|
|
- containerPort: 8080
|
|
|
|
|
name: http
|
|
|
|
|
env:
|
|
|
|
|
- name: STALWART_RECOVERY_ADMIN
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-admin-credentials
|
|
|
|
|
key: admin-password
|
|
|
|
|
optional: false
|
2026-07-04 13:22:36 +00:00
|
|
|
- name: TLS_CERTIFICATE
|
|
|
|
|
value: "/etc/stalwart/certs/tls.crt"
|
|
|
|
|
- name: TLS_PRIVATE_KEY
|
|
|
|
|
value: "/etc/stalwart/certs/tls.key"
|
2026-07-04 10:17:13 +00:00
|
|
|
volumeMounts:
|
|
|
|
|
- name: data
|
|
|
|
|
mountPath: /var/lib/stalwart
|
2026-07-04 10:34:28 +00:00
|
|
|
- name: bootstrap-config
|
|
|
|
|
mountPath: /etc/stalwart/config.json
|
|
|
|
|
subPath: config.json
|
2026-07-04 12:13:18 +00:00
|
|
|
- name: tls-certs
|
|
|
|
|
mountPath: /etc/stalwart/certs
|
|
|
|
|
readOnly: true
|
2026-07-04 10:17:13 +00:00
|
|
|
resources:
|
|
|
|
|
requests:
|
|
|
|
|
memory: "512Mi"
|
|
|
|
|
cpu: "250m"
|
|
|
|
|
limits:
|
|
|
|
|
memory: "2Gi"
|
|
|
|
|
cpu: "2000m"
|
Stalwart reliability hardening: fix k3s service networking issues
Root cause: k3s service ClusterIP routing instability causing intermittent
failures despite healthy pods. This is the 5th incident - prior fixes treated
symptoms, not the systemic networking fragility.
Changes:
- Add startup probe (60s delay, prevents premature service registration)
- Fix backup job env var substitution (use shell ${VAR}, not K8s $(VAR))
- Add comprehensive monitoring (ServiceMonitor, PrometheusRule, blackbox probes)
- Add alerting for service failures, high latency, pod restarts, backup failures
Evidence:
- Pod healthy (4d15h uptime, 0 restarts) but service ClusterIP routing broken
- Direct pod IP worked, service ClusterIP failed with "Connection reset by peer"
- Iptables rules correct, endpoints correct, but packets not flowing
- Required pod restart + Traefik restart to restore service
Monitoring now tests full service path from outside cluster, not just pod health.
Will alert immediately on failures instead of relying on reactive discovery.
Related: DEV-213, DEV-221, DEV-223, DEV-224, DEV-230, DEV-231
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-07-11 11:03:51 +00:00
|
|
|
startupProbe:
|
|
|
|
|
httpGet:
|
|
|
|
|
path: /
|
|
|
|
|
port: 8080
|
|
|
|
|
initialDelaySeconds: 60
|
|
|
|
|
periodSeconds: 10
|
|
|
|
|
timeoutSeconds: 5
|
|
|
|
|
failureThreshold: 6
|
2026-07-04 10:17:13 +00:00
|
|
|
livenessProbe:
|
|
|
|
|
httpGet:
|
|
|
|
|
path: /
|
|
|
|
|
port: 8080
|
|
|
|
|
initialDelaySeconds: 30
|
|
|
|
|
periodSeconds: 10
|
|
|
|
|
timeoutSeconds: 5
|
|
|
|
|
failureThreshold: 3
|
|
|
|
|
readinessProbe:
|
|
|
|
|
httpGet:
|
|
|
|
|
path: /
|
|
|
|
|
port: 8080
|
|
|
|
|
initialDelaySeconds: 10
|
|
|
|
|
periodSeconds: 5
|
|
|
|
|
timeoutSeconds: 3
|
|
|
|
|
failureThreshold: 3
|
|
|
|
|
volumes:
|
|
|
|
|
- name: data
|
|
|
|
|
persistentVolumeClaim:
|
|
|
|
|
claimName: stalwart-data
|
2026-07-04 10:34:28 +00:00
|
|
|
- name: bootstrap-config
|
2026-07-04 10:17:13 +00:00
|
|
|
configMap:
|
2026-07-04 10:34:28 +00:00
|
|
|
name: stalwart-bootstrap-config
|
2026-07-04 12:13:18 +00:00
|
|
|
- name: tls-certs
|
|
|
|
|
secret:
|
|
|
|
|
secretName: stalwart-tls
|
|
|
|
|
defaultMode: 0444
|
2026-07-04 10:17:13 +00:00
|
|
|
---
|
|
|
|
|
apiVersion: networking.k8s.io/v1
|
|
|
|
|
kind: Ingress
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-web
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
annotations:
|
|
|
|
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
|
|
|
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
|
|
|
|
spec:
|
|
|
|
|
ingressClassName: traefik
|
|
|
|
|
tls:
|
|
|
|
|
- hosts:
|
|
|
|
|
- mail.basicstack.de
|
|
|
|
|
secretName: stalwart-tls
|
|
|
|
|
rules:
|
|
|
|
|
- host: mail.basicstack.de
|
|
|
|
|
http:
|
|
|
|
|
paths:
|
|
|
|
|
- path: /
|
|
|
|
|
pathType: Prefix
|
|
|
|
|
backend:
|
|
|
|
|
service:
|
|
|
|
|
name: stalwart-http
|
|
|
|
|
port:
|
|
|
|
|
number: 8080
|
|
|
|
|
---
|
2026-07-11 11:21:30 +00:00
|
|
|
apiVersion: v1
|
|
|
|
|
kind: ServiceAccount
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
---
|
|
|
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
|
|
|
kind: Role
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
rules:
|
|
|
|
|
- apiGroups: ["apps"]
|
|
|
|
|
resources: ["statefulsets"]
|
|
|
|
|
verbs: ["get", "list", "patch"]
|
|
|
|
|
- apiGroups: ["apps"]
|
|
|
|
|
resources: ["statefulsets/scale"]
|
|
|
|
|
verbs: ["get", "update", "patch"]
|
|
|
|
|
- apiGroups: [""]
|
|
|
|
|
resources: ["pods"]
|
|
|
|
|
verbs: ["get", "list", "watch"]
|
|
|
|
|
---
|
|
|
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
|
|
|
kind: RoleBinding
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
roleRef:
|
|
|
|
|
apiGroup: rbac.authorization.k8s.io
|
|
|
|
|
kind: Role
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
subjects:
|
|
|
|
|
- kind: ServiceAccount
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
---
|
2026-07-04 10:17:13 +00:00
|
|
|
apiVersion: batch/v1
|
|
|
|
|
kind: CronJob
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-backup
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
schedule: "0 3 * * *" # 3 AM daily
|
|
|
|
|
successfulJobsHistoryLimit: 3
|
|
|
|
|
failedJobsHistoryLimit: 3
|
|
|
|
|
jobTemplate:
|
|
|
|
|
spec:
|
2026-07-11 11:21:30 +00:00
|
|
|
backoffLimit: 2
|
|
|
|
|
activeDeadlineSeconds: 600 # 10 minute timeout
|
2026-07-04 10:17:13 +00:00
|
|
|
template:
|
|
|
|
|
metadata:
|
|
|
|
|
labels:
|
|
|
|
|
app: stalwart-backup
|
|
|
|
|
spec:
|
2026-07-11 11:21:30 +00:00
|
|
|
serviceAccountName: stalwart-backup
|
2026-07-04 10:17:13 +00:00
|
|
|
restartPolicy: OnFailure
|
|
|
|
|
containers:
|
|
|
|
|
- name: backup
|
2026-07-11 11:21:30 +00:00
|
|
|
image: alpine:3.19
|
2026-07-04 10:17:13 +00:00
|
|
|
command:
|
|
|
|
|
- /bin/sh
|
|
|
|
|
- -c
|
|
|
|
|
- |
|
|
|
|
|
set -e
|
2026-07-11 11:21:30 +00:00
|
|
|
BACKUP_START=$(date +%s)
|
|
|
|
|
echo "=== Starting Stalwart backup at $(date) ==="
|
|
|
|
|
|
|
|
|
|
# Install required tools
|
|
|
|
|
echo "Installing kubectl and restic..."
|
|
|
|
|
apk add --no-cache kubectl restic curl bash
|
|
|
|
|
|
|
|
|
|
# Build restic repository URL from env vars
|
|
|
|
|
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
|
|
|
|
|
echo "Using restic repository: $RESTIC_REPOSITORY"
|
|
|
|
|
|
|
|
|
|
# Function to scale StatefulSet
|
|
|
|
|
scale_statefulset() {
|
|
|
|
|
local replicas=$1
|
|
|
|
|
echo "Scaling stalwart StatefulSet to $replicas replicas..."
|
|
|
|
|
kubectl scale statefulset stalwart -n stalwart --replicas=$replicas
|
|
|
|
|
|
|
|
|
|
if [ "$replicas" -eq 0 ]; then
|
|
|
|
|
echo "Waiting for pod to terminate..."
|
|
|
|
|
kubectl wait --for=delete pod/stalwart-0 -n stalwart --timeout=120s || true
|
|
|
|
|
sleep 5
|
|
|
|
|
else
|
|
|
|
|
echo "Waiting for pod to be ready..."
|
|
|
|
|
kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=120s || echo "Warning: Pod not ready after 120s"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Trap to ensure we scale back up on any exit
|
|
|
|
|
cleanup() {
|
|
|
|
|
EXIT_CODE=$?
|
|
|
|
|
echo "Cleanup: scaling stalwart back to 1 replica..."
|
|
|
|
|
scale_statefulset 1 || echo "ERROR: Failed to scale back up!"
|
|
|
|
|
|
|
|
|
|
BACKUP_END=$(date +%s)
|
|
|
|
|
DURATION=$((BACKUP_END - BACKUP_START))
|
|
|
|
|
|
|
|
|
|
if [ $EXIT_CODE -eq 0 ]; then
|
|
|
|
|
echo "=== Backup completed successfully in ${DURATION}s at $(date) ==="
|
|
|
|
|
else
|
|
|
|
|
echo "=== Backup FAILED after ${DURATION}s at $(date) ==="
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
exit $EXIT_CODE
|
|
|
|
|
}
|
|
|
|
|
trap cleanup EXIT INT TERM
|
|
|
|
|
|
|
|
|
|
# Scale down Stalwart
|
|
|
|
|
scale_statefulset 0
|
2026-07-04 10:17:13 +00:00
|
|
|
|
Stalwart reliability hardening: fix k3s service networking issues
Root cause: k3s service ClusterIP routing instability causing intermittent
failures despite healthy pods. This is the 5th incident - prior fixes treated
symptoms, not the systemic networking fragility.
Changes:
- Add startup probe (60s delay, prevents premature service registration)
- Fix backup job env var substitution (use shell ${VAR}, not K8s $(VAR))
- Add comprehensive monitoring (ServiceMonitor, PrometheusRule, blackbox probes)
- Add alerting for service failures, high latency, pod restarts, backup failures
Evidence:
- Pod healthy (4d15h uptime, 0 restarts) but service ClusterIP routing broken
- Direct pod IP worked, service ClusterIP failed with "Connection reset by peer"
- Iptables rules correct, endpoints correct, but packets not flowing
- Required pod restart + Traefik restart to restore service
Monitoring now tests full service path from outside cluster, not just pod health.
Will alert immediately on failures instead of relying on reactive discovery.
Related: DEV-213, DEV-221, DEV-223, DEV-224, DEV-230, DEV-231
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-07-11 11:03:51 +00:00
|
|
|
# Build restic repository URL from env vars (K8s doesn't expand $(VAR) in value fields)
|
|
|
|
|
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
|
|
|
|
|
echo "Using repository: $RESTIC_REPOSITORY"
|
|
|
|
|
|
2026-07-04 10:17:13 +00:00
|
|
|
# Initialize restic repo if needed
|
2026-07-11 11:21:30 +00:00
|
|
|
echo "Initializing restic repository..."
|
2026-07-04 10:17:13 +00:00
|
|
|
restic snapshots || restic init
|
|
|
|
|
|
|
|
|
|
# Backup the data directory
|
2026-07-11 11:21:30 +00:00
|
|
|
echo "Running backup..."
|
2026-07-04 10:17:13 +00:00
|
|
|
restic backup /var/lib/stalwart \
|
|
|
|
|
--tag stalwart \
|
|
|
|
|
--tag daily \
|
2026-07-11 11:21:30 +00:00
|
|
|
--host stalwart-k8s \
|
|
|
|
|
--verbose
|
2026-07-04 10:17:13 +00:00
|
|
|
|
|
|
|
|
# Prune old backups (keep 7 daily, 4 weekly, 6 monthly)
|
2026-07-11 11:21:30 +00:00
|
|
|
echo "Pruning old backups..."
|
2026-07-04 10:17:13 +00:00
|
|
|
restic forget \
|
2026-07-11 11:21:30 +00:00
|
|
|
--tag stalwart \
|
2026-07-04 10:17:13 +00:00
|
|
|
--keep-daily 7 \
|
|
|
|
|
--keep-weekly 4 \
|
|
|
|
|
--keep-monthly 6 \
|
|
|
|
|
--prune
|
|
|
|
|
|
2026-07-11 11:21:30 +00:00
|
|
|
echo "Backup operations completed"
|
2026-07-04 10:17:13 +00:00
|
|
|
env:
|
|
|
|
|
- name: RESTIC_PASSWORD
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: restic-password
|
|
|
|
|
- name: AWS_ACCESS_KEY_ID
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: access-key
|
|
|
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: secret-key
|
|
|
|
|
- name: S3_ENDPOINT
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: endpoint
|
|
|
|
|
- name: S3_BUCKET
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-s3-backup
|
|
|
|
|
key: bucket
|
|
|
|
|
volumeMounts:
|
|
|
|
|
- name: data
|
|
|
|
|
mountPath: /var/lib/stalwart
|
|
|
|
|
readOnly: true
|
|
|
|
|
volumes:
|
|
|
|
|
- name: data
|
|
|
|
|
persistentVolumeClaim:
|
|
|
|
|
claimName: stalwart-data
|