Root cause: ArgoCD continuously reverts the LoadBalancer services to use
externalTrafficPolicy: Cluster (the k8s default), causing k3s to assign
internal flannel VXLAN IPs (10.42.1.x) instead of the node public IPs.
With externalTrafficPolicy: Cluster, traffic can be routed to any node,
and k3s's service controller assigns the flannel overlay IPs. This breaks
external connectivity because those IPs are not routable from outside.
With externalTrafficPolicy: Local, traffic is only routed to pods on the
same node, and k3s assigns the node's actual public IP to the LoadBalancer.
This was the missing piece from the reliability hardening in commit b0f2acf.
Without this in git, any manual kubectl patch is reverted by ArgoCD sync.
Evidence: stalwart-smtp and stalwart-imap both showing LoadBalancer IPs:
10.42.1.1, 10.42.1.2, 10.42.1.3, 10.42.1.5 (internal flannel IPs)
Related: DEV-230, DEV-231, DEV-233, DEV-235
Co-Authored-By: Paperclip <noreply@paperclip.ing>
383 lines
10 KiB
YAML
383 lines
10 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: stalwart
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: stalwart-data
|
|
namespace: stalwart
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: hcloud-volumes-encrypted
|
|
resources:
|
|
requests:
|
|
storage: 20Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: stalwart-smtp
|
|
namespace: stalwart
|
|
spec:
|
|
type: LoadBalancer
|
|
externalTrafficPolicy: Local
|
|
selector:
|
|
app: stalwart
|
|
ports:
|
|
- name: smtp
|
|
port: 25
|
|
targetPort: 25
|
|
protocol: TCP
|
|
- name: submission
|
|
port: 587
|
|
targetPort: 587
|
|
protocol: TCP
|
|
- name: submissions
|
|
port: 465
|
|
targetPort: 465
|
|
protocol: TCP
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: stalwart-imap
|
|
namespace: stalwart
|
|
spec:
|
|
type: LoadBalancer
|
|
externalTrafficPolicy: Local
|
|
selector:
|
|
app: stalwart
|
|
ports:
|
|
- name: imap
|
|
port: 143
|
|
targetPort: 143
|
|
protocol: TCP
|
|
- name: imaps
|
|
port: 993
|
|
targetPort: 993
|
|
protocol: TCP
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: stalwart-http
|
|
namespace: stalwart
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
app: stalwart
|
|
ports:
|
|
- name: http
|
|
port: 8080
|
|
targetPort: 8080
|
|
protocol: TCP
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: stalwart
|
|
namespace: stalwart
|
|
annotations:
|
|
# Automatically restart this StatefulSet when the TLS certificate secret is updated
|
|
# This ensures the pod reloads new certificates after cert-manager renews them
|
|
secret.reloader.stakater.com/reload: "stalwart-tls"
|
|
spec:
|
|
serviceName: stalwart-http
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: stalwart
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: stalwart
|
|
spec:
|
|
initContainers:
|
|
- name: fix-permissions
|
|
image: busybox:latest
|
|
command: ["sh", "-c", "chown -R 2000:2000 /var/lib/stalwart && chmod -R 755 /var/lib/stalwart"]
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /var/lib/stalwart
|
|
containers:
|
|
- name: stalwart
|
|
image: stalwartlabs/stalwart:v0.16.11
|
|
ports:
|
|
- containerPort: 25
|
|
name: smtp
|
|
- containerPort: 587
|
|
name: submission
|
|
- containerPort: 465
|
|
name: submissions
|
|
- containerPort: 143
|
|
name: imap
|
|
- containerPort: 993
|
|
name: imaps
|
|
- containerPort: 8080
|
|
name: http
|
|
env:
|
|
- name: STALWART_RECOVERY_ADMIN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-admin-credentials
|
|
key: admin-password
|
|
optional: false
|
|
- name: TLS_CERTIFICATE
|
|
value: "/etc/stalwart/certs/tls.crt"
|
|
- name: TLS_PRIVATE_KEY
|
|
value: "/etc/stalwart/certs/tls.key"
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /var/lib/stalwart
|
|
- name: bootstrap-config
|
|
mountPath: /etc/stalwart/config.json
|
|
subPath: config.json
|
|
- name: tls-certs
|
|
mountPath: /etc/stalwart/certs
|
|
readOnly: true
|
|
resources:
|
|
requests:
|
|
memory: "512Mi"
|
|
cpu: "250m"
|
|
limits:
|
|
memory: "2Gi"
|
|
cpu: "2000m"
|
|
startupProbe:
|
|
httpGet:
|
|
path: /
|
|
port: 8080
|
|
initialDelaySeconds: 60
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
failureThreshold: 6
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /
|
|
port: 8080
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
failureThreshold: 3
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /
|
|
port: 8080
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 5
|
|
timeoutSeconds: 3
|
|
failureThreshold: 3
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: stalwart-data
|
|
- name: bootstrap-config
|
|
configMap:
|
|
name: stalwart-bootstrap-config
|
|
- name: tls-certs
|
|
secret:
|
|
secretName: stalwart-tls
|
|
defaultMode: 0444
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: stalwart-web
|
|
namespace: stalwart
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
|
spec:
|
|
ingressClassName: traefik
|
|
tls:
|
|
- hosts:
|
|
- mail.basicstack.de
|
|
secretName: stalwart-tls
|
|
rules:
|
|
- host: mail.basicstack.de
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: stalwart-http
|
|
port:
|
|
number: 8080
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
rules:
|
|
- apiGroups: ["apps"]
|
|
resources: ["statefulsets"]
|
|
verbs: ["get", "list", "patch"]
|
|
- apiGroups: ["apps"]
|
|
resources: ["statefulsets/scale"]
|
|
verbs: ["get", "update", "patch"]
|
|
- apiGroups: [""]
|
|
resources: ["pods"]
|
|
verbs: ["get", "list", "watch"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: stalwart-backup
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: stalwart-backup
|
|
namespace: stalwart
|
|
spec:
|
|
schedule: "0 3 * * *" # 3 AM daily
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 2
|
|
activeDeadlineSeconds: 600 # 10 minute timeout
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: stalwart-backup
|
|
spec:
|
|
serviceAccountName: stalwart-backup
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: backup
|
|
image: alpine:3.19
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
BACKUP_START=$(date +%s)
|
|
echo "=== Starting Stalwart backup at $(date) ==="
|
|
|
|
# Install required tools
|
|
echo "Installing kubectl and restic..."
|
|
apk add --no-cache kubectl restic curl bash
|
|
|
|
# Build restic repository URL from env vars
|
|
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
|
|
echo "Using restic repository: $RESTIC_REPOSITORY"
|
|
|
|
# Function to scale StatefulSet
|
|
scale_statefulset() {
|
|
local replicas=$1
|
|
echo "Scaling stalwart StatefulSet to $replicas replicas..."
|
|
kubectl scale statefulset stalwart -n stalwart --replicas=$replicas
|
|
|
|
if [ "$replicas" -eq 0 ]; then
|
|
echo "Waiting for pod to terminate..."
|
|
kubectl wait --for=delete pod/stalwart-0 -n stalwart --timeout=120s || true
|
|
sleep 5
|
|
else
|
|
echo "Waiting for pod to be ready..."
|
|
kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=120s || echo "Warning: Pod not ready after 120s"
|
|
fi
|
|
}
|
|
|
|
# Trap to ensure we scale back up on any exit
|
|
cleanup() {
|
|
EXIT_CODE=$?
|
|
echo "Cleanup: scaling stalwart back to 1 replica..."
|
|
scale_statefulset 1 || echo "ERROR: Failed to scale back up!"
|
|
|
|
BACKUP_END=$(date +%s)
|
|
DURATION=$((BACKUP_END - BACKUP_START))
|
|
|
|
if [ $EXIT_CODE -eq 0 ]; then
|
|
echo "=== Backup completed successfully in ${DURATION}s at $(date) ==="
|
|
else
|
|
echo "=== Backup FAILED after ${DURATION}s at $(date) ==="
|
|
fi
|
|
|
|
exit $EXIT_CODE
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# Scale down Stalwart
|
|
scale_statefulset 0
|
|
|
|
# Build restic repository URL from env vars (K8s doesn't expand $(VAR) in value fields)
|
|
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
|
|
echo "Using repository: $RESTIC_REPOSITORY"
|
|
|
|
# Initialize restic repo if needed
|
|
echo "Initializing restic repository..."
|
|
restic snapshots || restic init
|
|
|
|
# Backup the data directory
|
|
echo "Running backup..."
|
|
restic backup /var/lib/stalwart \
|
|
--tag stalwart \
|
|
--tag daily \
|
|
--host stalwart-k8s \
|
|
--verbose
|
|
|
|
# Prune old backups (keep 7 daily, 4 weekly, 6 monthly)
|
|
echo "Pruning old backups..."
|
|
restic forget \
|
|
--tag stalwart \
|
|
--keep-daily 7 \
|
|
--keep-weekly 4 \
|
|
--keep-monthly 6 \
|
|
--prune
|
|
|
|
echo "Backup operations completed"
|
|
env:
|
|
- name: RESTIC_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: restic-password
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: access-key
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: secret-key
|
|
- name: S3_ENDPOINT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: endpoint
|
|
- name: S3_BUCKET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: stalwart-s3-backup
|
|
key: bucket
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /var/lib/stalwart
|
|
readOnly: true
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: stalwart-data
|