2026-07-04 10:17:13 +00:00
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Namespace
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart
|
|
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Service
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-smtp
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
type: LoadBalancer
|
2026-07-25 09:35:16 +00:00
|
|
|
externalTrafficPolicy: Cluster
|
2026-07-04 10:17:13 +00:00
|
|
|
selector:
|
|
|
|
|
app: stalwart
|
|
|
|
|
ports:
|
|
|
|
|
- name: smtp
|
|
|
|
|
port: 25
|
|
|
|
|
targetPort: 25
|
|
|
|
|
protocol: TCP
|
|
|
|
|
- name: submission
|
|
|
|
|
port: 587
|
|
|
|
|
targetPort: 587
|
|
|
|
|
protocol: TCP
|
2026-07-22 18:04:05 +00:00
|
|
|
# Port 465 (SMTPS) removed per DEV-359 approval - enforce secure protocols
|
2026-07-04 10:17:13 +00:00
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Service
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-imap
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
type: LoadBalancer
|
2026-07-25 09:35:16 +00:00
|
|
|
externalTrafficPolicy: Cluster
|
2026-07-04 10:17:13 +00:00
|
|
|
selector:
|
|
|
|
|
app: stalwart
|
|
|
|
|
ports:
|
2026-07-22 18:04:05 +00:00
|
|
|
# Port 143 (IMAP) removed per DEV-359 approval - enforce secure protocols
|
2026-07-04 10:17:13 +00:00
|
|
|
- name: imaps
|
|
|
|
|
port: 993
|
|
|
|
|
targetPort: 993
|
|
|
|
|
protocol: TCP
|
|
|
|
|
---
|
|
|
|
|
apiVersion: v1
|
|
|
|
|
kind: Service
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-http
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
type: ClusterIP
|
|
|
|
|
selector:
|
|
|
|
|
app: stalwart
|
|
|
|
|
ports:
|
|
|
|
|
- name: http
|
|
|
|
|
port: 8080
|
|
|
|
|
targetPort: 8080
|
|
|
|
|
protocol: TCP
|
|
|
|
|
---
|
|
|
|
|
apiVersion: apps/v1
|
|
|
|
|
kind: StatefulSet
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart
|
|
|
|
|
namespace: stalwart
|
2026-07-04 16:48:48 +00:00
|
|
|
annotations:
|
|
|
|
|
# Automatically restart this StatefulSet when the TLS certificate secret is updated
|
|
|
|
|
# This ensures the pod reloads new certificates after cert-manager renews them
|
|
|
|
|
secret.reloader.stakater.com/reload: "stalwart-tls"
|
2026-07-04 10:17:13 +00:00
|
|
|
spec:
|
|
|
|
|
serviceName: stalwart-http
|
|
|
|
|
replicas: 1
|
|
|
|
|
selector:
|
|
|
|
|
matchLabels:
|
|
|
|
|
app: stalwart
|
|
|
|
|
template:
|
|
|
|
|
metadata:
|
|
|
|
|
labels:
|
|
|
|
|
app: stalwart
|
|
|
|
|
spec:
|
|
|
|
|
initContainers:
|
2026-08-01 09:47:22 +00:00
|
|
|
# Copy ConfigMap files to emptyDir to avoid subPath mounting race
|
|
|
|
|
- name: copy-config
|
|
|
|
|
image: busybox:latest
|
|
|
|
|
command:
|
|
|
|
|
- sh
|
|
|
|
|
- -c
|
|
|
|
|
- |
|
2026-08-01 09:52:26 +00:00
|
|
|
set -e
|
|
|
|
|
echo "Copying bootstrap config..."
|
|
|
|
|
cp /tmp/bootstrap-config/* /etc/stalwart/
|
|
|
|
|
echo "Copying stalwart.toml..."
|
|
|
|
|
cp /tmp/stalwart-config/stalwart.toml /etc/stalwart/stalwart.toml
|
|
|
|
|
echo "Config files copied successfully:"
|
2026-08-01 09:47:22 +00:00
|
|
|
ls -la /etc/stalwart/
|
|
|
|
|
volumeMounts:
|
|
|
|
|
- name: bootstrap-config-source
|
|
|
|
|
mountPath: /tmp/bootstrap-config
|
|
|
|
|
readOnly: true
|
2026-08-01 09:52:26 +00:00
|
|
|
- name: stalwart-config-source
|
|
|
|
|
mountPath: /tmp/stalwart-config
|
|
|
|
|
readOnly: true
|
2026-08-01 09:47:22 +00:00
|
|
|
- name: config
|
|
|
|
|
mountPath: /etc/stalwart
|
2026-07-04 10:17:13 +00:00
|
|
|
containers:
|
|
|
|
|
- name: stalwart
|
2026-08-23 16:46:31 +00:00
|
|
|
image: stalwartlabs/stalwart:v0.16.18
|
2026-07-04 10:17:13 +00:00
|
|
|
ports:
|
|
|
|
|
- containerPort: 25
|
|
|
|
|
name: smtp
|
|
|
|
|
- containerPort: 587
|
|
|
|
|
name: submission
|
2026-07-22 18:04:05 +00:00
|
|
|
# Port 465 (SMTPS) removed - enforce STARTTLS on 587
|
|
|
|
|
# Port 143 (IMAP) removed - enforce TLS on 993
|
2026-07-04 10:17:13 +00:00
|
|
|
- containerPort: 993
|
|
|
|
|
name: imaps
|
|
|
|
|
- containerPort: 8080
|
|
|
|
|
name: http
|
|
|
|
|
env:
|
2026-07-26 13:31:38 +00:00
|
|
|
#- name: STALWART_RECOVERY_ADMIN
|
|
|
|
|
# valueFrom:
|
|
|
|
|
# secretKeyRef:
|
|
|
|
|
# name: stalwart-admin-credentials
|
|
|
|
|
# key: admin-password
|
|
|
|
|
# optional: false
|
2026-07-04 13:22:36 +00:00
|
|
|
- name: TLS_CERTIFICATE
|
|
|
|
|
value: "/etc/stalwart/certs/tls.crt"
|
|
|
|
|
- name: TLS_PRIVATE_KEY
|
|
|
|
|
value: "/etc/stalwart/certs/tls.key"
|
2026-08-15 14:46:29 +00:00
|
|
|
# DEV-476: bootstrap config.json resolves authSecret via this env var.
|
|
|
|
|
- name: PGPASSWORD
|
|
|
|
|
valueFrom:
|
|
|
|
|
secretKeyRef:
|
|
|
|
|
name: stalwart-postgres-credentials
|
|
|
|
|
key: POSTGRES_PASSWORD
|
2026-07-04 10:17:13 +00:00
|
|
|
volumeMounts:
|
|
|
|
|
- name: data
|
|
|
|
|
mountPath: /var/lib/stalwart
|
2026-08-01 09:47:22 +00:00
|
|
|
- name: config
|
|
|
|
|
mountPath: /etc/stalwart
|
2026-07-04 12:13:18 +00:00
|
|
|
- name: tls-certs
|
|
|
|
|
mountPath: /etc/stalwart/certs
|
|
|
|
|
readOnly: true
|
chore(stalwart): retire RocksDB PVC + backup — Postgres backend is authoritative (DEV-474)
Post-cutover 14-day observation window on DEV-474 closed today with 14/14
healthy days. This removes every RocksDB-era artefact from the manifest and
updates the README to reflect the PostgreSQL backend as the default:
- Delete PVC `stalwart-data` (was 20Gi hcloud-volumes-encrypted, RWO).
- Drop the `fix-permissions` init-container (existed only to chown the
RocksDB PVC mount).
- Replace the StatefulSet's `data` volume with an `emptyDir` and keep the
`/var/lib/stalwart` mount as a plain scratch path (Stalwart's state is
now in the `stalwart-postgres` StatefulSet).
- Delete the `stalwart-backup` CronJob and its `stalwart-backup`
ServiceAccount / Role / RoleBinding — RocksDB snapshotting is obsolete
now that state lives in Postgres. Daily backup is via
`stalwart-postgres-backup` (restic tag `stalwart-postgres`).
- README: update version, backend, storage, backup, files, and deployment
sections; correct exposed-ports language to match SMTP 25/587 + IMAPS 993.
Executed alongside the merge:
- `restic forget --prune` removed all 11 snapshots on tag=stalwart (Job
`stalwart-restic-forget-rocksdb` on 2026-08-28 15:09 UTC, log preserved
on DEV-474). 16.165 MiB pruned. `restic snapshots --tag stalwart` now
returns empty; only the 14 tag=`stalwart-postgres` snapshots remain.
- `stalwart-backup` CronJob was pre-suspended in-cluster to prevent
tonight's 03:00 UTC fire from re-creating a stalwart-tagged snapshot
before this PR merges.
Rollback (if needed before merge): revert this commit; the old CronJob
would then re-create RocksDB snapshots from live PVC contents. After
merge the rollback path is the Postgres `pg_dump` restic chain only.
Refs: DEV-467, DEV-473 (cutover), DEV-474 (this observation + teardown),
DEV-548 (day-14 execution issue).
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-08-28 15:12:25 +00:00
|
|
|
# DEV-474: /var/lib/stalwart is no longer RocksDB storage (state now lives in
|
|
|
|
|
# the stalwart-postgres StatefulSet). Kept as emptyDir so stalwart still has
|
|
|
|
|
# a writable path if it needs one at runtime.
|
2026-07-04 10:17:13 +00:00
|
|
|
resources:
|
|
|
|
|
requests:
|
|
|
|
|
memory: "512Mi"
|
|
|
|
|
cpu: "250m"
|
|
|
|
|
limits:
|
|
|
|
|
memory: "2Gi"
|
|
|
|
|
cpu: "2000m"
|
Stalwart reliability hardening: fix k3s service networking issues
Root cause: k3s service ClusterIP routing instability causing intermittent
failures despite healthy pods. This is the 5th incident - prior fixes treated
symptoms, not the systemic networking fragility.
Changes:
- Add startup probe (60s delay, prevents premature service registration)
- Fix backup job env var substitution (use shell ${VAR}, not K8s $(VAR))
- Add comprehensive monitoring (ServiceMonitor, PrometheusRule, blackbox probes)
- Add alerting for service failures, high latency, pod restarts, backup failures
Evidence:
- Pod healthy (4d15h uptime, 0 restarts) but service ClusterIP routing broken
- Direct pod IP worked, service ClusterIP failed with "Connection reset by peer"
- Iptables rules correct, endpoints correct, but packets not flowing
- Required pod restart + Traefik restart to restore service
Monitoring now tests full service path from outside cluster, not just pod health.
Will alert immediately on failures instead of relying on reactive discovery.
Related: DEV-213, DEV-221, DEV-223, DEV-224, DEV-230, DEV-231
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-07-11 11:03:51 +00:00
|
|
|
startupProbe:
|
2026-08-01 08:14:41 +00:00
|
|
|
exec:
|
|
|
|
|
command:
|
|
|
|
|
- curl
|
|
|
|
|
- -f
|
|
|
|
|
- --max-time
|
|
|
|
|
- "3"
|
|
|
|
|
- http://localhost:8080/healthz/live
|
Stalwart reliability hardening: fix k3s service networking issues
Root cause: k3s service ClusterIP routing instability causing intermittent
failures despite healthy pods. This is the 5th incident - prior fixes treated
symptoms, not the systemic networking fragility.
Changes:
- Add startup probe (60s delay, prevents premature service registration)
- Fix backup job env var substitution (use shell ${VAR}, not K8s $(VAR))
- Add comprehensive monitoring (ServiceMonitor, PrometheusRule, blackbox probes)
- Add alerting for service failures, high latency, pod restarts, backup failures
Evidence:
- Pod healthy (4d15h uptime, 0 restarts) but service ClusterIP routing broken
- Direct pod IP worked, service ClusterIP failed with "Connection reset by peer"
- Iptables rules correct, endpoints correct, but packets not flowing
- Required pod restart + Traefik restart to restore service
Monitoring now tests full service path from outside cluster, not just pod health.
Will alert immediately on failures instead of relying on reactive discovery.
Related: DEV-213, DEV-221, DEV-223, DEV-224, DEV-230, DEV-231
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-07-11 11:03:51 +00:00
|
|
|
initialDelaySeconds: 60
|
|
|
|
|
periodSeconds: 10
|
|
|
|
|
timeoutSeconds: 5
|
|
|
|
|
failureThreshold: 6
|
2026-07-04 10:17:13 +00:00
|
|
|
livenessProbe:
|
2026-08-01 08:14:41 +00:00
|
|
|
exec:
|
|
|
|
|
command:
|
|
|
|
|
- curl
|
|
|
|
|
- -f
|
|
|
|
|
- --max-time
|
|
|
|
|
- "3"
|
|
|
|
|
- http://localhost:8080/healthz/live
|
2026-07-04 10:17:13 +00:00
|
|
|
initialDelaySeconds: 30
|
|
|
|
|
periodSeconds: 10
|
|
|
|
|
timeoutSeconds: 5
|
|
|
|
|
failureThreshold: 3
|
|
|
|
|
readinessProbe:
|
2026-08-01 08:14:41 +00:00
|
|
|
exec:
|
|
|
|
|
command:
|
|
|
|
|
- curl
|
|
|
|
|
- -f
|
|
|
|
|
- --max-time
|
|
|
|
|
- "3"
|
|
|
|
|
- http://localhost:8080/healthz/ready
|
2026-07-04 10:17:13 +00:00
|
|
|
initialDelaySeconds: 10
|
|
|
|
|
periodSeconds: 5
|
|
|
|
|
timeoutSeconds: 3
|
|
|
|
|
failureThreshold: 3
|
|
|
|
|
volumes:
|
chore(stalwart): retire RocksDB PVC + backup — Postgres backend is authoritative (DEV-474)
Post-cutover 14-day observation window on DEV-474 closed today with 14/14
healthy days. This removes every RocksDB-era artefact from the manifest and
updates the README to reflect the PostgreSQL backend as the default:
- Delete PVC `stalwart-data` (was 20Gi hcloud-volumes-encrypted, RWO).
- Drop the `fix-permissions` init-container (existed only to chown the
RocksDB PVC mount).
- Replace the StatefulSet's `data` volume with an `emptyDir` and keep the
`/var/lib/stalwart` mount as a plain scratch path (Stalwart's state is
now in the `stalwart-postgres` StatefulSet).
- Delete the `stalwart-backup` CronJob and its `stalwart-backup`
ServiceAccount / Role / RoleBinding — RocksDB snapshotting is obsolete
now that state lives in Postgres. Daily backup is via
`stalwart-postgres-backup` (restic tag `stalwart-postgres`).
- README: update version, backend, storage, backup, files, and deployment
sections; correct exposed-ports language to match SMTP 25/587 + IMAPS 993.
Executed alongside the merge:
- `restic forget --prune` removed all 11 snapshots on tag=stalwart (Job
`stalwart-restic-forget-rocksdb` on 2026-08-28 15:09 UTC, log preserved
on DEV-474). 16.165 MiB pruned. `restic snapshots --tag stalwart` now
returns empty; only the 14 tag=`stalwart-postgres` snapshots remain.
- `stalwart-backup` CronJob was pre-suspended in-cluster to prevent
tonight's 03:00 UTC fire from re-creating a stalwart-tagged snapshot
before this PR merges.
Rollback (if needed before merge): revert this commit; the old CronJob
would then re-create RocksDB snapshots from live PVC contents. After
merge the rollback path is the Postgres `pg_dump` restic chain only.
Refs: DEV-467, DEV-473 (cutover), DEV-474 (this observation + teardown),
DEV-548 (day-14 execution issue).
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-08-28 15:12:25 +00:00
|
|
|
# DEV-474: was persistentVolumeClaim.claimName=stalwart-data (RocksDB).
|
|
|
|
|
# Postgres-backed now; replaced with emptyDir.
|
2026-07-04 10:17:13 +00:00
|
|
|
- name: data
|
chore(stalwart): retire RocksDB PVC + backup — Postgres backend is authoritative (DEV-474)
Post-cutover 14-day observation window on DEV-474 closed today with 14/14
healthy days. This removes every RocksDB-era artefact from the manifest and
updates the README to reflect the PostgreSQL backend as the default:
- Delete PVC `stalwart-data` (was 20Gi hcloud-volumes-encrypted, RWO).
- Drop the `fix-permissions` init-container (existed only to chown the
RocksDB PVC mount).
- Replace the StatefulSet's `data` volume with an `emptyDir` and keep the
`/var/lib/stalwart` mount as a plain scratch path (Stalwart's state is
now in the `stalwart-postgres` StatefulSet).
- Delete the `stalwart-backup` CronJob and its `stalwart-backup`
ServiceAccount / Role / RoleBinding — RocksDB snapshotting is obsolete
now that state lives in Postgres. Daily backup is via
`stalwart-postgres-backup` (restic tag `stalwart-postgres`).
- README: update version, backend, storage, backup, files, and deployment
sections; correct exposed-ports language to match SMTP 25/587 + IMAPS 993.
Executed alongside the merge:
- `restic forget --prune` removed all 11 snapshots on tag=stalwart (Job
`stalwart-restic-forget-rocksdb` on 2026-08-28 15:09 UTC, log preserved
on DEV-474). 16.165 MiB pruned. `restic snapshots --tag stalwart` now
returns empty; only the 14 tag=`stalwart-postgres` snapshots remain.
- `stalwart-backup` CronJob was pre-suspended in-cluster to prevent
tonight's 03:00 UTC fire from re-creating a stalwart-tagged snapshot
before this PR merges.
Rollback (if needed before merge): revert this commit; the old CronJob
would then re-create RocksDB snapshots from live PVC contents. After
merge the rollback path is the Postgres `pg_dump` restic chain only.
Refs: DEV-467, DEV-473 (cutover), DEV-474 (this observation + teardown),
DEV-548 (day-14 execution issue).
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-08-28 15:12:25 +00:00
|
|
|
emptyDir: {}
|
2026-08-01 09:47:22 +00:00
|
|
|
# emptyDir for config files (populated by copy-config init-container)
|
|
|
|
|
- name: config
|
|
|
|
|
emptyDir: {}
|
2026-08-01 09:52:26 +00:00
|
|
|
# ConfigMap sources for init-container (no longer mounted directly via subPath)
|
2026-08-01 09:47:22 +00:00
|
|
|
- name: bootstrap-config-source
|
2026-07-04 10:17:13 +00:00
|
|
|
configMap:
|
2026-08-01 08:51:59 +00:00
|
|
|
name: stalwart-bootstrap-config
|
2026-08-01 09:52:26 +00:00
|
|
|
- name: stalwart-config-source
|
|
|
|
|
configMap:
|
|
|
|
|
name: stalwart-config
|
2026-07-04 12:13:18 +00:00
|
|
|
- name: tls-certs
|
|
|
|
|
secret:
|
|
|
|
|
secretName: stalwart-tls
|
|
|
|
|
defaultMode: 0444
|
2026-07-04 10:17:13 +00:00
|
|
|
---
|
|
|
|
|
apiVersion: networking.k8s.io/v1
|
|
|
|
|
kind: Ingress
|
|
|
|
|
metadata:
|
|
|
|
|
name: stalwart-web
|
|
|
|
|
namespace: stalwart
|
|
|
|
|
annotations:
|
|
|
|
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
|
|
|
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
|
|
|
|
spec:
|
|
|
|
|
ingressClassName: traefik
|
|
|
|
|
tls:
|
|
|
|
|
- hosts:
|
|
|
|
|
- mail.basicstack.de
|
|
|
|
|
secretName: stalwart-tls
|
|
|
|
|
rules:
|
|
|
|
|
- host: mail.basicstack.de
|
|
|
|
|
http:
|
|
|
|
|
paths:
|
|
|
|
|
- path: /
|
|
|
|
|
pathType: Prefix
|
|
|
|
|
backend:
|
|
|
|
|
service:
|
|
|
|
|
name: stalwart-http
|
|
|
|
|
port:
|
|
|
|
|
number: 8080
|