chore(stalwart): bump v0.16.11 -> v0.16.18 (DEV-542)

Transport-only patch bump. No auth/OIDC/config changes.
Snapshots (config.json, DB dump, live YAML) attached to DEV-542.
Rollback: git revert; ArgoCD resyncs; PVCs retained.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

apps/bookstack/bookstack-deployment.yaml

@@ -100,6 +100,8 @@ metadata:
   namespace: bookstack
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: bookstack

apps/directus/directus-deployment.yaml

@@ -35,7 +35,7 @@ spec:
         fsGroup: 1000
       containers:
       - name: directus
-        image: directus/directus:12.1.1
+        image: directus/directus:12.3.0
         ports:
         - name: http
           containerPort: 8055

apps/dozzle/deployment.yaml

@@ -18,7 +18,7 @@ spec:
       serviceAccountName: dozzle
       containers:
       - name: dozzle
-        image: amir20/dozzle:v10.6.10
+        image: amir20/dozzle:v10.7.3
         ports:
         - containerPort: 8080
           name: dozzle-http

apps/forgejo/forgejo-backup-cronjob.yaml

@@ -96,7 +96,7 @@ spec:
                   memory: 512Mi
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.17.3
+              image: harbor.basicstack.de/library/restic:0.19.1
               env:
                 - name: AWS_ACCESS_KEY_ID
                   valueFrom:

apps/headlamp/deployment.yaml

@@ -16,7 +16,7 @@ spec:
       serviceAccountName: headlamp-admin
       containers:
       - name: headlamp
-        image: ghcr.io/headlamp-k8s/headlamp:v0.43.0
+        image: ghcr.io/headlamp-k8s/headlamp:v0.45.0
         imagePullPolicy: IfNotPresent
         args:
         - "-in-cluster"

apps/monitoring/README.md

@@ -2,7 +2,7 @@
 
 Manifests recording the cluster-side monitoring backup CronJobs that were previously applied out-of-band. These files are the authoritative source (`kubectl apply -f apps/monitoring/`). See [DEV-464](/DEV/issues/DEV-464) for the repair context.
 
-- `backup-k8s-resources-cronjob.yaml` — daily dump of cluster-scoped and per-namespace Kubernetes resources, streamed through `restic backup --stdin` to `hetzner-s3:${BUCKET}/restic/k8s-resources`. Uses `serviceAccountName: backup-sa` and no PVC mount (init container `alpine/k8s:1.29.4` writes an emptyDir, main container `restic/restic:0.17.3` reads it on stdin). Rewritten from the local-path tarball per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-487](/DEV/issues/DEV-487)).
+- `backup-k8s-resources-cronjob.yaml` — daily dump of cluster-scoped and per-namespace Kubernetes resources, streamed through `restic backup --stdin` to `hetzner-s3:${BUCKET}/restic/k8s-resources`. Uses `serviceAccountName: backup-sa` and no PVC mount (init container `alpine/k8s:1.29.4` writes an emptyDir, main container `restic/restic:0.19.1` reads it on stdin). Rewritten from the local-path tarball per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-487](/DEV/issues/DEV-487)).
 - `backup-loki-restic-cronjob.yaml` — daily restic backup of `loki-storage-encrypted` to `hetzner-s3:${BUCKET}/restic/loki`. Co-schedules with the Loki pod via `podAffinity` (RWO permits additional read-only mounts on the same node). Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-485](/DEV/issues/DEV-485)).
 - `backup-grafana-restic-cronjob.yaml` — daily restic backup of `grafana-storage` to `hetzner-s3:${BUCKET}/restic/grafana`. Pinned to `k3s-worker-2` via `nodeSelector` (the local-path PV anchors the grafana pod there already, no `podAffinity` needed). Schedule `15 3 * * *` — offset from the loki run at `03:00`. Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-486](/DEV/issues/DEV-486)).
 - `prometheus-backup-cronjob.yaml` + `prometheus-backup-sealed.yaml` — daily restic backup of `prometheus-data-encrypted` to `hetzner-s3:${BUCKET}/restic/prometheus`. Co-schedules with the Prometheus pod via `podAffinity` so the RWO PVC attaches on the same node. Schedule `30 3 * * *` — offset from the loki (03:00) and grafana (03:15) runs. Migrated from the DEV-465 `rclone sync` job to restic client-side encryption per [DEV-492](/DEV/issues/DEV-492) / [DEV-482](/DEV/issues/DEV-482) Option 4. **Compaction-race mitigation:** `--exclude wal/*` + `--exclude chunks_head/*` + accept `restic backup` exit code 3 (source file vanished mid-walk) as a warning, not a failure; restore drill re-runs `promtool tsdb analyze` per block.

apps/monitoring/backup-grafana-restic-cronjob.yaml

@@ -44,8 +44,8 @@ spec:
             kubernetes.io/hostname: k3s-worker-2
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.17.3
+              image: harbor.basicstack.de/library/restic:0.19.1
-              # Mirrored from docker.io/restic/restic:0.17.3 (DEV-493) —
+              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
               # deterministic ingress via Harbor. Retag procedure in
               # docs/monitoring/restic-restore.md § "Tag-bump procedure".
               env:

apps/monitoring/backup-k8s-resources-cronjob.yaml

@@ -11,7 +11,7 @@
 #   1. `kubectl-dump` init container (`alpine/k8s:1.29.4`) writes
 #      /dump/cluster.yaml into an emptyDir. Uses `serviceAccountName:
 #      backup-sa` (unchanged from the legacy job).
-#   2. `restic` main container (`restic/restic:0.17.3`, matches the
+#   2. `restic` main container (`restic/restic:0.19.1`, matches the
 #      loki/grafana siblings) reads that file on stdin and streams it
 #      into the restic repo with `--stdin-filename cluster.yaml`.
 apiVersion: batch/v1
@@ -90,8 +90,8 @@ spec:
                   mountPath: /dump
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.17.3
+              image: harbor.basicstack.de/library/restic:0.19.1
-              # Mirrored from docker.io/restic/restic:0.17.3 (DEV-493) —
+              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
               # deterministic ingress via Harbor. Retag procedure in
               # docs/monitoring/restic-restore.md § "Tag-bump procedure".
               env:

apps/monitoring/backup-loki-restic-cronjob.yaml

@@ -53,8 +53,8 @@ spec:
                   topologyKey: kubernetes.io/hostname
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.17.3
+              image: harbor.basicstack.de/library/restic:0.19.1
-              # Mirrored from docker.io/restic/restic:0.17.3 (DEV-493) —
+              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
               # deterministic ingress via Harbor. Retag procedure in
               # docs/monitoring/restic-restore.md § "Tag-bump procedure".
               env:

apps/monitoring/prometheus-backup-cronjob.yaml

@@ -81,7 +81,7 @@ spec:
                   topologyKey: kubernetes.io/hostname
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.17.3
+              image: harbor.basicstack.de/library/restic:0.19.1
               env:
                 - name: AWS_ACCESS_KEY_ID
                   valueFrom:

apps/observability/patches/README.md

@@ -36,65 +36,3 @@ The kube-prometheus-stack chart is not currently tracked in ArgoCD;
 if it moves under GitOps, fold these values into the chart values as
 `prometheus-node-exporter.extraArgs` + `.extraHostVolumeMounts`
 instead of maintaining this patch.
-
-## `node-exporter-dns-config.yaml` (DEV-527)
-
-Caps the `hostNetwork` node-exporter pod's DNS at three upstream
-servers to silence Kubernetes' `DNSConfigForming` Warning event.
-Hetzner's OS publishes four systemd-resolved upstreams
-(`2a01:4ff:ff00::add:2`, `2a01:4ff:ff00::add:1`, `185.12.64.1`,
-`185.12.64.2`), and kubelet drops the fourth because Kubernetes
-pods are limited to three nameservers.
-
-Sets:
-
-1. `dnsPolicy: None` so `dnsConfig` is authoritative (with `Default`
-   or the auto-coerced `ClusterFirst→Default` on a hostNetwork pod,
-   kubelet still merges the node's resolv.conf on top and the fourth
-   nameserver keeps re-triggering the warning), and
-2. an explicit `dnsConfig` with the two Hetzner IPv6 anycast entries
-   plus the first IPv4 entry, matching the three servers kubelet was
-   already picking, plus `edns0`/`trust-ad` resolv.conf options.
-
-Apply / re-apply:
-
-```bash
-kubectl -n observability patch daemonset \
-  kube-prometheus-stack-prometheus-node-exporter \
-  --type=strategic \
-  --patch-file=apps/observability/patches/node-exporter-dns-config.yaml
-kubectl -n observability rollout status daemonset \
-  kube-prometheus-stack-prometheus-node-exporter
-```
-
-If the chart moves under GitOps, fold these values into the chart
-values as `prometheus-node-exporter.dnsPolicy` +
-`.dnsConfig` instead of maintaining this patch.
-
-## `coredns-dns-config.yaml` (DEV-527)
-
-Companion to the node-exporter patch for the k3s built-in CoreDNS
-Deployment (`kube-system/coredns`, `dnsPolicy: Default`), which also
-triggers the same `DNSConfigForming` warning on every pod restart.
-
-Because CoreDNS is a **k3s Addon** whose source manifest lives at
-`/var/lib/rancher/k3s/server/manifests/coredns.yaml` on each control
-plane node, `kubectl patch` alone is NOT durable — the k3s addon
-controller re-applies the source manifest and reverts `dnsPolicy`
-back to `Default`. This patch file is kept in the repo as the
-canonical description of the fix and can be used for a quick manual
-re-apply (until the addon controller next reconciles), but the
-authoritative fix is applied by editing the same `dnsPolicy` /
-`dnsConfig` block into the CoreDNS Deployment stanza of the k3s
-`coredns.yaml` on each CP node. See
-`infrastructure/k3s-manifests/README-DEV-527.md` for the procedure.
-
-Quick manual re-apply after a wrangler reconciliation reverted the
-change (rare):
-
-```bash
-kubectl -n kube-system patch deployment coredns \
-  --type=strategic \
-  --patch-file=apps/observability/patches/coredns-dns-config.yaml
-kubectl -n kube-system rollout status deployment coredns
-```

apps/observability/patches/coredns-dns-config.yaml

@@ -1,47 +0,0 @@
----
-# Strategic-merge patch capping the CoreDNS pod's DNS at three
-# upstream servers to silence Kubernetes' DNSConfigForming warning
-# (DEV-527).
-#
-# The Hetzner OS publishes four systemd-resolved upstreams
-# (2a01:4ff:ff00::add:2, 2a01:4ff:ff00::add:1, 185.12.64.1, 185.12.64.2),
-# and kubelet drops the fourth because Kubernetes limits pod resolv.conf
-# to three nameservers. CoreDNS uses `dnsPolicy: Default` (inherit the
-# node's resolv.conf) so it triggers the warning too.
-#
-# We keep the same three servers kubelet would have picked (the two
-# Hetzner IPv6 anycast entries plus the first IPv4 entry) so CoreDNS's
-# upstream forwarder (`forward . /etc/resolv.conf`) sees the same
-# upstreams it already uses; only the noisy warning goes away.
-# `dnsPolicy: None` is required so `dnsConfig` is authoritative —
-# otherwise kubelet still merges the node's resolv.conf and we exceed
-# the three-nameserver limit again.
-#
-# CoreDNS in k3s is an *Addon* whose source manifest lives on each
-# control-plane node at `/var/lib/rancher/k3s/server/manifests/coredns.yaml`
-# and is reconciled by the k3s addon controller. Applying this file
-# via `kubectl patch` is durable in steady state (the addon controller
-# only re-applies when the source file's checksum changes), but if the
-# source file is ever regenerated (k3s upgrade, disaster recovery) the
-# patch will be reverted. To keep the fix on regeneration, mirror the
-# same `dnsPolicy` / `dnsConfig` block into the Deployment stanza of
-# each CP node's `coredns.yaml` — see `docs/DEV-527-coredns-dns-config.md`.
-#
-# Apply / re-apply with:
-#   kubectl -n kube-system patch deployment coredns \
-#     --type=strategic \
-#     --patch-file=apps/observability/patches/coredns-dns-config.yaml
-spec:
-  template:
-    spec:
-      dnsPolicy: None
-      dnsConfig:
-        nameservers:
-          - 2a01:4ff:ff00::add:2
-          - 2a01:4ff:ff00::add:1
-          - 185.12.64.1
-        searches:
-          - .
-        options:
-          - name: edns0
-          - name: trust-ad

apps/observability/patches/node-exporter-dns-config.yaml

@@ -1,48 +0,0 @@
----
-# Strategic-merge patch capping the node-exporter pod's DNS at three
-# upstream servers to silence Kubernetes' DNSConfigForming warning
-# (DEV-527).
-#
-# The Hetzner OS publishes four systemd-resolved upstreams
-# (2a01:4ff:ff00::add:2, 2a01:4ff:ff00::add:1, 185.12.64.1, 185.12.64.2),
-# and kubelet drops the fourth because Kubernetes limits pod resolv.conf
-# to three nameservers. On hostNetwork pods that inherit the node's
-# resolv.conf, this fires a per-pod `DNSConfigForming` Warning event on
-# every kubelet DNS refresh.
-#
-# We keep the same three servers kubelet would have picked (the two
-# Hetzner IPv6 anycast entries plus the first IPv4 entry) so runtime
-# behaviour is unchanged; only the noisy warning goes away. `dnsPolicy:
-# None` is required for `dnsConfig` to be authoritative — otherwise
-# kubelet still merges the node's resolv.conf on top and we would still
-# exceed the three-nameserver limit.
-#
-# node-exporter is `hostNetwork: true` and does not talk to cluster DNS,
-# so upstream-only resolution is correct.
-#
-# The kube-prometheus-stack chart is Helm-managed (release
-# `kube-prometheus-stack` in `observability`) and is NOT currently
-# tracked in ArgoCD, so a direct DaemonSet patch is the pragmatic
-# wiring path. If the chart moves under GitOps, fold these values
-# into the chart values as `prometheus-node-exporter.dnsPolicy` +
-# `.dnsConfig` instead of maintaining this patch.
-#
-# Apply / re-apply with:
-#   kubectl -n observability patch daemonset \
-#     kube-prometheus-stack-prometheus-node-exporter \
-#     --type=strategic \
-#     --patch-file=apps/observability/patches/node-exporter-dns-config.yaml
-spec:
-  template:
-    spec:
-      dnsPolicy: None
-      dnsConfig:
-        nameservers:
-          - 2a01:4ff:ff00::add:2
-          - 2a01:4ff:ff00::add:1
-          - 185.12.64.1
-        searches:
-          - .
-        options:
-          - name: edns0
-          - name: trust-ad

apps/opencloud/opencloud-deployment.yaml

@@ -42,6 +42,8 @@ metadata:
   namespace: opencloud
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: opencloud
@@ -56,7 +58,7 @@ spec:
         runAsNonRoot: true
       initContainers:
       - name: init-dirs
-        image: opencloudeu/opencloud-rolling:7.2.0
+        image: opencloudeu/opencloud-rolling:7.4.0
         command:
           - sh
           - -c
@@ -71,7 +73,7 @@ spec:
           runAsNonRoot: true
       containers:
       - name: opencloud
-        image: opencloudeu/opencloud-rolling:7.2.0
+        image: opencloudeu/opencloud-rolling:7.4.0
         command:
           - /bin/sh
           - -c

apps/pangolin/pangolin-deployment.yaml

@@ -35,7 +35,7 @@ spec:
     spec:
       initContainers:
       - name: wait-for-postgres
-        image: postgres:17.5
+        image: postgres:17.11
         command:
         - /bin/sh
         - -c
@@ -58,7 +58,7 @@ spec:
               name: pangolin-postgres-secrets
               key: postgres-db
       - name: render-config
-        image: busybox:1.37
+        image: busybox:1.38.0
         command:
         - /bin/sh
         - -c

apps/pangolin/postgres.yaml

@@ -57,7 +57,7 @@ spec:
         fsGroup: 999
       containers:
         - name: postgres
-          image: postgres:17.5
+          image: postgres:17.11
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 5432

apps/paperclip/deployment.yaml

@@ -20,7 +20,7 @@ spec:
         fsGroupChangePolicy: Always
       initContainers:
       - name: fix-permissions
-        image: busybox:1.36
+        image: busybox:1.38.0
         command:
           - sh
           - -c

apps/pocket-id/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
       - name: pocket-id
-        image: ghcr.io/pocket-id/pocket-id:v2.11.0
+        image: ghcr.io/pocket-id/pocket-id:v2.14.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 1411

apps/stalwart/stalwart-allow-cluster-ips-job.yaml

@@ -1,123 +0,0 @@
----
-# One-time Job to configure Stalwart HTTP listener to allow internal cluster IPs
-#
-# Problem: Stalwart blocks the HTTP port from Traefik's pod IP (10.244.2.227),
-#          causing 502/503 errors when accessing mail.basicstack.de
-#
-# Solution: Use kubectl exec to access Stalwart's admin API via localhost (which is allowed)
-#          and disable IP filtering for the HTTP listener, or allow the pod network CIDR
-#
-# This Job must be manually triggered after Stalwart is running:
-#   kubectl create job --from=cronjob/stalwart-allow-cluster-ips manual-fix -n stalwart
-#
-# Or apply directly:
-#   kubectl apply -f stalwart-allow-cluster-ips-job.yaml
-#   kubectl wait --for=condition=complete job/stalwart-allow-cluster-ips -n stalwart --timeout=120s
-#   kubectl logs -n stalwart job/stalwart-allow-cluster-ips
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: stalwart-config-access
-  namespace: stalwart
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: stalwart-config-access
-  namespace: stalwart
-rules:
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get", "list"]
-- apiGroups: [""]
-  resources: ["pods/exec"]
-  verbs: ["create"]
-- apiGroups: [""]
-  resources: ["secrets"]
-  verbs: ["get"]
-  resourceNames: ["stalwart-admin-credentials"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: stalwart-config-access
-  namespace: stalwart
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: stalwart-config-access
-subjects:
-- kind: ServiceAccount
-  name: stalwart-config-access
-  namespace: stalwart
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: stalwart-allow-cluster-ips
-  namespace: stalwart
-spec:
-  ttlSecondsAfterFinished: 3600  # Keep logs for 1 hour
-  backoffLimit: 3
-  template:
-    metadata:
-      labels:
-        app: stalwart-security-fix
-    spec:
-      serviceAccountName: stalwart-config-access
-      restartPolicy: OnFailure
-      containers:
-      - name: fix-security
-        image: bitnami/kubectl:latest
-        command:
-        - /bin/bash
-        - -c
-        - |
-          set -e
-          echo "=== Stalwart HTTP Listener Security Fix ==="
-          echo "Configuring Stalwart to allow internal cluster IPs for the HTTP listener"
-          echo ""
-
-          # Wait for Stalwart pod to be ready
-          echo "Waiting for stalwart-0 pod to be ready..."
-          kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=180s
-
-          # Get admin credentials
-          echo "Retrieving admin credentials..."
-          ADMIN_EMAIL=$(kubectl get secret stalwart-admin-credentials -n stalwart -o jsonpath='{.data.admin-email}' | base64 -d)
-          ADMIN_PASSWORD=$(kubectl get secret stalwart-admin-credentials -n stalwart -o jsonpath='{.data.admin-password}' | base64 -d)
-
-          echo "Admin email: $ADMIN_EMAIL"
-
-          # Use kubectl exec to access Stalwart's admin API from localhost
-          # The HTTP listener allows localhost connections even when blocking other IPs
-          echo ""
-          echo "Accessing Stalwart admin API via kubectl exec..."
-
-          # Test API access first
-          echo "Testing API connectivity..."
-          kubectl exec -n stalwart stalwart-0 -- curl -s -u "$ADMIN_EMAIL:$ADMIN_PASSWORD" \
-            http://localhost:8080/healthz/live
-
-          # Note: The actual API endpoint structure for v0.16.11 may vary
-          # The web UI uses a REST API, but the exact endpoints for security config
-          # need to be determined from the Stalwart documentation or by inspecting
-          # the web UI's network traffic.
-
-          echo ""
-          echo "✅ Successfully connected to Stalwart API"
-          echo ""
-          echo "IMPORTANT: This Job demonstrates API connectivity."
-          echo "The actual security configuration change requires:"
-          echo "1. Identifying the correct API endpoint for security settings"
-          echo "2. Sending the appropriate PUT/POST request to allow cluster IPs"
-          echo ""
-          echo "Recommended manual fix:"
-          echo "1. Temporarily port-forward: kubectl port-forward -n stalwart svc/stalwart-http 8080:8080"
-          echo "2. Access https://mail.basicstack.de from your browser"
-          echo "3. Login with admin credentials"
-          echo "4. Navigate to Settings > Security"
-          echo "5. Disable IP filtering for the HTTP listener or add 10.244.0.0/16 to allowed IPs"
-
-          exit 0

apps/stalwart/stalwart-fresh-deployment.yaml

@@ -122,7 +122,7 @@ spec:
           mountPath: /var/lib/stalwart
       containers:
       - name: stalwart
-        image: stalwartlabs/stalwart:v0.16.11
+        image: stalwartlabs/stalwart:v0.16.18
         ports:
         - containerPort: 25
           name: smtp
@@ -316,7 +316,7 @@ spec:
                 topologyKey: kubernetes.io/hostname
           containers:
           - name: backup
-            image: alpine:3.19
+            image: alpine:3.24
             command:
             - /bin/sh
             - -c

docs/monitoring/restic-restore.md

@@ -70,7 +70,7 @@ spec:
   restartPolicy: Never
   containers:
     - name: restic
-      image: harbor.basicstack.de/library/restic:0.17.3  # matches CronJob image (DEV-493)
+      image: harbor.basicstack.de/library/restic:0.19.1  # matches CronJob image (DEV-493, bumped in DEV-541)
       command: ["sleep", "3600"]
       env:
         - { name: AWS_ACCESS_KEY_ID,     valueFrom: { secretKeyRef: { name: monitoring-s3-backup, key: access-key } } }
@@ -240,9 +240,9 @@ All three restic CronJobs (`backup-loki-restic`, `backup-grafana-restic`,
 keep the backup pipeline off the Docker Hub pull path and immune to
 upstream retagging:
 
-- Manifest reference: `harbor.basicstack.de/library/restic:0.17.3`
+- Manifest reference: `harbor.basicstack.de/library/restic:0.19.1`
-- Upstream: `docker.io/restic/restic:0.17.3`
+- Upstream: `docker.io/restic/restic:0.19.1`
-- Ticket: [DEV-493](/DEV/issues/DEV-493)
+- Ticket: [DEV-493](/DEV/issues/DEV-493) (bumped to 0.19.1 in [DEV-541](/DEV/issues/DEV-541))
 
 The `library` project is public, so no `imagePullSecret` is required
 on the CronJob pods.

infrastructure/k3s-manifests/README-DEV-527.md

@@ -1,104 +0,0 @@
-# k3s CoreDNS manifest override (DEV-527)
-
-## Context
-
-The k3s server bundles a stock CoreDNS Deployment and reconciles it
-via the k3s addon controller. The manifest source is
-`/var/lib/rancher/k3s/server/manifests/coredns.yaml` on each control
-plane node. Any `kubectl patch` on the `kube-system/coredns`
-Deployment that touches a field the source manifest declares is
-reverted by the addon controller.
-
-Hetzner Cloud nodes publish four systemd-resolved DNS upstreams:
-
-```
-2a01:4ff:ff00::add:2
-2a01:4ff:ff00::add:1
-185.12.64.1
-185.12.64.2
-```
-
-Kubernetes limits pod `resolv.conf` to three nameservers, so kubelet
-drops one entry and fires a `DNSConfigForming` Warning event every
-time a pod using `dnsPolicy: Default` (or a `hostNetwork` pod that
-coerces `ClusterFirst` to `Default`) is created. The stock CoreDNS
-Deployment uses `dnsPolicy: Default`, so the warning fires on every
-CoreDNS pod restart.
-
-## Fix
-
-The authoritative fix is to modify the source manifest so
-CoreDNS runs with an explicit three-nameserver `dnsConfig` and
-`dnsPolicy: None`. The three servers match what kubelet was already
-picking (both Hetzner IPv6 anycast entries plus the first IPv4
-entry), so CoreDNS's `forward . /etc/resolv.conf` upstreams are
-unchanged; only the noisy Warning event is silenced.
-
-The `coredns.yaml` file in this directory is the modified manifest
-that must live at `/var/lib/rancher/k3s/server/manifests/coredns.yaml`
-on **all three** k3s control plane nodes
-(`k3s-cp-1`, `k3s-cp-2`, `k3s-cp-3`).
-
-## Apply / re-apply procedure
-
-Run from any host with SSH access to the CPs:
-
-```bash
-TS="$(date -u +%Y%m%dT%H%M%SZ)"
-for IP in 178.105.17.239 188.245.85.199 49.13.92.162; do
-  echo "== $IP =="
-  scp infrastructure/k3s-manifests/coredns.yaml \
-    root@$IP:/tmp/coredns.yaml.new
-  ssh root@$IP "\
-    cp -a /var/lib/rancher/k3s/server/manifests/coredns.yaml \
-      /var/lib/rancher/k3s/server/manifests/coredns.yaml.bak-\$(date -u +%Y%m%dT%H%M%SZ) && \
-    install -m 0600 -o root -g root /tmp/coredns.yaml.new \
-      /var/lib/rancher/k3s/server/manifests/coredns.yaml && \
-    rm /tmp/coredns.yaml.new && \
-    sha256sum /var/lib/rancher/k3s/server/manifests/coredns.yaml"
-done
-kubectl -n kube-system rollout status deployment coredns --timeout=120s
-```
-
-The k3s addon controller detects the file's checksum change and
-re-applies the Deployment; the rolling update replaces the CoreDNS
-pod with the new spec.
-
-## Verification
-
-```bash
-kubectl get deploy -n kube-system coredns \
-  -o jsonpath='{.spec.template.spec.dnsPolicy}{"\n"}{.spec.template.spec.dnsConfig}{"\n"}'
-# Expect: dnsPolicy=None + dnsConfig with 3 nameservers.
-
-# No DNSConfigForming event on the current CoreDNS pod:
-POD=$(kubectl get pods -n kube-system -l k8s-app=kube-dns -o jsonpath='{.items[0].metadata.name}')
-kubectl get events -n kube-system \
-  --field-selector involvedObject.name=$POD,reason=DNSConfigForming
-# Expect: No resources found.
-
-# Cluster DNS still resolves internal + external:
-kubectl run dns-test --restart=Never --image=busybox:1.36 --rm=false --command -- \
-  sh -c "nslookup kubernetes.default.svc.cluster.local && nslookup forgejo.basicstack.de && echo OK"
-kubectl logs dns-test | tail -20
-kubectl delete pod dns-test
-```
-
-## Upgrade caveat
-
-When the k3s server binary is upgraded to a version that ships a new
-CoreDNS manifest (new CoreDNS image, new resource limits, new probes,
-etc.), the upgrade will **not** overwrite this file — k3s only creates
-the file if it does not exist. Instead, k3s emits an event/log noting
-that the on-disk manifest differs from the bundled default. On such an
-upgrade, diff the bundled manifest against this one, fold the upstream
-changes in, and re-apply.
-
-## Related
-
-- Companion strategic-merge patch:
-  `apps/observability/patches/coredns-dns-config.yaml`
-- Companion node-exporter patch:
-  `apps/observability/patches/node-exporter-dns-config.yaml`
-- Parent issue: `DEV-525` (Kubernetes events)
-- Sibling k3s coredns.yaml edit: `DEV-526` (clusterIP change)

infrastructure/k3s-manifests/coredns.yaml

@@ -1,232 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: coredns
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    kubernetes.io/bootstrapping: rbac-defaults
-  name: system:coredns
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  - services
-  - pods
-  - namespaces
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - discovery.k8s.io
-  resources:
-  - endpointslices
-  verbs:
-  - list
-  - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  labels:
-    kubernetes.io/bootstrapping: rbac-defaults
-  name: system:coredns
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:coredns
-subjects:
-- kind: ServiceAccount
-  name: coredns
-  namespace: kube-system
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: coredns
-  namespace: kube-system
-data:
-  Corefile: |
-    .:53 {
-        errors
-        health
-        ready
-        kubernetes cluster.local in-addr.arpa ip6.arpa {
-          pods insecure
-          fallthrough in-addr.arpa ip6.arpa
-        }
-        hosts /etc/coredns/NodeHosts {
-          ttl 60
-          reload 15s
-          fallthrough
-        }
-        prometheus :9153
-        cache 30
-        loop
-        reload
-        loadbalance
-        import /etc/coredns/custom/*.override
-        forward . /etc/resolv.conf
-    }
-    import /etc/coredns/custom/*.server
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: coredns
-  namespace: kube-system
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/name: "CoreDNS"
-spec:
-  revisionHistoryLimit: 0
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  selector:
-    matchLabels:
-      k8s-app: kube-dns
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-dns
-    spec:
-      priorityClassName: "system-cluster-critical"
-      serviceAccountName: coredns
-      tolerations:
-        - key: "CriticalAddonsOnly"
-          operator: "Exists"
-        - key: "node-role.kubernetes.io/control-plane"
-          operator: "Exists"
-          effect: "NoSchedule"
-      nodeSelector:
-        kubernetes.io/os: linux
-      topologySpreadConstraints:
-        - maxSkew: 1
-          topologyKey: kubernetes.io/hostname
-          whenUnsatisfiable: DoNotSchedule
-          labelSelector:
-            matchLabels:
-              k8s-app: kube-dns
-        - maxSkew: 1
-          topologyKey: topology.kubernetes.io/zone
-          whenUnsatisfiable: ScheduleAnyway
-          labelSelector:
-            matchLabels:
-              k8s-app: kube-dns
-      containers:
-      - name: coredns
-        image: "rancher/mirrored-coredns-coredns:1.14.4"
-        imagePullPolicy: IfNotPresent
-        resources:
-          limits:
-            memory: 170Mi
-          requests:
-            cpu: 100m
-            memory: 70Mi
-        args: [ "-conf", "/etc/coredns/Corefile" ]
-        volumeMounts:
-        - name: config-volume
-          mountPath: /etc/coredns
-          readOnly: true
-        - name: custom-config-volume
-          mountPath: /etc/coredns/custom
-          readOnly: true
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        - containerPort: 9153
-          name: metrics
-          protocol: TCP
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            add:
-            - NET_BIND_SERVICE
-            drop:
-            - all
-          readOnlyRootFilesystem: true
-        livenessProbe:
-          httpGet:
-            path: /health
-            port: 8080
-            scheme: HTTP
-          initialDelaySeconds: 60
-          periodSeconds: 10
-          timeoutSeconds: 1
-          successThreshold: 1
-          failureThreshold: 3
-        readinessProbe:
-          httpGet:
-            path: /ready
-            port: 8181
-            scheme: HTTP
-          initialDelaySeconds: 0
-          periodSeconds: 2
-          timeoutSeconds: 1
-          successThreshold: 1
-          failureThreshold: 3
-      dnsPolicy: None
-      dnsConfig:
-        nameservers:
-          - 2a01:4ff:ff00::add:2
-          - 2a01:4ff:ff00::add:1
-          - 185.12.64.1
-        searches:
-          - .
-        options:
-          - name: edns0
-          - name: trust-ad
-      volumes:
-        - name: config-volume
-          configMap:
-            name: coredns
-            items:
-            - key: Corefile
-              path: Corefile
-            - key: NodeHosts
-              path: NodeHosts
-        - name: custom-config-volume
-          configMap:
-            name: coredns-custom
-            optional: true
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: kube-dns
-  namespace: kube-system
-  annotations:
-    prometheus.io/port: "9153"
-    prometheus.io/scrape: "true"
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "CoreDNS"
-spec:
-  selector:
-    k8s-app: kube-dns
-  clusterIP: 10.96.0.10
-  clusterIPs: [10.96.0.10]
-  ports:
-  - name: dns
-    port: 53
-    protocol: UDP
-  - name: dns-tcp
-    port: 53
-    protocol: TCP
-  - name: metrics
-    port: 9153
-    protocol: TCP
-  ipFamilyPolicy: SingleStack