diff --git a/apps/observability/patches/README.md b/apps/observability/patches/README.md index 89a3c20..cf85d43 100644 --- a/apps/observability/patches/README.md +++ b/apps/observability/patches/README.md @@ -36,3 +36,65 @@ The kube-prometheus-stack chart is not currently tracked in ArgoCD; if it moves under GitOps, fold these values into the chart values as `prometheus-node-exporter.extraArgs` + `.extraHostVolumeMounts` instead of maintaining this patch. + +## `node-exporter-dns-config.yaml` (DEV-527) + +Caps the `hostNetwork` node-exporter pod's DNS at three upstream +servers to silence Kubernetes' `DNSConfigForming` Warning event. +Hetzner's OS publishes four systemd-resolved upstreams +(`2a01:4ff:ff00::add:2`, `2a01:4ff:ff00::add:1`, `185.12.64.1`, +`185.12.64.2`), and kubelet drops the fourth because Kubernetes +pods are limited to three nameservers. + +Sets: + +1. `dnsPolicy: None` so `dnsConfig` is authoritative (with `Default` + or the auto-coerced `ClusterFirst→Default` on a hostNetwork pod, + kubelet still merges the node's resolv.conf on top and the fourth + nameserver keeps re-triggering the warning), and +2. an explicit `dnsConfig` with the two Hetzner IPv6 anycast entries + plus the first IPv4 entry, matching the three servers kubelet was + already picking, plus `edns0`/`trust-ad` resolv.conf options. + +Apply / re-apply: + +```bash +kubectl -n observability patch daemonset \ + kube-prometheus-stack-prometheus-node-exporter \ + --type=strategic \ + --patch-file=apps/observability/patches/node-exporter-dns-config.yaml +kubectl -n observability rollout status daemonset \ + kube-prometheus-stack-prometheus-node-exporter +``` + +If the chart moves under GitOps, fold these values into the chart +values as `prometheus-node-exporter.dnsPolicy` + +`.dnsConfig` instead of maintaining this patch. + +## `coredns-dns-config.yaml` (DEV-527) + +Companion to the node-exporter patch for the k3s built-in CoreDNS +Deployment (`kube-system/coredns`, `dnsPolicy: Default`), which also +triggers the same `DNSConfigForming` warning on every pod restart. + +Because CoreDNS is a **k3s Addon** whose source manifest lives at +`/var/lib/rancher/k3s/server/manifests/coredns.yaml` on each control +plane node, `kubectl patch` alone is NOT durable — the k3s addon +controller re-applies the source manifest and reverts `dnsPolicy` +back to `Default`. This patch file is kept in the repo as the +canonical description of the fix and can be used for a quick manual +re-apply (until the addon controller next reconciles), but the +authoritative fix is applied by editing the same `dnsPolicy` / +`dnsConfig` block into the CoreDNS Deployment stanza of the k3s +`coredns.yaml` on each CP node. See +`infrastructure/k3s-manifests/README-DEV-527.md` for the procedure. + +Quick manual re-apply after a wrangler reconciliation reverted the +change (rare): + +```bash +kubectl -n kube-system patch deployment coredns \ + --type=strategic \ + --patch-file=apps/observability/patches/coredns-dns-config.yaml +kubectl -n kube-system rollout status deployment coredns +``` diff --git a/apps/observability/patches/coredns-dns-config.yaml b/apps/observability/patches/coredns-dns-config.yaml new file mode 100644 index 0000000..2f61a03 --- /dev/null +++ b/apps/observability/patches/coredns-dns-config.yaml @@ -0,0 +1,47 @@ +--- +# Strategic-merge patch capping the CoreDNS pod's DNS at three +# upstream servers to silence Kubernetes' DNSConfigForming warning +# (DEV-527). +# +# The Hetzner OS publishes four systemd-resolved upstreams +# (2a01:4ff:ff00::add:2, 2a01:4ff:ff00::add:1, 185.12.64.1, 185.12.64.2), +# and kubelet drops the fourth because Kubernetes limits pod resolv.conf +# to three nameservers. CoreDNS uses `dnsPolicy: Default` (inherit the +# node's resolv.conf) so it triggers the warning too. +# +# We keep the same three servers kubelet would have picked (the two +# Hetzner IPv6 anycast entries plus the first IPv4 entry) so CoreDNS's +# upstream forwarder (`forward . /etc/resolv.conf`) sees the same +# upstreams it already uses; only the noisy warning goes away. +# `dnsPolicy: None` is required so `dnsConfig` is authoritative — +# otherwise kubelet still merges the node's resolv.conf and we exceed +# the three-nameserver limit again. +# +# CoreDNS in k3s is an *Addon* whose source manifest lives on each +# control-plane node at `/var/lib/rancher/k3s/server/manifests/coredns.yaml` +# and is reconciled by the k3s addon controller. Applying this file +# via `kubectl patch` is durable in steady state (the addon controller +# only re-applies when the source file's checksum changes), but if the +# source file is ever regenerated (k3s upgrade, disaster recovery) the +# patch will be reverted. To keep the fix on regeneration, mirror the +# same `dnsPolicy` / `dnsConfig` block into the Deployment stanza of +# each CP node's `coredns.yaml` — see `docs/DEV-527-coredns-dns-config.md`. +# +# Apply / re-apply with: +# kubectl -n kube-system patch deployment coredns \ +# --type=strategic \ +# --patch-file=apps/observability/patches/coredns-dns-config.yaml +spec: + template: + spec: + dnsPolicy: None + dnsConfig: + nameservers: + - 2a01:4ff:ff00::add:2 + - 2a01:4ff:ff00::add:1 + - 185.12.64.1 + searches: + - . + options: + - name: edns0 + - name: trust-ad diff --git a/apps/observability/patches/node-exporter-dns-config.yaml b/apps/observability/patches/node-exporter-dns-config.yaml new file mode 100644 index 0000000..6160a97 --- /dev/null +++ b/apps/observability/patches/node-exporter-dns-config.yaml @@ -0,0 +1,48 @@ +--- +# Strategic-merge patch capping the node-exporter pod's DNS at three +# upstream servers to silence Kubernetes' DNSConfigForming warning +# (DEV-527). +# +# The Hetzner OS publishes four systemd-resolved upstreams +# (2a01:4ff:ff00::add:2, 2a01:4ff:ff00::add:1, 185.12.64.1, 185.12.64.2), +# and kubelet drops the fourth because Kubernetes limits pod resolv.conf +# to three nameservers. On hostNetwork pods that inherit the node's +# resolv.conf, this fires a per-pod `DNSConfigForming` Warning event on +# every kubelet DNS refresh. +# +# We keep the same three servers kubelet would have picked (the two +# Hetzner IPv6 anycast entries plus the first IPv4 entry) so runtime +# behaviour is unchanged; only the noisy warning goes away. `dnsPolicy: +# None` is required for `dnsConfig` to be authoritative — otherwise +# kubelet still merges the node's resolv.conf on top and we would still +# exceed the three-nameserver limit. +# +# node-exporter is `hostNetwork: true` and does not talk to cluster DNS, +# so upstream-only resolution is correct. +# +# The kube-prometheus-stack chart is Helm-managed (release +# `kube-prometheus-stack` in `observability`) and is NOT currently +# tracked in ArgoCD, so a direct DaemonSet patch is the pragmatic +# wiring path. If the chart moves under GitOps, fold these values +# into the chart values as `prometheus-node-exporter.dnsPolicy` + +# `.dnsConfig` instead of maintaining this patch. +# +# Apply / re-apply with: +# kubectl -n observability patch daemonset \ +# kube-prometheus-stack-prometheus-node-exporter \ +# --type=strategic \ +# --patch-file=apps/observability/patches/node-exporter-dns-config.yaml +spec: + template: + spec: + dnsPolicy: None + dnsConfig: + nameservers: + - 2a01:4ff:ff00::add:2 + - 2a01:4ff:ff00::add:1 + - 185.12.64.1 + searches: + - . + options: + - name: edns0 + - name: trust-ad diff --git a/infrastructure/k3s-manifests/README-DEV-527.md b/infrastructure/k3s-manifests/README-DEV-527.md new file mode 100644 index 0000000..2338581 --- /dev/null +++ b/infrastructure/k3s-manifests/README-DEV-527.md @@ -0,0 +1,104 @@ +# k3s CoreDNS manifest override (DEV-527) + +## Context + +The k3s server bundles a stock CoreDNS Deployment and reconciles it +via the k3s addon controller. The manifest source is +`/var/lib/rancher/k3s/server/manifests/coredns.yaml` on each control +plane node. Any `kubectl patch` on the `kube-system/coredns` +Deployment that touches a field the source manifest declares is +reverted by the addon controller. + +Hetzner Cloud nodes publish four systemd-resolved DNS upstreams: + +``` +2a01:4ff:ff00::add:2 +2a01:4ff:ff00::add:1 +185.12.64.1 +185.12.64.2 +``` + +Kubernetes limits pod `resolv.conf` to three nameservers, so kubelet +drops one entry and fires a `DNSConfigForming` Warning event every +time a pod using `dnsPolicy: Default` (or a `hostNetwork` pod that +coerces `ClusterFirst` to `Default`) is created. The stock CoreDNS +Deployment uses `dnsPolicy: Default`, so the warning fires on every +CoreDNS pod restart. + +## Fix + +The authoritative fix is to modify the source manifest so +CoreDNS runs with an explicit three-nameserver `dnsConfig` and +`dnsPolicy: None`. The three servers match what kubelet was already +picking (both Hetzner IPv6 anycast entries plus the first IPv4 +entry), so CoreDNS's `forward . /etc/resolv.conf` upstreams are +unchanged; only the noisy Warning event is silenced. + +The `coredns.yaml` file in this directory is the modified manifest +that must live at `/var/lib/rancher/k3s/server/manifests/coredns.yaml` +on **all three** k3s control plane nodes +(`k3s-cp-1`, `k3s-cp-2`, `k3s-cp-3`). + +## Apply / re-apply procedure + +Run from any host with SSH access to the CPs: + +```bash +TS="$(date -u +%Y%m%dT%H%M%SZ)" +for IP in 178.105.17.239 188.245.85.199 49.13.92.162; do + echo "== $IP ==" + scp infrastructure/k3s-manifests/coredns.yaml \ + root@$IP:/tmp/coredns.yaml.new + ssh root@$IP "\ + cp -a /var/lib/rancher/k3s/server/manifests/coredns.yaml \ + /var/lib/rancher/k3s/server/manifests/coredns.yaml.bak-\$(date -u +%Y%m%dT%H%M%SZ) && \ + install -m 0600 -o root -g root /tmp/coredns.yaml.new \ + /var/lib/rancher/k3s/server/manifests/coredns.yaml && \ + rm /tmp/coredns.yaml.new && \ + sha256sum /var/lib/rancher/k3s/server/manifests/coredns.yaml" +done +kubectl -n kube-system rollout status deployment coredns --timeout=120s +``` + +The k3s addon controller detects the file's checksum change and +re-applies the Deployment; the rolling update replaces the CoreDNS +pod with the new spec. + +## Verification + +```bash +kubectl get deploy -n kube-system coredns \ + -o jsonpath='{.spec.template.spec.dnsPolicy}{"\n"}{.spec.template.spec.dnsConfig}{"\n"}' +# Expect: dnsPolicy=None + dnsConfig with 3 nameservers. + +# No DNSConfigForming event on the current CoreDNS pod: +POD=$(kubectl get pods -n kube-system -l k8s-app=kube-dns -o jsonpath='{.items[0].metadata.name}') +kubectl get events -n kube-system \ + --field-selector involvedObject.name=$POD,reason=DNSConfigForming +# Expect: No resources found. + +# Cluster DNS still resolves internal + external: +kubectl run dns-test --restart=Never --image=busybox:1.36 --rm=false --command -- \ + sh -c "nslookup kubernetes.default.svc.cluster.local && nslookup forgejo.basicstack.de && echo OK" +kubectl logs dns-test | tail -20 +kubectl delete pod dns-test +``` + +## Upgrade caveat + +When the k3s server binary is upgraded to a version that ships a new +CoreDNS manifest (new CoreDNS image, new resource limits, new probes, +etc.), the upgrade will **not** overwrite this file — k3s only creates +the file if it does not exist. Instead, k3s emits an event/log noting +that the on-disk manifest differs from the bundled default. On such an +upgrade, diff the bundled manifest against this one, fold the upstream +changes in, and re-apply. + +## Related + +- Companion strategic-merge patch: + `apps/observability/patches/coredns-dns-config.yaml` +- Companion node-exporter patch: + `apps/observability/patches/node-exporter-dns-config.yaml` +- Parent issue: `DEV-525` (Kubernetes events) +- Sibling k3s coredns.yaml edit: `DEV-526` (clusterIP change) diff --git a/infrastructure/k3s-manifests/coredns.yaml b/infrastructure/k3s-manifests/coredns.yaml new file mode 100644 index 0000000..107a56b --- /dev/null +++ b/infrastructure/k3s-manifests/coredns.yaml @@ -0,0 +1,232 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system +data: + Corefile: | + .:53 { + errors + health + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + hosts /etc/coredns/NodeHosts { + ttl 60 + reload 15s + fallthrough + } + prometheus :9153 + cache 30 + loop + reload + loadbalance + import /etc/coredns/custom/*.override + forward . /etc/resolv.conf + } + import /etc/coredns/custom/*.server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" +spec: + revisionHistoryLimit: 0 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + spec: + priorityClassName: "system-cluster-critical" + serviceAccountName: coredns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: + kubernetes.io/os: linux + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + k8s-app: kube-dns + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + k8s-app: kube-dns + containers: + - name: coredns + image: "rancher/mirrored-coredns-coredns:1.14.4" + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + - name: custom-config-volume + mountPath: /etc/coredns/custom + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 2 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + dnsPolicy: None + dnsConfig: + nameservers: + - 2a01:4ff:ff00::add:2 + - 2a01:4ff:ff00::add:1 + - 185.12.64.1 + searches: + - . + options: + - name: edns0 + - name: trust-ad + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + - key: NodeHosts + path: NodeHosts + - name: custom-config-volume + configMap: + name: coredns-custom + optional: true +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.96.0.10 + clusterIPs: [10.96.0.10] + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + ipFamilyPolicy: SingleStack