From 19af6918f09eee0e73e9da56b4301e5bd5f2d366 Mon Sep 17 00:00:00 2001 From: CTO Agent Date: Sun, 16 Aug 2026 20:22:43 +0000 Subject: [PATCH] feat(os-update): reconcile Docker + label after node updates (DEV-499) The weekly rolling OS-update cycle has been silently stripping docker.io from worker nodes (DEV-498), breaking the Forgejo runner whose hostPath mount for /var/run/docker.sock requires the socket to exist. Fix in two layers: - Defensive pin: update-node.sh now runs `apt-mark manual docker.io` in the apt phase (whenever it is installed) so `apt-get autoremove --purge` cannot silently drop it during subsequent upgrades. - Post-reboot reconciliation: new `ensure-node-docker.sh` installs docker.io if missing, enables + starts the systemd unit, waits for /var/run/docker.sock, and re-applies the `basicstack.de/docker=true` label. Wired into update-node.sh between kubelet-Ready and uncordon. No-op on nodes without the label (safe for cp-1 and the update runner). Verified idempotent against all 5 labeled workers; `apt-mark manual docker.io` now set on every worker (survived across reboots by design). Co-Authored-By: Paperclip --- infrastructure/OS_UPDATE_PROCEDURE.md | 10 ++ infrastructure/scripts/os-update/README.md | 6 +- .../scripts/os-update/ensure-node-docker.sh | 112 ++++++++++++++++++ .../scripts/os-update/update-node.sh | 16 ++- 4 files changed, 142 insertions(+), 2 deletions(-) create mode 100755 infrastructure/scripts/os-update/ensure-node-docker.sh diff --git a/infrastructure/OS_UPDATE_PROCEDURE.md b/infrastructure/OS_UPDATE_PROCEDURE.md index fd351d2..ffb3809 100644 --- a/infrastructure/OS_UPDATE_PROCEDURE.md +++ b/infrastructure/OS_UPDATE_PROCEDURE.md @@ -205,6 +205,16 @@ kubectl get node "$NODE" If Ready never returns True: escalate. Do NOT change k3s config. +### 5b. Reconcile Docker on labeled nodes (before uncordon) + +For nodes carrying the `basicstack.de/docker=true` label (currently the workers that run the Forgejo runner), ensure `docker.io` is present + running before scheduling resumes. Without this the runner pod comes back `ContainerCreating` because `hostPath` requires the docker socket to exist (see [DEV-498](/DEV/issues/DEV-498) and [DEV-499](/DEV/issues/DEV-499)). + +```bash +scripts/os-update/ensure-node-docker.sh "$NODE" +``` + +This step is a no-op on nodes without the label. `update-node.sh` runs it automatically between step 5 and step 6; the script is also safe to run ad-hoc after any manual OS operation. As a defensive extra measure, the apt phase (step 3) now runs `apt-mark manual docker.io` on any node where it is installed, so `apt-get autoremove --purge` cannot silently strip it. + ### 6. Uncordon ```bash diff --git a/infrastructure/scripts/os-update/README.md b/infrastructure/scripts/os-update/README.md index a9e3fb5..0aaf95a 100644 --- a/infrastructure/scripts/os-update/README.md +++ b/infrastructure/scripts/os-update/README.md @@ -9,7 +9,8 @@ Scripts that implement the weekly rolling Ubuntu OS-update procedure. | Script | Purpose | |--------|---------| | `cluster-health.sh` | Non-zero if any node is not Ready, any pod is not Running/Ready, or any Deployment/StatefulSet is below its desired replica count. Used at preflight and after each node. | -| `update-node.sh ` | Drain, apt-update, reboot-if-required, wait for Ready, uncordon, post-node health check. Retriable per-node. | +| `update-node.sh ` | Drain, apt-update, reboot-if-required, wait for Ready, ensure Docker on labeled nodes, uncordon, post-node health check. Retriable per-node. | +| `ensure-node-docker.sh ` | Idempotent: on nodes labeled `basicstack.de/docker=true`, install docker.io if missing, `apt-mark manual`, enable+start the docker service, wait for `/var/run/docker.sock`, re-apply the label. No-op on nodes without the label. Called from `update-node.sh` between "kubelet Ready" and "uncordon"; also runnable ad-hoc for recovery. | | `os-update.sh` | Full cycle runner: preflight → etcd snapshot → ordered per-node loop → finalization + apt history digest. | ## Order of operations (encoded in `os-update.sh`) @@ -42,6 +43,9 @@ Scripts that implement the weekly rolling Ubuntu OS-update procedure. # Restart a partial cycle from a specific node onward. ./os-update.sh --start-from k3s-worker-4 + +# Ad-hoc: reconcile Docker on a single node (e.g. after emergency ops). +./ensure-node-docker.sh k3s-worker-3 ``` Logs land in `/tmp/os-update-/` on the machine that ran the cycle. diff --git a/infrastructure/scripts/os-update/ensure-node-docker.sh b/infrastructure/scripts/os-update/ensure-node-docker.sh new file mode 100755 index 0000000..cf3c7ab --- /dev/null +++ b/infrastructure/scripts/os-update/ensure-node-docker.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# ensure-node-docker.sh — ensure docker.io is installed, enabled, and running on +# a node that carries (or should carry) the `basicstack.de/docker=true` label. +# +# Runs from the operator machine (or the control plane); needs kubectl and +# ssh access to root@. +# +# Behavior: +# - If the node has label `basicstack.de/docker=true`, ensure docker.io is +# installed, marked `apt-mark manual`, systemd `docker` is enabled+active, +# and /var/run/docker.sock exists. +# - Re-apply the label (idempotent) so that returning nodes always end the +# step in a known state. +# - If the node does NOT carry the label, this is a no-op — do NOT install +# docker on nodes that were not designated to run the Forgejo runner. +# +# Usage: +# ensure-node-docker.sh +# +# Exit codes: +# 0 — node is either not designated (no label) or Docker is confirmed healthy +# 1 — hard failure: label present but docker could not be brought up +# +# Motivated by DEV-499: the weekly rolling OS update was purging docker.io from +# workers, which broke the Forgejo runner (DEV-498). Making Docker part of the +# post-reboot reconciliation removes the manual "apt-get install docker.io && +# systemctl enable --now docker && kubectl label node" step. +set -euo pipefail + +NODE="${1:-}" +if [ -z "$NODE" ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}" +SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new}" +DOCKER_LABEL_KEY="basicstack.de/docker" + +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] ensure-docker/$NODE: $*"; } +die() { log "FATAL: $*"; exit 1; } + +# node -> ssh target. Keep in sync with update-node.sh::node_ssh_target. +node_ssh_target() { + case "$1" in + k3s-cp-1) echo "root@178.105.17.239" ;; + k3s-worker-1) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;; + k3s-worker-2) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;; + k3s-worker-3) echo "root@167.233.121.121" ;; + k3s-worker-4) echo "root@128.140.3.80" ;; + k3s-worker-5) echo "root@167.233.192.86" ;; + k3s-update-runner) echo "root@167.233.79.65" ;; + *) die "unknown node $1 — update node_ssh_target() in $0" ;; + esac +} + +# --- 1. is this node designated to run Docker? -------------------------------- +kubectl get node "$NODE" >/dev/null || die "node $NODE not found in cluster" +LABEL_VAL=$(kubectl get node "$NODE" \ + -o jsonpath="{.metadata.labels.${DOCKER_LABEL_KEY//./\\.}}" 2>/dev/null || echo "") + +if [ "$LABEL_VAL" != "true" ]; then + log "no ${DOCKER_LABEL_KEY}=true label — skipping Docker reconciliation" + exit 0 +fi + +SSH_TARGET=$(node_ssh_target "$NODE") +log "label ${DOCKER_LABEL_KEY}=true present — reconciling docker.io via $SSH_TARGET" + +# --- 2. ensure docker.io on the node ----------------------------------------- +REMOTE=$(cat <<'REMOTE' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +APT_OPTS='-y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold' + +# Recover from any half-finished dpkg state before touching apt. +dpkg --configure -a >/dev/null 2>&1 || true + +if ! dpkg -s docker.io >/dev/null 2>&1; then + echo "docker.io not installed — installing" + apt-get update + apt-get $APT_OPTS install docker.io +else + echo "docker.io already installed" +fi + +# Keep docker.io out of the reach of apt-get autoremove --purge, which is what +# the weekly OS-update cycle runs. Idempotent. +apt-mark manual docker.io >/dev/null + +systemctl enable --now docker + +# Wait for the docker socket to appear so the forgejo-runner pod can bind it. +for _ in $(seq 1 30); do + if [ -S /var/run/docker.sock ]; then break; fi + sleep 1 +done +if [ ! -S /var/run/docker.sock ]; then + echo "docker.sock missing after enable" >&2 + systemctl status docker --no-pager | tail -20 >&2 + exit 1 +fi + +echo "docker OK: $(docker version --format '{{.Server.Version}}')" +REMOTE +) + +ssh $SSH_OPTS $SSH_TARGET "bash -s" <<< "$REMOTE" | sed 's/^/ /' + +# --- 3. re-apply the label (idempotent, in case someone stripped it) --------- +kubectl label node "$NODE" "${DOCKER_LABEL_KEY}=true" --overwrite >/dev/null +log "docker + label reconciled OK" diff --git a/infrastructure/scripts/os-update/update-node.sh b/infrastructure/scripts/os-update/update-node.sh index 99580f1..c8837f7 100755 --- a/infrastructure/scripts/os-update/update-node.sh +++ b/infrastructure/scripts/os-update/update-node.sh @@ -101,6 +101,13 @@ fi apt-get update +# Before any upgrade/autoremove pass, pin docker.io as manually-installed on +# nodes where it is present. Without this, apt autoremove --purge has, in the +# past, taken docker.io out from under the Forgejo runner (DEV-498/DEV-499). +if dpkg -s docker.io >/dev/null 2>&1; then + apt-mark manual docker.io >/dev/null +fi + # Try upgrade; on broken deps, one attempt at apt-get -f install then retry. if ! apt-get $APT_OPTS upgrade; then echo "upgrade failed, attempting apt-get -f install" @@ -167,6 +174,14 @@ done [ "$READY" = "True" ] || die "kubelet on $NODE never returned Ready — escalate (do NOT change k3s config)" log " Ready=True" +# --- 5b. ensure Docker on designated nodes ------------------------------------ +# Do this BEFORE uncordoning so the runner pod's first scheduling attempt +# succeeds instead of racing through ContainerCreating. Idempotent no-op on +# nodes that are not labeled basicstack.de/docker=true. (DEV-499) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +log "[5b/7] ensure docker.io on nodes labeled basicstack.de/docker=true" +"$SCRIPT_DIR/ensure-node-docker.sh" "$NODE" + # --- 6. uncordon -------------------------------------------------------------- log "[6/7] uncordon $NODE" kubectl uncordon "$NODE" @@ -175,7 +190,6 @@ kubectl uncordon "$NODE" log "[7/7] post-node settle (${POST_UNCORDON_WAIT_SECONDS}s) + health check" sleep "$POST_UNCORDON_WAIT_SECONDS" -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" if RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh"; then log "=== $NODE update: OK ===" else