diff --git a/infrastructure/scripts/os-update/README.md b/infrastructure/scripts/os-update/README.md old mode 100644 new mode 100755 index 0aaf95a..51bc742 --- a/infrastructure/scripts/os-update/README.md +++ b/infrastructure/scripts/os-update/README.md @@ -23,6 +23,7 @@ Scripts that implement the weekly rolling Ubuntu OS-update procedure. ## Guardrails the scripts enforce - Preflight cluster health failure → refuse to start. +- SSH-drift preflight (DEV-587): before draining anything, verify the operator can SSH to every planned node and that cp-1's pubkey is present in each target's `/root/.ssh/authorized_keys`. Missing pubkeys are auto-appended via the operator's own SSH; nodes that reject the operator entirely fail the whole cycle with the exact repair command. Preflight report is written to `/tmp/os-update-/ssh-preflight.txt`. Escape hatch: `--skip-ssh-preflight` for bootstrap. - Drain with a PDB conflict → uncordon and mark the node `SKIPPED_DRAIN_FAILED`, never `--force`. - Node reboot fails to come back within timeout → hard stop, escalate. Do NOT rebuild the node or touch k3s config. - kubelet doesn't return `Ready` → hard stop, escalate. Do NOT touch k3s config. diff --git a/infrastructure/scripts/os-update/os-update.sh b/infrastructure/scripts/os-update/os-update.sh index 21b4e02..fbd7915 100755 --- a/infrastructure/scripts/os-update/os-update.sh +++ b/infrastructure/scripts/os-update/os-update.sh @@ -7,18 +7,23 @@ # 3. compute node order: # workers (non-Stalwart first, Stalwart-hosting fsn1 workers last) # then control-plane nodes (etcd followers first, leader last). +# 3b. SSH-key drift preflight: verify operator can reach every planned node +# and that cp-1's pubkey is present in root's authorized_keys (auto-heal +# by appending if missing; hard-fail if unreachable). See DEV-587. # 4. call update-node.sh for each worker and update-cp-node.sh for each CP, # halting on any failure # 5. finalization: health snapshot + apt history digest # # Usage: # os-update.sh [--dry-run] [--start-from ] [--only ] -# [--skip-cps] [--only-cps] +# [--skip-cps] [--only-cps] [--skip-ssh-preflight] # # Flags: -# --skip-cps run workers only (legacy behavior, useful for narrow patches). -# --only-cps skip workers, only walk the CP phase. -# --only update a single node (worker or CP, routed automatically). +# --skip-cps run workers only (legacy behavior, useful for narrow patches). +# --only-cps skip workers, only walk the CP phase. +# --only update a single node (worker or CP, routed automatically). +# --skip-ssh-preflight skip the SSH-drift preflight (escape hatch for bootstrap +# scenarios where cp-1's pubkey is intentionally absent). # # Environment: # CONTROL_PLANE_HOST (default 178.105.17.239) @@ -37,16 +42,18 @@ START_FROM="" ONLY="" SKIP_CPS=0 ONLY_CPS=0 +SKIP_SSH_PREFLIGHT="${SKIP_SSH_PREFLIGHT:-0}" while [ $# -gt 0 ]; do case "$1" in - --dry-run) DRY_RUN=1; shift ;; - --start-from) START_FROM="$2"; shift 2 ;; - --only) ONLY="$2"; shift 2 ;; - --skip-cps) SKIP_CPS=1; shift ;; - --only-cps) ONLY_CPS=1; shift ;; - -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; - *) echo "unknown arg: $1" >&2; exit 2 ;; + --dry-run) DRY_RUN=1; shift ;; + --start-from) START_FROM="$2"; shift 2 ;; + --only) ONLY="$2"; shift 2 ;; + --skip-cps) SKIP_CPS=1; shift ;; + --only-cps) ONLY_CPS=1; shift ;; + --skip-ssh-preflight) SKIP_SSH_PREFLIGHT=1; shift ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; esac done @@ -58,6 +65,27 @@ mkdir -p "$LOG_DIR" log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_DIR/main.log"; } die() { log "FATAL: $*"; exit 1; } +# node_ssh_endpoint +# Prints the ssh argv fragment (may include `-J bastion`) used to reach a node. +# Private-IP workers go through cp-1 via ProxyJump; every other node has a +# public IP. Returns non-zero for an unknown node. Callers pass the output +# unquoted so `-J` is treated as its own arg. +node_ssh_endpoint() { + case "$1" in + k3s-cp-1) echo "root@178.105.17.239" ;; + k3s-cp-2) echo "root@188.245.85.199" ;; + k3s-cp-3) echo "root@49.13.92.162" ;; + k3s-worker-1) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;; + k3s-worker-2) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;; + k3s-worker-3) echo "root@167.233.121.121" ;; + k3s-worker-4) echo "root@128.140.3.80" ;; + k3s-worker-5) echo "root@167.233.192.86" ;; + k3s-worker-6) echo "root@49.12.223.133" ;; + k3s-update-runner) echo "root@167.233.79.65" ;; + *) return 1 ;; + esac +} + log "=== os-update.sh cycle $STAMP ===" log "log dir: $LOG_DIR" @@ -221,6 +249,117 @@ if [ "$DRY_RUN" -eq 1 ]; then exit 0 fi +# --- 3c. SSH-key drift preflight --------------------------------------------- +# DEV-587 / DEV-586 (2026-09-27) — some k3s nodes drift to lose cp-1's SSH +# pubkey from /root/.ssh/authorized_keys (probable Hetzner cloud-init resync +# from the HC SSH-keys API; the operator key is NOT in that keyset). When this +# happens mid-cycle the whole run aborts partway through drain / peer-probe. +# This preflight proves the operator can SSH to every planned node and that +# cp-1's pubkey is present in root's authorized_keys — auto-heals drift via +# the operator's own SSH access, and hard-fails BEFORE anything is drained if +# a node is genuinely unreachable. +ssh_drift_preflight() { + local ok=() repaired=() problem=() + + log "[preflight] SSH-key drift check for planned nodes" + + local cp1_pubkey cp1_key_body + if ! cp1_pubkey=$(ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \ + root@"$CONTROL_PLANE_HOST" 'cat /root/.ssh/id_rsa.pub' 2>/dev/null); then + die "SSH-drift preflight: cannot read cp-1 pubkey from $CONTROL_PLANE_HOST — fix operator->cp-1 SSH first" + fi + cp1_pubkey=$(printf '%s' "$cp1_pubkey" | tr -d '\r\n') + cp1_key_body=$(printf '%s' "$cp1_pubkey" | awk '{print $2}') + [ -n "$cp1_key_body" ] || die "SSH-drift preflight: could not parse cp-1 pubkey (got: $cp1_pubkey)" + log " cp-1 pubkey OK (${cp1_key_body:0:32}...)" + + local plan_nodes=() + for n in "${WORKER_ORDER[@]:-}" "${CP_ORDER[@]:-}"; do + [ -z "$n" ] && continue + plan_nodes+=("$n") + done + if [ ${#plan_nodes[@]} -eq 0 ]; then + log " (no planned nodes — nothing to preflight)" + return 0 + fi + + for n in "${plan_nodes[@]}"; do + local endpoint + if ! endpoint=$(node_ssh_endpoint "$n"); then + log " $n: unknown ssh endpoint — refusing" + problem+=("$n (no ssh mapping in node_ssh_endpoint)") + continue + fi + + # 1. reachability from the operator machine. + # shellcheck disable=SC2086 # $endpoint may include `-J bastion`; deliberate word-split. + if ! ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \ + $endpoint true 2>/dev/null; then + problem+=("$n ($endpoint - operator SSH refused)") + continue + fi + + # 2. cp-1 is the source of the key; drift check would be pointless on it. + if [ "$n" = "k3s-cp-1" ]; then + ok+=("$n") + continue + fi + + # 3. cp-1 pubkey present? + # shellcheck disable=SC2086 + if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \ + $endpoint "grep -qF '$cp1_key_body' /root/.ssh/authorized_keys" 2>/dev/null; then + ok+=("$n") + continue + fi + + # 4. auto-heal — append cp-1's pubkey via the (already verified) operator SSH. + log " $n: cp-1 pubkey MISSING from /root/.ssh/authorized_keys — appending" + # shellcheck disable=SC2086 + if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \ + $endpoint "install -d -m 700 /root/.ssh && printf '%s\n' '$cp1_pubkey' >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys" \ + 2>/dev/null; then + # shellcheck disable=SC2086 + if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \ + $endpoint "grep -qF '$cp1_key_body' /root/.ssh/authorized_keys" 2>/dev/null; then + repaired+=("$n") + else + problem+=("$n (auto-heal wrote but grep-verify failed)") + fi + else + problem+=("$n ($endpoint - auto-heal write refused)") + fi + done + + log "[preflight] SSH-drift summary:" + log " ok (${#ok[@]}): ${ok[*]:-none}" + log " repaired (${#repaired[@]}): ${repaired[*]:-none}" + log " problem (${#problem[@]}): ${problem[*]:-none}" + + { + echo "# SSH-drift preflight — $(date -u +%FT%TZ)" + echo "# cp-1 pubkey: $cp1_pubkey" + printf 'ok\t%s\n' "${ok[*]:-}" + printf 'repaired\t%s\n' "${repaired[*]:-}" + printf 'problem\t%s\n' "${problem[*]:-}" + } > "$LOG_DIR/ssh-preflight.txt" + + if [ ${#problem[@]} -gt 0 ]; then + log "SSH-drift preflight FAILED for: ${problem[*]}" + log " cp-1 pubkey to install:" + log " $cp1_pubkey" + log " From any machine that CAN reach the failing node, run:" + log " ssh root@ \"grep -qF '$cp1_key_body' /root/.ssh/authorized_keys || echo '$cp1_pubkey' >> /root/.ssh/authorized_keys\"" + die "SSH-drift preflight failed — refuse to start rolling update" + fi +} + +if [ "$SKIP_SSH_PREFLIGHT" -eq 1 ]; then + log "[preflight] SSH-drift check SKIPPED (--skip-ssh-preflight)" +else + ssh_drift_preflight +fi + # --- 4. worker loop ---------------------------------------------------------- updated=() skipped=() @@ -298,20 +437,11 @@ log "[final] apt history digest" { for n in "${updated[@]}"; do echo "=== $n ===" - # resolve ssh target via a mini-eval of node_ssh_target-equivalent - case "$n" in - k3s-cp-1) t="root@178.105.17.239" ;; - k3s-cp-2) t="root@188.245.85.199" ;; - k3s-cp-3) t="root@49.13.92.162" ;; - k3s-worker-1) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;; - k3s-worker-2) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;; - k3s-worker-3) t="root@167.233.121.121" ;; - k3s-worker-4) t="root@128.140.3.80" ;; - k3s-worker-5) t="root@167.233.192.86" ;; - k3s-worker-6) t="root@49.12.223.133" ;; - k3s-update-runner) t="root@167.233.79.65" ;; - *) echo " (unknown ssh target)"; continue ;; - esac + if ! t=$(node_ssh_endpoint "$n"); then + echo " (unknown ssh target)" + continue + fi + # shellcheck disable=SC2086 # $t may include `-J bastion`; deliberate word-split. ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new $t \ 'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -40' 2>/dev/null \ || echo " (could not read apt history)"