Compare commits
No commits in common. "5b627f9e7330c18225d006d0eaf189b1f67e9b70" and "a71c5d9a6328fbf6b246fa8995ce2efa24bbd7c2" have entirely different histories.
5b627f9e73
...
a71c5d9a63
4 changed files with 31 additions and 204 deletions
|
|
@ -84,12 +84,6 @@ Environment variables the scripts expect:
|
|||
- Drain timeout: `DRAIN_TIMEOUT_SECONDS` — default `600`
|
||||
- Reboot wait: `REBOOT_MAX_WAIT_SECONDS` — default `600`
|
||||
- Post-uncordon settle: `POST_UNCORDON_WAIT_SECONDS` — default `180`
|
||||
- Post-uncordon bounded health retry (DEV-595):
|
||||
- `HEALTH_MAX_RETRIES` — default `10` (extra attempts after the first)
|
||||
- `HEALTH_RETRY_INTERVAL_SECONDS` — default `30`
|
||||
- Short-circuits immediately on persistent pod errors
|
||||
(`CrashLoopBackOff`, `ImagePullBackOff`, `ErrImagePull`,
|
||||
`CreateContainerConfigError`, `InvalidImageName`).
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -23,11 +23,10 @@ Scripts that implement the weekly rolling Ubuntu OS-update procedure.
|
|||
## Guardrails the scripts enforce
|
||||
|
||||
- Preflight cluster health failure → refuse to start.
|
||||
- SSH-drift preflight (DEV-587): before draining anything, verify the operator can SSH to every planned node and that cp-1's pubkey is present in each target's `/root/.ssh/authorized_keys`. Missing pubkeys are auto-appended via the operator's own SSH; nodes that reject the operator entirely fail the whole cycle with the exact repair command. Preflight report is written to `/tmp/os-update-<UTC-ts>/ssh-preflight.txt`. Escape hatch: `--skip-ssh-preflight` for bootstrap.
|
||||
- Drain with a PDB conflict → uncordon and mark the node `SKIPPED_DRAIN_FAILED`, never `--force`.
|
||||
- Node reboot fails to come back within timeout → hard stop, escalate. Do NOT rebuild the node or touch k3s config.
|
||||
- kubelet doesn't return `Ready` → hard stop, escalate. Do NOT touch k3s config.
|
||||
- Post-node cluster health check red → first retry up to `HEALTH_MAX_RETRIES` (default 10) more times at `HEALTH_RETRY_INTERVAL_SECONDS` (default 30s) to let transient reinit windows settle (DEV-595). Short-circuit halt immediately on any `CrashLoopBackOff`/`ImagePullBackOff`/`ErrImagePull`/`CreateContainerConfigError`/`InvalidImageName`. Still red after all attempts → hard stop, do not proceed to the next node.
|
||||
- Post-node cluster health check red → hard stop, do not proceed to the next node.
|
||||
- Absolute rule: **no k3s config, no manifests, no PVs, no service files touched** — apt/dpkg only.
|
||||
|
||||
## Typical invocations
|
||||
|
|
|
|||
|
|
@ -7,23 +7,18 @@
|
|||
# 3. compute node order:
|
||||
# workers (non-Stalwart first, Stalwart-hosting fsn1 workers last)
|
||||
# then control-plane nodes (etcd followers first, leader last).
|
||||
# 3b. SSH-key drift preflight: verify operator can reach every planned node
|
||||
# and that cp-1's pubkey is present in root's authorized_keys (auto-heal
|
||||
# by appending if missing; hard-fail if unreachable). See DEV-587.
|
||||
# 4. call update-node.sh for each worker and update-cp-node.sh for each CP,
|
||||
# halting on any failure
|
||||
# 5. finalization: health snapshot + apt history digest
|
||||
#
|
||||
# Usage:
|
||||
# os-update.sh [--dry-run] [--start-from <node>] [--only <node>]
|
||||
# [--skip-cps] [--only-cps] [--skip-ssh-preflight]
|
||||
# [--skip-cps] [--only-cps]
|
||||
#
|
||||
# Flags:
|
||||
# --skip-cps run workers only (legacy behavior, useful for narrow patches).
|
||||
# --only-cps skip workers, only walk the CP phase.
|
||||
# --only <n> update a single node (worker or CP, routed automatically).
|
||||
# --skip-ssh-preflight skip the SSH-drift preflight (escape hatch for bootstrap
|
||||
# scenarios where cp-1's pubkey is intentionally absent).
|
||||
# --skip-cps run workers only (legacy behavior, useful for narrow patches).
|
||||
# --only-cps skip workers, only walk the CP phase.
|
||||
# --only <n> update a single node (worker or CP, routed automatically).
|
||||
#
|
||||
# Environment:
|
||||
# CONTROL_PLANE_HOST (default 178.105.17.239)
|
||||
|
|
@ -42,18 +37,16 @@ START_FROM=""
|
|||
ONLY=""
|
||||
SKIP_CPS=0
|
||||
ONLY_CPS=0
|
||||
SKIP_SSH_PREFLIGHT="${SKIP_SSH_PREFLIGHT:-0}"
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--dry-run) DRY_RUN=1; shift ;;
|
||||
--start-from) START_FROM="$2"; shift 2 ;;
|
||||
--only) ONLY="$2"; shift 2 ;;
|
||||
--skip-cps) SKIP_CPS=1; shift ;;
|
||||
--only-cps) ONLY_CPS=1; shift ;;
|
||||
--skip-ssh-preflight) SKIP_SSH_PREFLIGHT=1; shift ;;
|
||||
-h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
--dry-run) DRY_RUN=1; shift ;;
|
||||
--start-from) START_FROM="$2"; shift 2 ;;
|
||||
--only) ONLY="$2"; shift 2 ;;
|
||||
--skip-cps) SKIP_CPS=1; shift ;;
|
||||
--only-cps) ONLY_CPS=1; shift ;;
|
||||
-h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
|
@ -65,27 +58,6 @@ mkdir -p "$LOG_DIR"
|
|||
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_DIR/main.log"; }
|
||||
die() { log "FATAL: $*"; exit 1; }
|
||||
|
||||
# node_ssh_endpoint <node-name>
|
||||
# Prints the ssh argv fragment (may include `-J bastion`) used to reach a node.
|
||||
# Private-IP workers go through cp-1 via ProxyJump; every other node has a
|
||||
# public IP. Returns non-zero for an unknown node. Callers pass the output
|
||||
# unquoted so `-J` is treated as its own arg.
|
||||
node_ssh_endpoint() {
|
||||
case "$1" in
|
||||
k3s-cp-1) echo "root@178.105.17.239" ;;
|
||||
k3s-cp-2) echo "root@188.245.85.199" ;;
|
||||
k3s-cp-3) echo "root@49.13.92.162" ;;
|
||||
k3s-worker-1) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
|
||||
k3s-worker-2) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
|
||||
k3s-worker-3) echo "root@167.233.121.121" ;;
|
||||
k3s-worker-4) echo "root@128.140.3.80" ;;
|
||||
k3s-worker-5) echo "root@167.233.192.86" ;;
|
||||
k3s-worker-6) echo "root@49.12.223.133" ;;
|
||||
k3s-update-runner) echo "root@167.233.79.65" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
log "=== os-update.sh cycle $STAMP ==="
|
||||
log "log dir: $LOG_DIR"
|
||||
|
||||
|
|
@ -249,117 +221,6 @@ if [ "$DRY_RUN" -eq 1 ]; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# --- 3c. SSH-key drift preflight ---------------------------------------------
|
||||
# DEV-587 / DEV-586 (2026-09-27) — some k3s nodes drift to lose cp-1's SSH
|
||||
# pubkey from /root/.ssh/authorized_keys (probable Hetzner cloud-init resync
|
||||
# from the HC SSH-keys API; the operator key is NOT in that keyset). When this
|
||||
# happens mid-cycle the whole run aborts partway through drain / peer-probe.
|
||||
# This preflight proves the operator can SSH to every planned node and that
|
||||
# cp-1's pubkey is present in root's authorized_keys — auto-heals drift via
|
||||
# the operator's own SSH access, and hard-fails BEFORE anything is drained if
|
||||
# a node is genuinely unreachable.
|
||||
ssh_drift_preflight() {
|
||||
local ok=() repaired=() problem=()
|
||||
|
||||
log "[preflight] SSH-key drift check for planned nodes"
|
||||
|
||||
local cp1_pubkey cp1_key_body
|
||||
if ! cp1_pubkey=$(ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
root@"$CONTROL_PLANE_HOST" 'cat /root/.ssh/id_rsa.pub' 2>/dev/null); then
|
||||
die "SSH-drift preflight: cannot read cp-1 pubkey from $CONTROL_PLANE_HOST — fix operator->cp-1 SSH first"
|
||||
fi
|
||||
cp1_pubkey=$(printf '%s' "$cp1_pubkey" | tr -d '\r\n')
|
||||
cp1_key_body=$(printf '%s' "$cp1_pubkey" | awk '{print $2}')
|
||||
[ -n "$cp1_key_body" ] || die "SSH-drift preflight: could not parse cp-1 pubkey (got: $cp1_pubkey)"
|
||||
log " cp-1 pubkey OK (${cp1_key_body:0:32}...)"
|
||||
|
||||
local plan_nodes=()
|
||||
for n in "${WORKER_ORDER[@]:-}" "${CP_ORDER[@]:-}"; do
|
||||
[ -z "$n" ] && continue
|
||||
plan_nodes+=("$n")
|
||||
done
|
||||
if [ ${#plan_nodes[@]} -eq 0 ]; then
|
||||
log " (no planned nodes — nothing to preflight)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
for n in "${plan_nodes[@]}"; do
|
||||
local endpoint
|
||||
if ! endpoint=$(node_ssh_endpoint "$n"); then
|
||||
log " $n: unknown ssh endpoint — refusing"
|
||||
problem+=("$n (no ssh mapping in node_ssh_endpoint)")
|
||||
continue
|
||||
fi
|
||||
|
||||
# 1. reachability from the operator machine.
|
||||
# shellcheck disable=SC2086 # $endpoint may include `-J bastion`; deliberate word-split.
|
||||
if ! ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
$endpoint true 2>/dev/null; then
|
||||
problem+=("$n ($endpoint - operator SSH refused)")
|
||||
continue
|
||||
fi
|
||||
|
||||
# 2. cp-1 is the source of the key; drift check would be pointless on it.
|
||||
if [ "$n" = "k3s-cp-1" ]; then
|
||||
ok+=("$n")
|
||||
continue
|
||||
fi
|
||||
|
||||
# 3. cp-1 pubkey present?
|
||||
# shellcheck disable=SC2086
|
||||
if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
$endpoint "grep -qF '$cp1_key_body' /root/.ssh/authorized_keys" 2>/dev/null; then
|
||||
ok+=("$n")
|
||||
continue
|
||||
fi
|
||||
|
||||
# 4. auto-heal — append cp-1's pubkey via the (already verified) operator SSH.
|
||||
log " $n: cp-1 pubkey MISSING from /root/.ssh/authorized_keys — appending"
|
||||
# shellcheck disable=SC2086
|
||||
if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
$endpoint "install -d -m 700 /root/.ssh && printf '%s\n' '$cp1_pubkey' >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys" \
|
||||
2>/dev/null; then
|
||||
# shellcheck disable=SC2086
|
||||
if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
$endpoint "grep -qF '$cp1_key_body' /root/.ssh/authorized_keys" 2>/dev/null; then
|
||||
repaired+=("$n")
|
||||
else
|
||||
problem+=("$n (auto-heal wrote but grep-verify failed)")
|
||||
fi
|
||||
else
|
||||
problem+=("$n ($endpoint - auto-heal write refused)")
|
||||
fi
|
||||
done
|
||||
|
||||
log "[preflight] SSH-drift summary:"
|
||||
log " ok (${#ok[@]}): ${ok[*]:-none}"
|
||||
log " repaired (${#repaired[@]}): ${repaired[*]:-none}"
|
||||
log " problem (${#problem[@]}): ${problem[*]:-none}"
|
||||
|
||||
{
|
||||
echo "# SSH-drift preflight — $(date -u +%FT%TZ)"
|
||||
echo "# cp-1 pubkey: $cp1_pubkey"
|
||||
printf 'ok\t%s\n' "${ok[*]:-}"
|
||||
printf 'repaired\t%s\n' "${repaired[*]:-}"
|
||||
printf 'problem\t%s\n' "${problem[*]:-}"
|
||||
} > "$LOG_DIR/ssh-preflight.txt"
|
||||
|
||||
if [ ${#problem[@]} -gt 0 ]; then
|
||||
log "SSH-drift preflight FAILED for: ${problem[*]}"
|
||||
log " cp-1 pubkey to install:"
|
||||
log " $cp1_pubkey"
|
||||
log " From any machine that CAN reach the failing node, run:"
|
||||
log " ssh root@<node> \"grep -qF '$cp1_key_body' /root/.ssh/authorized_keys || echo '$cp1_pubkey' >> /root/.ssh/authorized_keys\""
|
||||
die "SSH-drift preflight failed — refuse to start rolling update"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$SKIP_SSH_PREFLIGHT" -eq 1 ]; then
|
||||
log "[preflight] SSH-drift check SKIPPED (--skip-ssh-preflight)"
|
||||
else
|
||||
ssh_drift_preflight
|
||||
fi
|
||||
|
||||
# --- 4. worker loop ----------------------------------------------------------
|
||||
updated=()
|
||||
skipped=()
|
||||
|
|
@ -437,11 +298,20 @@ log "[final] apt history digest"
|
|||
{
|
||||
for n in "${updated[@]}"; do
|
||||
echo "=== $n ==="
|
||||
if ! t=$(node_ssh_endpoint "$n"); then
|
||||
echo " (unknown ssh target)"
|
||||
continue
|
||||
fi
|
||||
# shellcheck disable=SC2086 # $t may include `-J bastion`; deliberate word-split.
|
||||
# resolve ssh target via a mini-eval of node_ssh_target-equivalent
|
||||
case "$n" in
|
||||
k3s-cp-1) t="root@178.105.17.239" ;;
|
||||
k3s-cp-2) t="root@188.245.85.199" ;;
|
||||
k3s-cp-3) t="root@49.13.92.162" ;;
|
||||
k3s-worker-1) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
|
||||
k3s-worker-2) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
|
||||
k3s-worker-3) t="root@167.233.121.121" ;;
|
||||
k3s-worker-4) t="root@128.140.3.80" ;;
|
||||
k3s-worker-5) t="root@167.233.192.86" ;;
|
||||
k3s-worker-6) t="root@49.12.223.133" ;;
|
||||
k3s-update-runner) t="root@167.233.79.65" ;;
|
||||
*) echo " (unknown ssh target)"; continue ;;
|
||||
esac
|
||||
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new $t \
|
||||
'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -40' 2>/dev/null \
|
||||
|| echo " (could not read apt history)"
|
||||
|
|
|
|||
|
|
@ -15,8 +15,6 @@
|
|||
# DRAIN_TIMEOUT_SECONDS (default 600)
|
||||
# REBOOT_MAX_WAIT_SECONDS (default 600)
|
||||
# POST_UNCORDON_WAIT_SECONDS (default 180)
|
||||
# HEALTH_MAX_RETRIES (default 10) — extra post-uncordon health retries
|
||||
# HEALTH_RETRY_INTERVAL_SECONDS (default 30) — delay between those retries
|
||||
# CONTROL_PLANE_HOST (default 178.105.17.239)
|
||||
# SSH_OPTS (default "-o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new")
|
||||
# ASSUME_YES=1 to skip interactive confirmations
|
||||
|
|
@ -31,15 +29,9 @@ fi
|
|||
DRAIN_TIMEOUT_SECONDS="${DRAIN_TIMEOUT_SECONDS:-600}"
|
||||
REBOOT_MAX_WAIT_SECONDS="${REBOOT_MAX_WAIT_SECONDS:-600}"
|
||||
POST_UNCORDON_WAIT_SECONDS="${POST_UNCORDON_WAIT_SECONDS:-180}"
|
||||
HEALTH_MAX_RETRIES="${HEALTH_MAX_RETRIES:-10}"
|
||||
HEALTH_RETRY_INTERVAL_SECONDS="${HEALTH_RETRY_INTERVAL_SECONDS:-30}"
|
||||
CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}"
|
||||
SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new}"
|
||||
|
||||
# Pod-level signatures that do not recover on their own — short-circuit the
|
||||
# post-uncordon retry loop and halt immediately (DEV-595).
|
||||
PERSISTENT_FAILURE_PATTERN='CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|InvalidImageName'
|
||||
|
||||
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; }
|
||||
die() { log "FATAL: $*"; exit 1; }
|
||||
|
||||
|
|
@ -196,39 +188,11 @@ log "[6/7] uncordon $NODE"
|
|||
kubectl uncordon "$NODE"
|
||||
|
||||
# --- 7. post-node settle ------------------------------------------------------
|
||||
# First check at POST_UNCORDON_WAIT_SECONDS, then up to HEALTH_MAX_RETRIES more
|
||||
# attempts at HEALTH_RETRY_INTERVAL_SECONDS. Transient yellow signals (Init:*,
|
||||
# Running-but-not-Ready, deployment-below-replicas during reschedule) often
|
||||
# clear within a minute or two; previously they tripped a FATAL on the first
|
||||
# check. A pod in CrashLoopBackOff (or image-pull failure) is persistent and
|
||||
# short-circuits the loop immediately. (DEV-595)
|
||||
log "[7/7] post-node settle (${POST_UNCORDON_WAIT_SECONDS}s) + health check"
|
||||
sleep "$POST_UNCORDON_WAIT_SECONDS"
|
||||
|
||||
attempt=1
|
||||
max_attempts=$((HEALTH_MAX_RETRIES + 1))
|
||||
while :; do
|
||||
set +e
|
||||
HEALTH_OUT=$(RETRY_ON_TRANSIENT=0 "$SCRIPT_DIR/cluster-health.sh" 2>&1)
|
||||
HEALTH_RC=$?
|
||||
set -e
|
||||
|
||||
echo "$HEALTH_OUT"
|
||||
|
||||
if [ "$HEALTH_RC" -eq 0 ]; then
|
||||
log "=== $NODE update: OK (health attempt $attempt/$max_attempts) ==="
|
||||
break
|
||||
fi
|
||||
|
||||
if echo "$HEALTH_OUT" | grep -qE "$PERSISTENT_FAILURE_PATTERN"; then
|
||||
die "cluster health failed after updating $NODE with persistent pod error (CrashLoopBackOff/ImagePullBackOff/etc.) on attempt $attempt — STOP the cycle, do NOT continue"
|
||||
fi
|
||||
|
||||
if [ "$attempt" -ge "$max_attempts" ]; then
|
||||
die "cluster health still failing after $attempt attempts (interval ${HEALTH_RETRY_INTERVAL_SECONDS}s) following update of $NODE — STOP the cycle, do NOT continue"
|
||||
fi
|
||||
|
||||
log " cluster health yellow on attempt $attempt/$max_attempts (no persistent signatures) — retrying in ${HEALTH_RETRY_INTERVAL_SECONDS}s"
|
||||
sleep "$HEALTH_RETRY_INTERVAL_SECONDS"
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
if RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh"; then
|
||||
log "=== $NODE update: OK ==="
|
||||
else
|
||||
die "cluster health failed after updating $NODE — STOP the cycle, do NOT continue"
|
||||
fi
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue