34350a03bf
Since DEV-510 landed HA control plane (2026-08-22), the cluster runs cp-1
plus cp-2/cp-3. The previous exclusion in os-update.sh matched `k3s-cp-1`
by hard-coded name only, which would have caused cp-2 and cp-3 to be
treated as regular fsn1 workers and drained/rebooted without the
CP-specific procedure.
Fix: select the CP list from `kubectl get nodes -l
node-role.kubernetes.io/control-plane` and skip any of those nodes. This
covers all present and future CPs automatically.
Also updated OS_UPDATE_PROCEDURE.md topology table and order rule to
document that all three CPs exist and are excluded from the weekly
cycle. The HA-aware CP OS-update procedure is a separate follow-up.
Verified on the current cluster:
[plan] EXCLUDING control-plane nodes: k3s-cp-1 k3s-cp-2 k3s-cp-3
[plan] ordered nodes (6): k3s-worker-4 k3s-update-runner k3s-worker-1
k3s-worker-2 k3s-worker-3 k3s-worker-5
Co-Authored-By: Paperclip <noreply@paperclip.ing>
196 lines
7.4 KiB
Bash
Executable file
196 lines
7.4 KiB
Bash
Executable file
#!/bin/bash
|
|
# os-update.sh — full weekly rolling OS-update cycle.
|
|
#
|
|
# Behavior:
|
|
# 1. preflight cluster health (fail-closed)
|
|
# 2. take an etcd snapshot
|
|
# 3. compute node order (workers first, control plane last;
|
|
# Stalwart-hosting fsn1 workers moved to end of workers group)
|
|
# 4. call update-node.sh for each node, halting on any failure
|
|
# 5. finalization: health snapshot + apt history digest
|
|
#
|
|
# Usage:
|
|
# os-update.sh [--dry-run] [--start-from <node>] [--only <node>]
|
|
#
|
|
# Environment:
|
|
# CONTROL_PLANE_HOST (default 178.105.17.239)
|
|
# All env vars honored by update-node.sh are honored here as well.
|
|
#
|
|
# Read OS_UPDATE_PROCEDURE.md alongside this script; the script mirrors it
|
|
# step-for-step and the doc is the authoritative reference.
|
|
set -euo pipefail
|
|
|
|
CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}"
|
|
DRY_RUN=0
|
|
START_FROM=""
|
|
ONLY=""
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--dry-run) DRY_RUN=1; shift ;;
|
|
--start-from) START_FROM="$2"; shift 2 ;;
|
|
--only) ONLY="$2"; shift 2 ;;
|
|
-h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
|
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
STAMP=$(date -u +%Y%m%dT%H%M%SZ)
|
|
LOG_DIR="/tmp/os-update-${STAMP}"
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_DIR/main.log"; }
|
|
die() { log "FATAL: $*"; exit 1; }
|
|
|
|
log "=== os-update.sh cycle $STAMP ==="
|
|
log "log dir: $LOG_DIR"
|
|
|
|
# --- 0. sanity checks ---------------------------------------------------------
|
|
command -v kubectl >/dev/null || die "kubectl not on PATH"
|
|
command -v jq >/dev/null || die "jq not on PATH (needed for health checks)"
|
|
|
|
# --- 1. preflight -------------------------------------------------------------
|
|
log "[preflight] cluster health"
|
|
if ! RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh" | tee "$LOG_DIR/preflight.log"; then
|
|
die "cluster is not healthy at preflight — refuse to start OS updates"
|
|
fi
|
|
|
|
# --- 2. etcd snapshot ---------------------------------------------------------
|
|
if [ "$DRY_RUN" -eq 0 ]; then
|
|
log "[preflight] taking k3s etcd snapshot"
|
|
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@"$CONTROL_PLANE_HOST" \
|
|
"k3s etcd-snapshot save --name pre-os-update-$(date +%Y%m%d)" | tee "$LOG_DIR/etcd-snapshot.log" || \
|
|
log "WARN: etcd snapshot failed — continuing but this is a risk on CP reboot"
|
|
else
|
|
log "[preflight] DRY-RUN — skipping etcd snapshot"
|
|
fi
|
|
|
|
# --- 3. node ordering ---------------------------------------------------------
|
|
# Ordering rule:
|
|
# - workers only
|
|
# - within workers: nodes NOT hosting Stalwart first, Stalwart-hosting fsn1 nodes last
|
|
# - ALL control-plane nodes are EXCLUDED and never updated by this script. Rationale:
|
|
# * kine/etcd write-path is sensitive to concurrent drains (see kine thundering-herd
|
|
# guardrails in OS_UPDATE_PROCEDURE.md, added after DEV-495).
|
|
# * rebooting a CP removes one apiserver — needs external liveness monitoring.
|
|
# * CP nodes host their own StatefulSet workloads that need batched eviction.
|
|
# Since DEV-510 (2026-08-22) the cluster runs HA control plane (cp-1/cp-2/cp-3). The
|
|
# exclusion here is role-based so ALL current and future CPs are covered automatically.
|
|
# To update a CP, use `scripts/os-update/update-cp-1.sh` (currently cp-1-only; will be
|
|
# generalized to any CP as part of the HA-aware CP OS-update procedure follow-up).
|
|
STALWART_NODE=$(kubectl -n stalwart get pod -l app=stalwart -o jsonpath='{.items[*].spec.nodeName}' 2>/dev/null | tr ' ' '\n' | sort -u || true)
|
|
# If the pod's not currently up (e.g. Pending) we still want to protect fsn1 workers.
|
|
|
|
CP_NODES=$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' || true)
|
|
ALL_NODES=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n')
|
|
|
|
is_cp() {
|
|
local n="$1"
|
|
for c in $CP_NODES; do [ "$c" = "$n" ] && return 0; done
|
|
return 1
|
|
}
|
|
|
|
workers=()
|
|
stalwart_workers=()
|
|
for n in $ALL_NODES; do
|
|
is_cp "$n" && continue
|
|
if [ -n "$STALWART_NODE" ] && [ "$n" = "$STALWART_NODE" ]; then
|
|
stalwart_workers+=("$n")
|
|
continue
|
|
fi
|
|
# Any fsn1 worker is a potential Stalwart host — schedule after non-Stalwart nodes.
|
|
loc=$(kubectl get node "$n" -o jsonpath='{.metadata.labels.csi\.hetzner\.cloud/location}' 2>/dev/null || echo "")
|
|
if [ "$loc" = "fsn1" ]; then
|
|
stalwart_workers+=("$n")
|
|
else
|
|
workers+=("$n")
|
|
fi
|
|
done
|
|
|
|
ORDER=("${workers[@]}" "${stalwart_workers[@]}")
|
|
# NOTE: CP nodes intentionally excluded. To update a CP, see CP1_UPDATE_PROCEDURE.md.
|
|
if [ -n "$CP_NODES" ]; then
|
|
log "[plan] EXCLUDING control-plane nodes: $(echo $CP_NODES | tr '\n' ' ')— use scripts/os-update/update-cp-1.sh (see CP1_UPDATE_PROCEDURE.md)"
|
|
fi
|
|
|
|
if [ -n "$ONLY" ]; then
|
|
ORDER=("$ONLY")
|
|
elif [ -n "$START_FROM" ]; then
|
|
new=()
|
|
skip=1
|
|
for n in "${ORDER[@]}"; do
|
|
[ "$n" = "$START_FROM" ] && skip=0
|
|
[ $skip -eq 0 ] && new+=("$n")
|
|
done
|
|
ORDER=("${new[@]}")
|
|
fi
|
|
|
|
log "[plan] ordered nodes (${#ORDER[@]}): ${ORDER[*]}"
|
|
printf '%s\n' "${ORDER[@]}" > "$LOG_DIR/plan.txt"
|
|
|
|
if [ "$DRY_RUN" -eq 1 ]; then
|
|
log "DRY-RUN — plan written, no node touched. Exiting."
|
|
exit 0
|
|
fi
|
|
|
|
# --- 4. per-node loop ---------------------------------------------------------
|
|
updated=()
|
|
skipped=()
|
|
for n in "${ORDER[@]}"; do
|
|
log "===================================================================="
|
|
log "==> updating $n"
|
|
log "===================================================================="
|
|
NODE_LOG="$LOG_DIR/${n}.log"
|
|
set +e
|
|
"$SCRIPT_DIR/update-node.sh" "$n" 2>&1 | tee "$NODE_LOG"
|
|
rc=${PIPESTATUS[0]}
|
|
set -e
|
|
case $rc in
|
|
0) updated+=("$n") ;;
|
|
3) skipped+=("$n:drain-blocked") ;;
|
|
*) die "update-node.sh failed for $n (rc=$rc). Cycle halted. See $NODE_LOG" ;;
|
|
esac
|
|
done
|
|
|
|
# --- 5. finalization ----------------------------------------------------------
|
|
log "===================================================================="
|
|
log "==> finalization"
|
|
log "===================================================================="
|
|
|
|
log "[final] cluster health"
|
|
"$SCRIPT_DIR/cluster-health.sh" | tee "$LOG_DIR/final-health.log" || \
|
|
die "final cluster health failed after cycle. Do NOT declare success."
|
|
|
|
log "[final] apt history digest"
|
|
{
|
|
for n in "${updated[@]}"; do
|
|
echo "=== $n ==="
|
|
# resolve ssh target via a mini-eval of node_ssh_target-equivalent
|
|
case "$n" in
|
|
k3s-cp-1) t="root@178.105.17.239" ;;
|
|
k3s-worker-1) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
|
|
k3s-worker-2) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
|
|
k3s-worker-3) t="root@167.233.121.121" ;;
|
|
k3s-worker-4) t="root@128.140.3.80" ;;
|
|
k3s-worker-5) t="root@167.233.192.86" ;;
|
|
k3s-update-runner) t="root@167.233.79.65" ;;
|
|
*) echo " (unknown ssh target)"; continue ;;
|
|
esac
|
|
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new $t \
|
|
'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -40' 2>/dev/null \
|
|
|| echo " (could not read apt history)"
|
|
done
|
|
} | tee "$LOG_DIR/apt-history.log"
|
|
|
|
log "=== summary ==="
|
|
log "updated (${#updated[@]}): ${updated[*]:-none}"
|
|
log "skipped (${#skipped[@]}): ${skipped[*]:-none}"
|
|
log "logs: $LOG_DIR"
|
|
|
|
if [ ${#skipped[@]} -gt 0 ]; then
|
|
log "cycle finished with skipped nodes — return code 4 so the caller can escalate"
|
|
exit 4
|
|
fi
|
|
|
|
log "cycle complete."
|