b7e34236db
k3s-worker-6 was joined on 2026-08-30 but never added to the node_ssh_target case in update-node.sh, so the weekly cycle would die with 'unknown node k3s-worker-6' as soon as it reached that node. Map it to its public IP (49.12.223.133) — same pattern as worker-3/4/5. Also add the corresponding case in os-update.sh's apt-history digest. Co-Authored-By: Paperclip <noreply@paperclip.ing>
331 lines
12 KiB
Bash
Executable file
331 lines
12 KiB
Bash
Executable file
#!/bin/bash
|
|
# os-update.sh — full weekly rolling OS-update cycle.
|
|
#
|
|
# Behavior:
|
|
# 1. preflight cluster health (fail-closed)
|
|
# 2. take an etcd snapshot
|
|
# 3. compute node order:
|
|
# workers (non-Stalwart first, Stalwart-hosting fsn1 workers last)
|
|
# then control-plane nodes (etcd followers first, leader last).
|
|
# 4. call update-node.sh for each worker and update-cp-node.sh for each CP,
|
|
# halting on any failure
|
|
# 5. finalization: health snapshot + apt history digest
|
|
#
|
|
# Usage:
|
|
# os-update.sh [--dry-run] [--start-from <node>] [--only <node>]
|
|
# [--skip-cps] [--only-cps]
|
|
#
|
|
# Flags:
|
|
# --skip-cps run workers only (legacy behavior, useful for narrow patches).
|
|
# --only-cps skip workers, only walk the CP phase.
|
|
# --only <n> update a single node (worker or CP, routed automatically).
|
|
#
|
|
# Environment:
|
|
# CONTROL_PLANE_HOST (default 178.105.17.239)
|
|
# POST_CP_WAIT_SECONDS (default 180) — settle between CPs after uncordon.
|
|
# All env vars honored by update-node.sh / update-cp-node.sh are honored here.
|
|
#
|
|
# Read OS_UPDATE_PROCEDURE.md alongside this script; the script mirrors it
|
|
# step-for-step and the doc is the authoritative reference. CP handling is
|
|
# delegated to update-cp-node.sh; see CP_UPDATE_PROCEDURE.md.
|
|
set -euo pipefail
|
|
|
|
CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}"
|
|
POST_CP_WAIT_SECONDS="${POST_CP_WAIT_SECONDS:-180}"
|
|
DRY_RUN=0
|
|
START_FROM=""
|
|
ONLY=""
|
|
SKIP_CPS=0
|
|
ONLY_CPS=0
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--dry-run) DRY_RUN=1; shift ;;
|
|
--start-from) START_FROM="$2"; shift 2 ;;
|
|
--only) ONLY="$2"; shift 2 ;;
|
|
--skip-cps) SKIP_CPS=1; shift ;;
|
|
--only-cps) ONLY_CPS=1; shift ;;
|
|
-h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
|
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
STAMP=$(date -u +%Y%m%dT%H%M%SZ)
|
|
LOG_DIR="/tmp/os-update-${STAMP}"
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_DIR/main.log"; }
|
|
die() { log "FATAL: $*"; exit 1; }
|
|
|
|
log "=== os-update.sh cycle $STAMP ==="
|
|
log "log dir: $LOG_DIR"
|
|
|
|
# --- 0. sanity checks ---------------------------------------------------------
|
|
command -v kubectl >/dev/null || die "kubectl not on PATH"
|
|
command -v jq >/dev/null || die "jq not on PATH (needed for health checks)"
|
|
|
|
# --- 1. preflight -------------------------------------------------------------
|
|
log "[preflight] cluster health"
|
|
if ! RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh" | tee "$LOG_DIR/preflight.log"; then
|
|
die "cluster is not healthy at preflight — refuse to start OS updates"
|
|
fi
|
|
|
|
# --- 2. etcd snapshot ---------------------------------------------------------
|
|
if [ "$DRY_RUN" -eq 0 ]; then
|
|
log "[preflight] taking k3s etcd snapshot"
|
|
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@"$CONTROL_PLANE_HOST" \
|
|
"k3s etcd-snapshot save --name pre-os-update-$(date +%Y%m%d)" | tee "$LOG_DIR/etcd-snapshot.log" || \
|
|
log "WARN: etcd snapshot failed — continuing but this is a risk on CP reboot"
|
|
else
|
|
log "[preflight] DRY-RUN — skipping etcd snapshot"
|
|
fi
|
|
|
|
# --- 3. node ordering ---------------------------------------------------------
|
|
# Ordering rule:
|
|
# - workers first (non-Stalwart, then Stalwart-hosting fsn1 last)
|
|
# - control-plane nodes AFTER workers, one at a time, etcd followers first,
|
|
# leader last. Rationale (see CP_UPDATE_PROCEDURE.md):
|
|
# * two of three CPs down = etcd quorum loss = write outage. Only one CP
|
|
# cordoned/rebooting at a time.
|
|
# * draining a follower is a no-op for raft leadership; draining the leader
|
|
# forces a re-election, so doing followers first minimises leader flapping.
|
|
# Since DEV-510 (2026-08-22) the cluster runs HA embedded etcd across
|
|
# cp-1/cp-2/cp-3 with 3-node quorum. This script delegates all CP steps to
|
|
# update-cp-node.sh, which handles swap (idempotent), preflight, drain, apt,
|
|
# reboot with /livez polling, etcd rejoin check, and finalize.
|
|
STALWART_NODE=$(kubectl -n stalwart get pod -l app=stalwart -o jsonpath='{.items[*].spec.nodeName}' 2>/dev/null | tr ' ' '\n' | sort -u || true)
|
|
# If the pod's not currently up (e.g. Pending) we still want to protect fsn1 workers.
|
|
|
|
CP_NODES=$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' || true)
|
|
ALL_NODES=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n')
|
|
|
|
is_cp() {
|
|
local n="$1"
|
|
for c in $CP_NODES; do [ "$c" = "$n" ] && return 0; done
|
|
return 1
|
|
}
|
|
|
|
workers=()
|
|
stalwart_workers=()
|
|
for n in $ALL_NODES; do
|
|
is_cp "$n" && continue
|
|
if [ -n "$STALWART_NODE" ] && [ "$n" = "$STALWART_NODE" ]; then
|
|
stalwart_workers+=("$n")
|
|
continue
|
|
fi
|
|
# Any fsn1 worker is a potential Stalwart host — schedule after non-Stalwart nodes.
|
|
loc=$(kubectl get node "$n" -o jsonpath='{.metadata.labels.csi\.hetzner\.cloud/location}' 2>/dev/null || echo "")
|
|
if [ "$loc" = "fsn1" ]; then
|
|
stalwart_workers+=("$n")
|
|
else
|
|
workers+=("$n")
|
|
fi
|
|
done
|
|
|
|
WORKER_ORDER=("${workers[@]}" "${stalwart_workers[@]}")
|
|
|
|
# --- 3b. control-plane ordering (leader last) --------------------------------
|
|
# CP-node → public IP map. Kept out-of-cluster so a full-cluster incident
|
|
# doesn't hide etcd from the operator machine.
|
|
declare -A CP_HOST=(
|
|
[k3s-cp-1]="178.105.17.239"
|
|
[k3s-cp-2]="188.245.85.199"
|
|
[k3s-cp-3]="49.13.92.162"
|
|
)
|
|
|
|
etcd_leader_ip() {
|
|
# Query etcd leader endpoint from the operator's control-plane host. Uses
|
|
# the same TLS material as update-cp-node.sh. Prints "IP" on success,
|
|
# nothing on failure.
|
|
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -o BatchMode=yes \
|
|
root@"$CONTROL_PLANE_HOST" \
|
|
'ETCDCTL_API=3 etcdctl \
|
|
--endpoints=https://127.0.0.1:2379 \
|
|
--cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
|
|
--cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
|
|
--key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \
|
|
endpoint status --cluster -w simple 2>/dev/null' \
|
|
| awk -F, '$5 ~ /true/ {print $1}' \
|
|
| sed -E 's|https?://([^:]+):.*|\1|'
|
|
}
|
|
|
|
CP_ORDER=()
|
|
if [ -n "$CP_NODES" ]; then
|
|
LEADER_IP="$(etcd_leader_ip || true)"
|
|
leader_node=""
|
|
followers=()
|
|
for n in $CP_NODES; do
|
|
ip="${CP_HOST[$n]:-}"
|
|
if [ -n "$ip" ] && [ "$ip" = "$LEADER_IP" ]; then
|
|
leader_node="$n"
|
|
else
|
|
followers+=("$n")
|
|
fi
|
|
done
|
|
if [ -n "$leader_node" ]; then
|
|
CP_ORDER=("${followers[@]}" "$leader_node")
|
|
log "[plan] etcd leader is $leader_node ($LEADER_IP) — updating followers first, leader last"
|
|
else
|
|
# Couldn't identify leader — refuse to guess. Fall back to no CPs.
|
|
CP_ORDER=()
|
|
log "[plan] WARN could not identify etcd leader — skipping CP phase this cycle"
|
|
fi
|
|
fi
|
|
|
|
# Route --only <node> automatically to worker or CP list.
|
|
if [ -n "$ONLY" ]; then
|
|
if is_cp "$ONLY"; then
|
|
CP_ORDER=("$ONLY")
|
|
WORKER_ORDER=()
|
|
else
|
|
WORKER_ORDER=("$ONLY")
|
|
CP_ORDER=()
|
|
fi
|
|
elif [ -n "$START_FROM" ]; then
|
|
new=()
|
|
skip=1
|
|
for n in "${WORKER_ORDER[@]}"; do
|
|
[ "$n" = "$START_FROM" ] && skip=0
|
|
[ $skip -eq 0 ] && new+=("$n")
|
|
done
|
|
WORKER_ORDER=("${new[@]}")
|
|
# If START_FROM is a CP, trim CP_ORDER instead.
|
|
if is_cp "$START_FROM"; then
|
|
new_cp=()
|
|
skip=1
|
|
for n in "${CP_ORDER[@]:-}"; do
|
|
[ -z "$n" ] && continue
|
|
[ "$n" = "$START_FROM" ] && skip=0
|
|
[ $skip -eq 0 ] && new_cp+=("$n")
|
|
done
|
|
CP_ORDER=("${new_cp[@]}")
|
|
WORKER_ORDER=()
|
|
fi
|
|
fi
|
|
|
|
[ "$SKIP_CPS" -eq 1 ] && CP_ORDER=()
|
|
[ "$ONLY_CPS" -eq 1 ] && WORKER_ORDER=()
|
|
|
|
log "[plan] workers (${#WORKER_ORDER[@]}): ${WORKER_ORDER[*]:-none}"
|
|
log "[plan] control-planes (${#CP_ORDER[@]}): ${CP_ORDER[*]:-none}"
|
|
{
|
|
echo "# workers"
|
|
printf '%s\n' "${WORKER_ORDER[@]:-}"
|
|
echo "# control-planes"
|
|
printf '%s\n' "${CP_ORDER[@]:-}"
|
|
} > "$LOG_DIR/plan.txt"
|
|
|
|
if [ "$DRY_RUN" -eq 1 ]; then
|
|
log "DRY-RUN — plan written, no node touched. Exiting."
|
|
exit 0
|
|
fi
|
|
|
|
# --- 4. worker loop ----------------------------------------------------------
|
|
updated=()
|
|
skipped=()
|
|
for n in "${WORKER_ORDER[@]}"; do
|
|
[ -z "$n" ] && continue
|
|
log "===================================================================="
|
|
log "==> updating worker $n"
|
|
log "===================================================================="
|
|
NODE_LOG="$LOG_DIR/${n}.log"
|
|
set +e
|
|
"$SCRIPT_DIR/update-node.sh" "$n" 2>&1 | tee "$NODE_LOG"
|
|
rc=${PIPESTATUS[0]}
|
|
set -e
|
|
case $rc in
|
|
0) updated+=("$n") ;;
|
|
3) skipped+=("$n:drain-blocked") ;;
|
|
*) die "update-node.sh failed for $n (rc=$rc). Cycle halted. See $NODE_LOG" ;;
|
|
esac
|
|
done
|
|
|
|
# --- 4b. control-plane loop --------------------------------------------------
|
|
# One CP at a time, followers first, leader last. Delegates to update-cp-node.sh
|
|
# --run (Phase A add-swap is idempotent, Phase B..F run with ASSUME_YES=1).
|
|
# Between CPs we run cluster-health.sh and re-verify etcd cluster status; halt
|
|
# on any yellow signal.
|
|
if [ ${#CP_ORDER[@]} -gt 0 ]; then
|
|
log "===================================================================="
|
|
log "==> control-plane phase (${#CP_ORDER[@]} nodes): ${CP_ORDER[*]}"
|
|
log "===================================================================="
|
|
first_cp=1
|
|
for cp in "${CP_ORDER[@]}"; do
|
|
[ -z "$cp" ] && continue
|
|
if [ $first_cp -eq 0 ]; then
|
|
log "-- health gate between CPs (settle ${POST_CP_WAIT_SECONDS}s, then re-check)"
|
|
sleep "$POST_CP_WAIT_SECONDS"
|
|
if ! RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh" 2>&1 | tee "$LOG_DIR/inter-cp-health-before-${cp}.log"; then
|
|
die "cluster health degraded between CPs — halting before $cp. See $LOG_DIR/"
|
|
fi
|
|
# Re-check etcd cluster status; every member must still be started.
|
|
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@"$CONTROL_PLANE_HOST" \
|
|
'ETCDCTL_API=3 etcdctl \
|
|
--endpoints=https://127.0.0.1:2379 \
|
|
--cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
|
|
--cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
|
|
--key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \
|
|
endpoint status --cluster -w table' 2>&1 | tee -a "$LOG_DIR/inter-cp-health-before-${cp}.log" || \
|
|
die "etcd endpoint status failed between CPs — halting before $cp"
|
|
fi
|
|
log "===================================================================="
|
|
log "==> updating control-plane $cp"
|
|
log "===================================================================="
|
|
CP_LOG="$LOG_DIR/${cp}.log"
|
|
set +e
|
|
ASSUME_YES=1 "$SCRIPT_DIR/update-cp-node.sh" "$cp" --run 2>&1 | tee "$CP_LOG"
|
|
rc=${PIPESTATUS[0]}
|
|
set -e
|
|
case $rc in
|
|
0) updated+=("$cp") ;;
|
|
*) die "update-cp-node.sh failed for $cp (rc=$rc). Cycle halted. See $CP_LOG" ;;
|
|
esac
|
|
first_cp=0
|
|
done
|
|
fi
|
|
|
|
# --- 5. finalization ----------------------------------------------------------
|
|
log "===================================================================="
|
|
log "==> finalization"
|
|
log "===================================================================="
|
|
|
|
log "[final] cluster health"
|
|
"$SCRIPT_DIR/cluster-health.sh" | tee "$LOG_DIR/final-health.log" || \
|
|
die "final cluster health failed after cycle. Do NOT declare success."
|
|
|
|
log "[final] apt history digest"
|
|
{
|
|
for n in "${updated[@]}"; do
|
|
echo "=== $n ==="
|
|
# resolve ssh target via a mini-eval of node_ssh_target-equivalent
|
|
case "$n" in
|
|
k3s-cp-1) t="root@178.105.17.239" ;;
|
|
k3s-cp-2) t="root@188.245.85.199" ;;
|
|
k3s-cp-3) t="root@49.13.92.162" ;;
|
|
k3s-worker-1) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
|
|
k3s-worker-2) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
|
|
k3s-worker-3) t="root@167.233.121.121" ;;
|
|
k3s-worker-4) t="root@128.140.3.80" ;;
|
|
k3s-worker-5) t="root@167.233.192.86" ;;
|
|
k3s-worker-6) t="root@49.12.223.133" ;;
|
|
k3s-update-runner) t="root@167.233.79.65" ;;
|
|
*) echo " (unknown ssh target)"; continue ;;
|
|
esac
|
|
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new $t \
|
|
'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -40' 2>/dev/null \
|
|
|| echo " (could not read apt history)"
|
|
done
|
|
} | tee "$LOG_DIR/apt-history.log"
|
|
|
|
log "=== summary ==="
|
|
log "updated (${#updated[@]}): ${updated[*]:-none}"
|
|
log "skipped (${#skipped[@]}): ${skipped[*]:-none}"
|
|
log "logs: $LOG_DIR"
|
|
|
|
if [ ${#skipped[@]} -gt 0 ]; then
|
|
log "cycle finished with skipped nodes — return code 4 so the caller can escalate"
|
|
exit 4
|
|
fi
|
|
|
|
log "cycle complete."
|