596 lines
21 KiB
Bash
596 lines
21 KiB
Bash
|
#!/bin/bash
|
||
|
|
# update-cp-node.sh — controlled OS update for any HA k3s control-plane node.
|
||
|
|
#
|
||
|
|
# See ../CP_UPDATE_PROCEDURE.md for the full design rationale.
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# update-cp-node.sh <node> --dry-run # print what would be done, touch nothing
|
||
|
|
# update-cp-node.sh <node> --add-swap # Phase A only (idempotent, safe standalone)
|
||
|
|
# update-cp-node.sh <node> --preflight # Phase B only
|
||
|
|
# update-cp-node.sh <node> --drain # Phase C only (cordon + drain)
|
||
|
|
# update-cp-node.sh <node> --apt # Phase D only (requires <node> already fully drained)
|
||
|
|
# update-cp-node.sh <node> --reboot # Phase E only (requires --apt reported REBOOT_REQUIRED=yes)
|
||
|
|
# update-cp-node.sh <node> --finalize # Phase F only (uncordon + verify)
|
||
|
|
# update-cp-node.sh <node> --run # all phases with confirmation between each (or ASSUME_YES=1)
|
||
|
|
#
|
||
|
|
# <node> must be one of: k3s-cp-1, k3s-cp-2, k3s-cp-3.
|
||
|
|
#
|
||
|
|
# Environment overrides:
|
||
|
|
# SWAP_SIZE_MB default 4096 (>=2048 required)
|
||
|
|
# SWAP_PATH default /swapfile
|
||
|
|
# DRAIN_TIMEOUT_SECONDS default 600
|
||
|
|
# REBOOT_MAX_WAIT_SECONDS default 600
|
||
|
|
# POST_UNCORDON_WAIT_SECONDS default 180
|
||
|
|
# MIN_TARGET_MEM_MIB default 200 (target-CP MemAvailable floor mid-drain)
|
||
|
|
# MAX_KUBECTL_SECONDS default 5 (kine-latency guardrail; softened for HA etcd)
|
||
|
|
# SSH_OPTS default "-o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
|
||
|
|
# ASSUME_YES=1 skip interactive confirmations in --run
|
||
|
|
# DRY_RUN=1 do not execute state-mutating commands; log-only
|
||
|
|
#
|
||
|
|
# Log directory contract (matches update-node.sh):
|
||
|
|
# Every command's stdout+stderr is tee'd to /tmp/os-update-<node>-<UTC-ts>.log on
|
||
|
|
# the operator machine. Attach it to the execution ticket at the end.
|
||
|
|
#
|
||
|
|
# NEVER touches k3s config, k3s services, containerd, or any manifest. Only
|
||
|
|
# fixes it will attempt: apt/dpkg recovery on the target CP (Phase D). Any other
|
||
|
|
# problem -> escalate and STOP.
|
||
|
|
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# CP topology — public/routable IPs used to reach each CP via ssh + /livez.
|
||
|
|
# Kept out-of-cluster on purpose: during a full-cluster incident the operator
|
||
|
|
# machine must be able to reach each CP without going through k3s.
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
declare -A CP_HOST=(
|
||
|
|
[k3s-cp-1]="178.105.17.239"
|
||
|
|
[k3s-cp-2]="188.245.85.199"
|
||
|
|
[k3s-cp-3]="49.13.92.162"
|
||
|
|
)
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Arg parse — expect <node> as $1
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
usage() { grep -E '^# ' "$0" | sed 's/^# \{0,1\}//'; exit 2; }
|
||
|
|
[ $# -ge 2 ] || usage
|
||
|
|
|
||
|
|
NODE="$1"; shift
|
||
|
|
TARGET_HOST="${CP_HOST[$NODE]:-}"
|
||
|
|
if [ -z "$TARGET_HOST" ]; then
|
||
|
|
echo "unknown CP node: $NODE (allowed: ${!CP_HOST[*]})" >&2
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Config
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
SWAP_SIZE_MB="${SWAP_SIZE_MB:-4096}"
|
||
|
|
SWAP_PATH="${SWAP_PATH:-/swapfile}"
|
||
|
|
DRAIN_TIMEOUT_SECONDS="${DRAIN_TIMEOUT_SECONDS:-600}"
|
||
|
|
REBOOT_MAX_WAIT_SECONDS="${REBOOT_MAX_WAIT_SECONDS:-600}"
|
||
|
|
POST_UNCORDON_WAIT_SECONDS="${POST_UNCORDON_WAIT_SECONDS:-180}"
|
||
|
|
MIN_TARGET_MEM_MIB="${MIN_TARGET_MEM_MIB:-200}"
|
||
|
|
MAX_KUBECTL_SECONDS="${MAX_KUBECTL_SECONDS:-5}"
|
||
|
|
SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new}"
|
||
|
|
DRY_RUN="${DRY_RUN:-0}"
|
||
|
|
ASSUME_YES="${ASSUME_YES:-0}"
|
||
|
|
|
||
|
|
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
||
|
|
LOG_LOCAL="/tmp/os-update-${NODE}-${TS}.log"
|
||
|
|
|
||
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
|
|
HEALTH_SCRIPT="$SCRIPT_DIR/cluster-health.sh"
|
||
|
|
|
||
|
|
# Pick a peer CP for etcd status probes. Requirements:
|
||
|
|
# - not the target
|
||
|
|
# - etcdctl available (via non-login ssh PATH)
|
||
|
|
# We probe each candidate at parse time. Order preference: cp-1 first (it has
|
||
|
|
# etcdctl installed since the pre-HA era), then cp-2, then cp-3. The Phase A
|
||
|
|
# `--add-swap` step also installs `etcd-client` via apt so cp-2/cp-3 pick up
|
||
|
|
# etcdctl on their first pass.
|
||
|
|
peer_host_for() {
|
||
|
|
local target="$1"
|
||
|
|
local order=(k3s-cp-1 k3s-cp-2 k3s-cp-3)
|
||
|
|
for n in "${order[@]}"; do
|
||
|
|
[ "$n" = "$target" ] && continue
|
||
|
|
local h="${CP_HOST[$n]}"
|
||
|
|
if ssh $SSH_OPTS -o BatchMode=yes "root@$h" 'command -v etcdctl >/dev/null 2>&1' 2>/dev/null; then
|
||
|
|
echo "$h"; return 0
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
# In DRY_RUN we don't need a real etcdctl-capable peer.
|
||
|
|
if [ "${DRY_RUN:-0}" = "1" ]; then
|
||
|
|
for n in "${order[@]}"; do
|
||
|
|
[ "$n" != "$target" ] && { echo "${CP_HOST[$n]}"; return 0; }
|
||
|
|
done
|
||
|
|
fi
|
||
|
|
# No non-target CP has etcdctl. Return empty so Phase B/E can fail
|
||
|
|
# explicitly with a targeted "install etcd-client on cp-X first" message.
|
||
|
|
echo ""
|
||
|
|
return 0
|
||
|
|
}
|
||
|
|
PEER_HOST="$(peer_host_for "$NODE")"
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Logging + safe-run helpers
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_LOCAL"; }
|
||
|
|
die() { log "FATAL: $*"; exit 1; }
|
||
|
|
warn() { log "WARN: $*"; }
|
||
|
|
|
||
|
|
run() {
|
||
|
|
if [ "$DRY_RUN" = "1" ]; then
|
||
|
|
log "DRY-RUN would exec: $*"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
log "exec: $*"
|
||
|
|
"$@" 2>&1 | tee -a "$LOG_LOCAL"
|
||
|
|
}
|
||
|
|
|
||
|
|
ssh_run() {
|
||
|
|
local target="${1:-root@$TARGET_HOST}"; shift || true
|
||
|
|
if [ "$DRY_RUN" = "1" ]; then
|
||
|
|
log "DRY-RUN would ssh $target: $*"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
log "ssh $target: $*"
|
||
|
|
ssh $SSH_OPTS "$target" "$@" 2>&1 | tee -a "$LOG_LOCAL"
|
||
|
|
}
|
||
|
|
|
||
|
|
# ssh_run_stdin: pipe a heredoc through bash -s on the target; used for multi-line remote blocks.
|
||
|
|
ssh_run_stdin() {
|
||
|
|
local target="root@${TARGET_HOST}"
|
||
|
|
if [ "$DRY_RUN" = "1" ]; then
|
||
|
|
log "DRY-RUN would ssh $target with stdin script:"
|
||
|
|
sed 's/^/ | /' | tee -a "$LOG_LOCAL"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
log "ssh $target (heredoc)"
|
||
|
|
ssh $SSH_OPTS "$target" "bash -s" 2>&1 | tee -a "$LOG_LOCAL"
|
||
|
|
}
|
||
|
|
|
||
|
|
confirm() {
|
||
|
|
local prompt="$1"
|
||
|
|
if [ "$ASSUME_YES" = "1" ]; then
|
||
|
|
log "confirm SKIPPED (ASSUME_YES=1): $prompt"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
echo -n " >>> $prompt Continue? [y/N] "
|
||
|
|
read -r a
|
||
|
|
case "$a" in y|Y|yes|YES) return 0 ;; *) die "aborted by operator" ;; esac
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# etcd helpers — run etcdctl on the target OR on a peer CP.
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
ETCDCTL_ENV='ETCDCTL_API=3 etcdctl \
|
||
|
|
--endpoints=https://127.0.0.1:2379 \
|
||
|
|
--cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
|
||
|
|
--cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
|
||
|
|
--key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key'
|
||
|
|
|
||
|
|
etcd_status_via() {
|
||
|
|
# $1 = host (IP)
|
||
|
|
local host="$1"
|
||
|
|
ssh $SSH_OPTS "root@$host" "$ETCDCTL_ENV endpoint status --cluster -w table" 2>&1
|
||
|
|
}
|
||
|
|
|
||
|
|
etcd_leader_name() {
|
||
|
|
# returns the etcd member NAME (e.g. k3s-cp-3-9d305472) whose row shows IS LEADER=true.
|
||
|
|
# Uses simple-format output (no -w table) for reliable parsing.
|
||
|
|
local host="$1"
|
||
|
|
ssh $SSH_OPTS "root@$host" "$ETCDCTL_ENV endpoint status --cluster -w simple" 2>/dev/null \
|
||
|
|
| awk -F, '$5 ~ /true/ {print $1}'
|
||
|
|
}
|
||
|
|
|
||
|
|
target_is_leader() {
|
||
|
|
# returns 0 if the target IP is the current etcd leader, 1 otherwise.
|
||
|
|
local host="$1" # peer to query from
|
||
|
|
local target="$2" # target IP to compare
|
||
|
|
local leader_ep
|
||
|
|
leader_ep=$(ssh $SSH_OPTS "root@$host" "$ETCDCTL_ENV endpoint status --cluster -w simple" 2>/dev/null \
|
||
|
|
| awk -F, '$5 ~ /true/ {print $1}')
|
||
|
|
# leader_ep looks like https://49.13.92.162:2379
|
||
|
|
echo "$leader_ep" | grep -q "://${target}:" && return 0 || return 1
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Phase A — Add swap on the target CP (idempotent)
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
phase_add_swap() {
|
||
|
|
log "=== Phase A: add swap on $NODE ($SWAP_SIZE_MB MiB at $SWAP_PATH) ==="
|
||
|
|
[ "$SWAP_SIZE_MB" -ge 2048 ] || die "SWAP_SIZE_MB=$SWAP_SIZE_MB below 2048 MiB guardrail"
|
||
|
|
|
||
|
|
cat <<REMOTE | ssh_run_stdin
|
||
|
|
set -euo pipefail
|
||
|
|
SWAP_PATH="$SWAP_PATH"
|
||
|
|
SIZE_MB="$SWAP_SIZE_MB"
|
||
|
|
|
||
|
|
# Skip if a swapfile at this path is already active.
|
||
|
|
if swapon --show=NAME 2>/dev/null | grep -qx "\$SWAP_PATH"; then
|
||
|
|
echo "swap already on at \$SWAP_PATH -- skipping"
|
||
|
|
free -h
|
||
|
|
exit 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Root filesystem free space check -- abort if less than 2*swap free.
|
||
|
|
avail_mb=\$(df -m --output=avail / | tail -1 | tr -d ' ')
|
||
|
|
need_mb=\$(( SIZE_MB * 2 ))
|
||
|
|
if [ "\$avail_mb" -lt "\$need_mb" ]; then
|
||
|
|
echo "ERROR: only \${avail_mb} MiB free on /, need \${need_mb} MiB (2x swap for safety)"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Create swapfile. fallocate is fast; dd is the fallback.
|
||
|
|
if ! fallocate -l "\${SIZE_MB}M" "\$SWAP_PATH" 2>/dev/null; then
|
||
|
|
dd if=/dev/zero of="\$SWAP_PATH" bs=1M count="\$SIZE_MB" status=progress
|
||
|
|
fi
|
||
|
|
chmod 600 "\$SWAP_PATH"
|
||
|
|
mkswap "\$SWAP_PATH"
|
||
|
|
swapon "\$SWAP_PATH"
|
||
|
|
|
||
|
|
# Persist via fstab (dedup).
|
||
|
|
if ! grep -q "^\$SWAP_PATH " /etc/fstab; then
|
||
|
|
echo "\$SWAP_PATH none swap sw 0 0" >> /etc/fstab
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Moderate swappiness -- swap as safety net, not aggressive paging.
|
||
|
|
sysctl -w vm.swappiness=10
|
||
|
|
if [ ! -f /etc/sysctl.d/99-k3s-swap.conf ] || ! grep -q '^vm.swappiness' /etc/sysctl.d/99-k3s-swap.conf; then
|
||
|
|
echo 'vm.swappiness=10' > /etc/sysctl.d/99-k3s-swap.conf
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "--- swap after ---"
|
||
|
|
free -h
|
||
|
|
swapon --show
|
||
|
|
sysctl vm.swappiness
|
||
|
|
|
||
|
|
# Ensure etcdctl is available for etcd-quorum probes (idempotent apt install).
|
||
|
|
# Needed because when this CP is the target of a later update, another CP
|
||
|
|
# must probe etcd cluster status; if cp-1 is the target, one of cp-2/cp-3
|
||
|
|
# is the probing peer and must have etcdctl.
|
||
|
|
if ! command -v etcdctl >/dev/null 2>&1; then
|
||
|
|
echo "--- installing etcd-client (provides etcdctl) ---"
|
||
|
|
export DEBIAN_FRONTEND=noninteractive
|
||
|
|
apt-get update -y >/dev/null
|
||
|
|
apt-get install -y etcd-client
|
||
|
|
command -v etcdctl && etcdctl version
|
||
|
|
fi
|
||
|
|
REMOTE
|
||
|
|
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
log "verifying kubelet still Ready after swap add"
|
||
|
|
local ready
|
||
|
|
ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo Unknown)
|
||
|
|
[ "$ready" = "True" ] || die "kubelet on $NODE not Ready after swap add -- halt"
|
||
|
|
log "kubelet Ready=True -- Phase A complete"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Guardrail probes
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
kubectl_latency_s() {
|
||
|
|
local start end
|
||
|
|
start=$(date +%s)
|
||
|
|
kubectl get nodes >/dev/null 2>&1 || echo "kubectl-error" >&2
|
||
|
|
end=$(date +%s)
|
||
|
|
echo $(( end - start ))
|
||
|
|
}
|
||
|
|
|
||
|
|
target_mem_available_mib() {
|
||
|
|
ssh $SSH_OPTS "root@$TARGET_HOST" "awk '/^MemAvailable:/{printf \"%d\n\", \$2/1024}' /proc/meminfo" 2>/dev/null || echo 0
|
||
|
|
}
|
||
|
|
|
||
|
|
guard_kine_healthy() {
|
||
|
|
local s
|
||
|
|
s=$(kubectl_latency_s)
|
||
|
|
if [ "$s" -gt "$MAX_KUBECTL_SECONDS" ]; then
|
||
|
|
die "kubectl get nodes took ${s}s (>${MAX_KUBECTL_SECONDS}s) -- etcd slow, HALT"
|
||
|
|
fi
|
||
|
|
log " kubectl-latency ok: ${s}s"
|
||
|
|
}
|
||
|
|
|
||
|
|
guard_target_memory() {
|
||
|
|
local m
|
||
|
|
m=$(target_mem_available_mib)
|
||
|
|
if [ "$m" -lt "$MIN_TARGET_MEM_MIB" ]; then
|
||
|
|
die "$NODE MemAvailable=${m} MiB below ${MIN_TARGET_MEM_MIB} MiB floor -- HALT"
|
||
|
|
fi
|
||
|
|
log " $NODE mem ok: MemAvailable=${m} MiB"
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Phase B — Preflight
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
phase_preflight() {
|
||
|
|
log "=== Phase B: preflight for $NODE ==="
|
||
|
|
|
||
|
|
log "-- cluster health"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
if ! RETRY_ON_TRANSIENT=1 "$HEALTH_SCRIPT" 2>&1 | tee -a "$LOG_LOCAL"; then
|
||
|
|
die "cluster is not healthy -- refuse to start $NODE update"
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
log "DRY-RUN would run: $HEALTH_SCRIPT"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- confirm $NODE is a control-plane node"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
local is_cp
|
||
|
|
is_cp=$(kubectl get node "$NODE" -o jsonpath='{.metadata.labels.node-role\.kubernetes\.io/control-plane}' 2>/dev/null || echo "")
|
||
|
|
[ "$is_cp" = "true" ] || die "$NODE is not labelled control-plane -- refuse (use update-node.sh for workers)"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- verify no OTHER CP is currently cordoned"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
local other_cordoned
|
||
|
|
other_cordoned=$(kubectl get nodes -l node-role.kubernetes.io/control-plane=true \
|
||
|
|
-o json | jq -r --arg n "$NODE" '.items[] | select(.metadata.name != $n) | select(.spec.unschedulable == true) | .metadata.name' \
|
||
|
|
| tr '\n' ' ')
|
||
|
|
if [ -n "${other_cordoned// /}" ]; then
|
||
|
|
die "another CP is already cordoned: $other_cordoned -- refuse (one CP at a time)"
|
||
|
|
fi
|
||
|
|
log " no other CP cordoned -- proceeding"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- etcd cluster status (all members must be started)"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
if [ -z "$PEER_HOST" ]; then
|
||
|
|
die "no non-target CP has etcdctl installed -- run \`update-cp-node.sh <other-cp> --add-swap\` first on one of the OTHER CPs (that step installs etcd-client), then retry"
|
||
|
|
fi
|
||
|
|
etcd_status_via "$PEER_HOST" | tee -a "$LOG_LOCAL"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- swap on $NODE"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
local swap_total
|
||
|
|
swap_total=$(ssh $SSH_OPTS "root@$TARGET_HOST" "awk '/^SwapTotal:/{print \$2}' /proc/meminfo")
|
||
|
|
[ "${swap_total:-0}" -ge $((2 * 1024 * 1024)) ] \
|
||
|
|
|| die "$NODE SwapTotal=${swap_total} KiB below 2 GiB -- run --add-swap first"
|
||
|
|
log " $NODE SwapTotal=$(( swap_total / 1024 )) MiB"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- kubectl-latency probe"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then guard_kine_healthy; fi
|
||
|
|
|
||
|
|
log "-- record current etcd leader"
|
||
|
|
if [ "$DRY_RUN" != "1" ] && [ -n "$PEER_HOST" ]; then
|
||
|
|
if target_is_leader "$PEER_HOST" "$TARGET_HOST"; then
|
||
|
|
warn "$NODE IS the current etcd leader. Per CP ordering rule, prefer updating a follower first."
|
||
|
|
warn " Not aborting -- operator/agent must confirm this is intentional."
|
||
|
|
else
|
||
|
|
log " $NODE is a FOLLOWER -- safe to proceed."
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- k3s etcd snapshot"
|
||
|
|
ssh_run "root@$TARGET_HOST" "k3s etcd-snapshot save --name pre-cp-os-update-${NODE}-${TS}"
|
||
|
|
ssh_run "root@$TARGET_HOST" "ls -la /var/lib/rancher/k3s/server/db/snapshots/ | tail -10"
|
||
|
|
|
||
|
|
log "=== Phase B: preflight OK ==="
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Phase C — Cordon + drain
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
phase_drain() {
|
||
|
|
log "=== Phase C: cordon + drain $NODE ==="
|
||
|
|
|
||
|
|
log "-- cordon $NODE"
|
||
|
|
run kubectl cordon "$NODE"
|
||
|
|
|
||
|
|
log "-- drain $NODE (timeout ${DRAIN_TIMEOUT_SECONDS}s)"
|
||
|
|
set +e
|
||
|
|
if [ "$DRY_RUN" = "1" ]; then
|
||
|
|
log "DRY-RUN would run: kubectl drain $NODE --ignore-daemonsets --delete-emptydir-data --timeout=${DRAIN_TIMEOUT_SECONDS}s"
|
||
|
|
local rc=0
|
||
|
|
else
|
||
|
|
kubectl drain "$NODE" \
|
||
|
|
--ignore-daemonsets \
|
||
|
|
--delete-emptydir-data \
|
||
|
|
--timeout="${DRAIN_TIMEOUT_SECONDS}s" 2>&1 | tee -a "$LOG_LOCAL"
|
||
|
|
local rc=${PIPESTATUS[0]}
|
||
|
|
fi
|
||
|
|
set -e
|
||
|
|
|
||
|
|
if [ "$rc" -ne 0 ]; then
|
||
|
|
log "drain FAILED (rc=$rc). Never force. Uncordoning."
|
||
|
|
run kubectl uncordon "$NODE"
|
||
|
|
die "drain failed on $NODE -- investigate PDB / orphan pods; do NOT proceed"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- post-drain guardrails"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
guard_kine_healthy
|
||
|
|
guard_target_memory
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "=== Phase C: $NODE drained ==="
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Phase D — apt on the target
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
phase_apt() {
|
||
|
|
log "=== Phase D: apt on $NODE ==="
|
||
|
|
|
||
|
|
cat <<'REMOTE' | ssh_run_stdin
|
||
|
|
set -euo pipefail
|
||
|
|
export DEBIAN_FRONTEND=noninteractive
|
||
|
|
APT_OPTS='-y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold'
|
||
|
|
|
||
|
|
uname -r > /root/pre-apt-kernel
|
||
|
|
dpkg-query -W -f='${Package}\t${Version}\n' > /root/pre-apt-packages.tsv
|
||
|
|
echo "pre-apt kernel: $(cat /root/pre-apt-kernel)"
|
||
|
|
|
||
|
|
if dpkg --audit | grep -qE .; then
|
||
|
|
echo "dpkg audit reported issues, running dpkg --configure -a"
|
||
|
|
dpkg --configure -a || true
|
||
|
|
fi
|
||
|
|
|
||
|
|
apt-get update
|
||
|
|
|
||
|
|
if ! apt-get $APT_OPTS upgrade; then
|
||
|
|
echo "upgrade failed, attempting apt-get -f install"
|
||
|
|
apt-get $APT_OPTS -f install
|
||
|
|
apt-get $APT_OPTS upgrade
|
||
|
|
fi
|
||
|
|
|
||
|
|
apt-get $APT_OPTS dist-upgrade
|
||
|
|
apt-get $APT_OPTS autoremove --purge
|
||
|
|
apt-get clean
|
||
|
|
|
||
|
|
if [ -f /var/run/reboot-required ]; then
|
||
|
|
echo "REBOOT_REQUIRED=yes"
|
||
|
|
echo "REBOOT_REASON<<EOF"
|
||
|
|
cat /var/run/reboot-required.pkgs 2>/dev/null || echo "(no package list)"
|
||
|
|
echo "EOF"
|
||
|
|
else
|
||
|
|
echo "REBOOT_REQUIRED=no"
|
||
|
|
fi
|
||
|
|
REMOTE
|
||
|
|
|
||
|
|
log "=== Phase D: apt complete (check REBOOT_REQUIRED in the log) ==="
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Phase E — Reboot and wait for target /livez + kubelet Ready + etcd rejoin
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
phase_reboot() {
|
||
|
|
log "=== Phase E: reboot $NODE ==="
|
||
|
|
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
log "issuing 'systemctl reboot' on $NODE (ssh will drop; expected)"
|
||
|
|
ssh $SSH_OPTS "root@$TARGET_HOST" 'systemctl reboot' 2>&1 | tee -a "$LOG_LOCAL" || true
|
||
|
|
log "waiting 15s for ssh to fully drop before polling"
|
||
|
|
sleep 15
|
||
|
|
else
|
||
|
|
log "DRY-RUN would ssh root@$TARGET_HOST 'systemctl reboot'"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- poll $NODE api-server /livez (timeout ${REBOOT_MAX_WAIT_SECONDS}s)"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
local deadline=$(( $(date +%s) + REBOOT_MAX_WAIT_SECONDS ))
|
||
|
|
local code=000
|
||
|
|
while [ $(date +%s) -lt $deadline ]; do
|
||
|
|
code=$(curl -sk -o /dev/null -w '%{http_code}' "https://$TARGET_HOST:6443/livez" 2>/dev/null || echo 000)
|
||
|
|
if [ "$code" = "200" ]; then
|
||
|
|
log " $NODE api-server /livez=200"
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
sleep 5
|
||
|
|
done
|
||
|
|
[ "$code" = "200" ] || die "$NODE api-server did not return within ${REBOOT_MAX_WAIT_SECONDS}s -- escalate; check 'hcloud server describe $NODE' and Hetzner console"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- verify etcd cluster status from peer ($PEER_HOST) -- $NODE should be 'started'"
|
||
|
|
if [ "$DRY_RUN" != "1" ] && [ -n "$PEER_HOST" ]; then
|
||
|
|
etcd_status_via "$PEER_HOST" | tee -a "$LOG_LOCAL"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- wait for kubelet Ready on $NODE (max 300s)"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
local deadline=$(( $(date +%s) + 300 ))
|
||
|
|
local ready=Unknown
|
||
|
|
while [ $(date +%s) -lt $deadline ]; do
|
||
|
|
ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo Unknown)
|
||
|
|
[ "$ready" = "True" ] && break
|
||
|
|
sleep 5
|
||
|
|
done
|
||
|
|
[ "$ready" = "True" ] || die "kubelet on $NODE never returned Ready -- escalate (do NOT change k3s config)"
|
||
|
|
log " kubelet Ready=True"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "=== Phase E: $NODE is back ==="
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Phase F — Uncordon + verify + finalize
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
phase_finalize() {
|
||
|
|
log "=== Phase F: uncordon + verify $NODE ==="
|
||
|
|
|
||
|
|
run kubectl uncordon "$NODE"
|
||
|
|
|
||
|
|
log "-- settle wait ${POST_UNCORDON_WAIT_SECONDS}s"
|
||
|
|
[ "$DRY_RUN" = "1" ] || sleep "$POST_UNCORDON_WAIT_SECONDS"
|
||
|
|
|
||
|
|
log "-- cluster health"
|
||
|
|
if [ "$DRY_RUN" != "1" ]; then
|
||
|
|
if ! RETRY_ON_TRANSIENT=1 "$HEALTH_SCRIPT" 2>&1 | tee -a "$LOG_LOCAL"; then
|
||
|
|
die "cluster health failed after $NODE update -- escalate, do NOT touch k3s"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- etcd cluster status (all three should be started)"
|
||
|
|
if [ "$DRY_RUN" != "1" ] && [ -n "$PEER_HOST" ]; then
|
||
|
|
etcd_status_via "$PEER_HOST" | tee -a "$LOG_LOCAL"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "-- apt history summary (audit)"
|
||
|
|
ssh_run "root@$TARGET_HOST" 'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -60'
|
||
|
|
|
||
|
|
log "-- old snapshots (>30d) -- listing only, review manually"
|
||
|
|
ssh_run "root@$TARGET_HOST" 'find /var/lib/rancher/k3s/server/db/snapshots/ -type f -mtime +30 -name "pre-*" -print 2>/dev/null || true'
|
||
|
|
|
||
|
|
log "=== $NODE OS update complete -- attach $LOG_LOCAL to the execution ticket ==="
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# --run — orchestrate all phases with confirmations
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
phase_run_all() {
|
||
|
|
log "=== full $NODE update run (log: $LOG_LOCAL) ==="
|
||
|
|
|
||
|
|
confirm "Phase A (add swap) -- proceed?"
|
||
|
|
phase_add_swap
|
||
|
|
|
||
|
|
confirm "Phase B (preflight) -- proceed?"
|
||
|
|
phase_preflight
|
||
|
|
|
||
|
|
confirm "Phase C (cordon + drain) -- proceed?"
|
||
|
|
phase_drain
|
||
|
|
|
||
|
|
confirm "Phase D (apt) -- proceed?"
|
||
|
|
phase_apt
|
||
|
|
|
||
|
|
confirm "Phase E (reboot $NODE; api-server on THIS node unavailable ~90-180s, other 2 CPs keep serving) -- proceed?"
|
||
|
|
phase_reboot
|
||
|
|
|
||
|
|
confirm "Phase F (uncordon + verify) -- proceed?"
|
||
|
|
phase_finalize
|
||
|
|
|
||
|
|
log "=== FULL RUN COMPLETE for $NODE ==="
|
||
|
|
}
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
# Phase dispatch
|
||
|
|
# --------------------------------------------------------------------------- #
|
||
|
|
: > "$LOG_LOCAL"
|
||
|
|
log "update-cp-node.sh started (NODE=$NODE, TARGET_HOST=$TARGET_HOST, PEER_HOST=$PEER_HOST, DRY_RUN=$DRY_RUN)"
|
||
|
|
log "log file: $LOG_LOCAL"
|
||
|
|
|
||
|
|
case "$1" in
|
||
|
|
--dry-run)
|
||
|
|
DRY_RUN=1
|
||
|
|
export DRY_RUN
|
||
|
|
log "DRY_RUN=1 -- walking Phases A..F without touching state"
|
||
|
|
phase_add_swap
|
||
|
|
phase_preflight
|
||
|
|
phase_drain
|
||
|
|
phase_apt
|
||
|
|
phase_reboot
|
||
|
|
phase_finalize
|
||
|
|
;;
|
||
|
|
--add-swap) phase_add_swap ;;
|
||
|
|
--preflight) phase_preflight ;;
|
||
|
|
--drain) phase_drain ;;
|
||
|
|
--apt) phase_apt ;;
|
||
|
|
--reboot) phase_reboot ;;
|
||
|
|
--finalize) phase_finalize ;;
|
||
|
|
--run) phase_run_all ;;
|
||
|
|
-h|--help) usage ;;
|
||
|
|
*) echo "unknown arg: $1" >&2; usage ;;
|
||
|
|
esac
|