05000b3e06
cp-1 is excluded from os-update.sh because a one-shot drain would trigger the kine cascade documented in DEV-495. This adds: - CP1_UPDATE_PROCEDURE.md: swap-add (Phase A), preflight, batched stateful eviction (Phase C), drain+apt (Phase D), reboot with external livez monitor (Phase E), uncordon+verify (Phase F), and rollback paths. - scripts/os-update/update-cp-1.sh: subcommand-per-phase runner with the same /tmp/os-update-cp-1-<ts>.log contract as update-node.sh; supports --dry-run, --add-swap, --preflight, --drain-stateful, --apt, --reboot, --finalize, --run. - os-update.sh: explicitly excludes k3s-cp-1 with a pointer to the cp-1 script; kine thundering-herd guardrails preserved. - OS_UPDATE_PROCEDURE.md: cross-reference to the cp-1 procedure. Execution requires separate board approval; this change is design + dry-run artifact only. Co-Authored-By: Paperclip <noreply@paperclip.ing>
577 lines
21 KiB
Bash
Executable file
577 lines
21 KiB
Bash
Executable file
#!/bin/bash
|
|
# update-cp-1.sh — controlled OS update for the k3s control-plane node.
|
|
#
|
|
# See ../CP1_UPDATE_PROCEDURE.md for the full design rationale.
|
|
#
|
|
# Usage:
|
|
# update-cp-1.sh --dry-run # print what would be done, touch nothing
|
|
# update-cp-1.sh --add-swap # Phase A only (idempotent, safe standalone)
|
|
# update-cp-1.sh --preflight # Phase B only
|
|
# update-cp-1.sh --drain-stateful # Phase C only (cordon + batched sts moves)
|
|
# update-cp-1.sh --apt # Phase D only (requires cp-1 fully drained)
|
|
# update-cp-1.sh --reboot # Phase E only (requires --apt reported REBOOT_REQUIRED=yes)
|
|
# update-cp-1.sh --finalize # Phase F only (uncordon + verify)
|
|
# update-cp-1.sh --run # all phases with a confirmation between each (or ASSUME_YES=1)
|
|
#
|
|
# Environment overrides:
|
|
# CP1_HOST default 178.105.17.239
|
|
# SWAP_SIZE_MB default 4096 (>=2048 required)
|
|
# SWAP_PATH default /swapfile
|
|
# DRAIN_TIMEOUT_SECONDS default 600
|
|
# REBOOT_MAX_WAIT_SECONDS default 600
|
|
# POST_UNCORDON_WAIT_SECONDS default 180
|
|
# STATEFUL_SETTLE_SECONDS default 60 (pause between batched sts moves)
|
|
# MIN_WORKER_MEM_MIB default 500 (per-worker MemAvailable floor at preflight)
|
|
# MIN_CP1_MEM_MIB default 200 (cp-1 MemAvailable floor mid-drain)
|
|
# MAX_KUBECTL_SECONDS default 5 (kine-health guardrail)
|
|
# SSH_OPTS default "-o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
|
|
# ASSUME_YES=1 skip interactive confirmations in --run
|
|
# DRY_RUN=1 do not execute state-mutating commands; log-only
|
|
#
|
|
# Log directory contract (matches update-node.sh):
|
|
# Every command's stdout+stderr is tee'd to /tmp/os-update-cp-1-<UTC-ts>.log on
|
|
# the operator machine AND to /tmp/os-update-cp-1-<UTC-ts>.log on cp-1 (via the
|
|
# ssh command wrappers). Attach both to the execution ticket at the end.
|
|
#
|
|
# NEVER touches k3s config, k3s services, containerd, or any manifest. Only
|
|
# fixes it will attempt: apt/dpkg recovery on cp-1 (see Phase D). Any other
|
|
# problem → escalate and STOP.
|
|
|
|
set -euo pipefail
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Config
|
|
# --------------------------------------------------------------------------- #
|
|
CP1_HOST="${CP1_HOST:-178.105.17.239}"
|
|
SWAP_SIZE_MB="${SWAP_SIZE_MB:-4096}"
|
|
SWAP_PATH="${SWAP_PATH:-/swapfile}"
|
|
DRAIN_TIMEOUT_SECONDS="${DRAIN_TIMEOUT_SECONDS:-600}"
|
|
REBOOT_MAX_WAIT_SECONDS="${REBOOT_MAX_WAIT_SECONDS:-600}"
|
|
POST_UNCORDON_WAIT_SECONDS="${POST_UNCORDON_WAIT_SECONDS:-180}"
|
|
STATEFUL_SETTLE_SECONDS="${STATEFUL_SETTLE_SECONDS:-60}"
|
|
MIN_WORKER_MEM_MIB="${MIN_WORKER_MEM_MIB:-500}"
|
|
MIN_CP1_MEM_MIB="${MIN_CP1_MEM_MIB:-200}"
|
|
MAX_KUBECTL_SECONDS="${MAX_KUBECTL_SECONDS:-5}"
|
|
SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new}"
|
|
DRY_RUN="${DRY_RUN:-0}"
|
|
ASSUME_YES="${ASSUME_YES:-0}"
|
|
|
|
NODE="k3s-cp-1"
|
|
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
LOG_LOCAL="/tmp/os-update-cp-1-${TS}.log"
|
|
LOG_REMOTE="/tmp/os-update-cp-1-${TS}.log"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
HEALTH_SCRIPT="$SCRIPT_DIR/cluster-health.sh"
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Stateful eviction ledger — order matters (idle -> active, small -> heavy).
|
|
# --------------------------------------------------------------------------- #
|
|
# The list is derived live from the cluster in preflight and stored to
|
|
# /tmp/cp1-stateful-plan-<ts>.txt. This ORDER is the authoritative fallback.
|
|
STATEFUL_ORDER=(
|
|
"harbor:harbor-redis-0"
|
|
"nextcloud:nextcloud-redis-replicas-0"
|
|
"harbor:harbor-database-0"
|
|
"stalwart:stalwart-postgres-0"
|
|
"stalwart:stalwart-0"
|
|
)
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Logging + safe-run helpers
|
|
# --------------------------------------------------------------------------- #
|
|
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_LOCAL"; }
|
|
die() { log "FATAL: $*"; exit 1; }
|
|
warn() { log "WARN: $*"; }
|
|
|
|
# run/ssh_run/kubectl_run: honour DRY_RUN — print, do not execute state changes.
|
|
run() {
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
log "DRY-RUN would exec: $*"
|
|
return 0
|
|
fi
|
|
log "exec: $*"
|
|
"$@" 2>&1 | tee -a "$LOG_LOCAL"
|
|
}
|
|
|
|
ssh_run() {
|
|
local target="root@${CP1_HOST}"
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
log "DRY-RUN would ssh $target: $*"
|
|
return 0
|
|
fi
|
|
log "ssh $target: $*"
|
|
ssh $SSH_OPTS "$target" "$@" 2>&1 | tee -a "$LOG_LOCAL"
|
|
}
|
|
|
|
# ssh_run_stdin: pipe a heredoc through bash -s on cp-1; used for multi-line remote
|
|
# blocks (swap add, apt).
|
|
ssh_run_stdin() {
|
|
local target="root@${CP1_HOST}"
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
log "DRY-RUN would ssh $target with stdin script:"
|
|
sed 's/^/ | /' | tee -a "$LOG_LOCAL"
|
|
return 0
|
|
fi
|
|
log "ssh $target (heredoc)"
|
|
ssh $SSH_OPTS "$target" "bash -s" 2>&1 | tee -a "$LOG_LOCAL"
|
|
}
|
|
|
|
confirm() {
|
|
local prompt="$1"
|
|
if [ "$ASSUME_YES" = "1" ]; then
|
|
log "confirm SKIPPED (ASSUME_YES=1): $prompt"
|
|
return 0
|
|
fi
|
|
echo -n " >>> $prompt Continue? [y/N] "
|
|
read -r a
|
|
case "$a" in y|Y|yes|YES) return 0 ;; *) die "aborted by operator" ;; esac
|
|
}
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Phase A — Add swap on cp-1 (idempotent)
|
|
# --------------------------------------------------------------------------- #
|
|
phase_add_swap() {
|
|
log "=== Phase A: add swap on $NODE ($SWAP_SIZE_MB MiB at $SWAP_PATH) ==="
|
|
[ "$SWAP_SIZE_MB" -ge 2048 ] || die "SWAP_SIZE_MB=$SWAP_SIZE_MB below 2048 MiB guardrail"
|
|
|
|
cat <<REMOTE | ssh_run_stdin
|
|
set -euo pipefail
|
|
SWAP_PATH="$SWAP_PATH"
|
|
SIZE_MB="$SWAP_SIZE_MB"
|
|
|
|
# Skip if a swapfile at this path is already active.
|
|
if swapon --show=NAME 2>/dev/null | grep -qx "\$SWAP_PATH"; then
|
|
echo "swap already on at \$SWAP_PATH — skipping"
|
|
free -h
|
|
exit 0
|
|
fi
|
|
|
|
# Root filesystem free space check — abort if less than 2*swap free.
|
|
avail_mb=\$(df -m --output=avail / | tail -1 | tr -d ' ')
|
|
need_mb=\$(( SIZE_MB * 2 ))
|
|
if [ "\$avail_mb" -lt "\$need_mb" ]; then
|
|
echo "ERROR: only \${avail_mb} MiB free on /, need \${need_mb} MiB (2x swap for safety)"
|
|
exit 1
|
|
fi
|
|
|
|
# Create swapfile. fallocate is fast; dd is the fallback.
|
|
if ! fallocate -l "\${SIZE_MB}M" "\$SWAP_PATH" 2>/dev/null; then
|
|
dd if=/dev/zero of="\$SWAP_PATH" bs=1M count="\$SIZE_MB" status=progress
|
|
fi
|
|
chmod 600 "\$SWAP_PATH"
|
|
mkswap "\$SWAP_PATH"
|
|
swapon "\$SWAP_PATH"
|
|
|
|
# Persist via fstab (dedup).
|
|
if ! grep -q "^\$SWAP_PATH " /etc/fstab; then
|
|
echo "\$SWAP_PATH none swap sw 0 0" >> /etc/fstab
|
|
fi
|
|
|
|
# Moderate swappiness — swap as safety net, not aggressive paging.
|
|
sysctl -w vm.swappiness=10
|
|
if [ ! -f /etc/sysctl.d/99-k3s-swap.conf ] || ! grep -q '^vm.swappiness' /etc/sysctl.d/99-k3s-swap.conf; then
|
|
echo 'vm.swappiness=10' > /etc/sysctl.d/99-k3s-swap.conf
|
|
fi
|
|
|
|
echo "--- swap after ---"
|
|
free -h
|
|
swapon --show
|
|
sysctl vm.swappiness
|
|
REMOTE
|
|
|
|
# Verify from operator view.
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
log "verifying kubelet still Ready after swap add"
|
|
local ready
|
|
ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')
|
|
[ "$ready" = "True" ] || die "kubelet on $NODE not Ready after swap add — halt"
|
|
log "kubelet Ready=True — Phase A complete"
|
|
fi
|
|
}
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Guardrail probes
|
|
# --------------------------------------------------------------------------- #
|
|
kubectl_latency_s() {
|
|
# returns integer seconds elapsed for `kubectl get nodes >/dev/null`.
|
|
local start end
|
|
start=$(date +%s)
|
|
kubectl get nodes >/dev/null 2>&1 || echo "kubectl-error" >&2
|
|
end=$(date +%s)
|
|
echo $(( end - start ))
|
|
}
|
|
|
|
cp1_mem_available_mib() {
|
|
# returns integer MiB `MemAvailable` on cp-1.
|
|
ssh $SSH_OPTS "root@$CP1_HOST" "awk '/^MemAvailable:/{printf \"%d\n\", \$2/1024}' /proc/meminfo" 2>/dev/null || echo 0
|
|
}
|
|
|
|
guard_kine_healthy() {
|
|
local s
|
|
s=$(kubectl_latency_s)
|
|
if [ "$s" -gt "$MAX_KUBECTL_SECONDS" ]; then
|
|
die "kubectl get nodes took ${s}s (>${MAX_KUBECTL_SECONDS}s) — kine cascade risk, HALT"
|
|
fi
|
|
log " kine ok: kubectl get nodes took ${s}s"
|
|
}
|
|
|
|
guard_cp1_memory() {
|
|
local m
|
|
m=$(cp1_mem_available_mib)
|
|
if [ "$m" -lt "$MIN_CP1_MEM_MIB" ]; then
|
|
die "cp-1 MemAvailable=${m} MiB below ${MIN_CP1_MEM_MIB} MiB floor — HALT"
|
|
fi
|
|
log " cp-1 mem ok: MemAvailable=${m} MiB"
|
|
}
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Phase B — Preflight
|
|
# --------------------------------------------------------------------------- #
|
|
phase_preflight() {
|
|
log "=== Phase B: preflight ==="
|
|
|
|
log "-- cluster health"
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
if ! RETRY_ON_TRANSIENT=1 "$HEALTH_SCRIPT" 2>&1 | tee -a "$LOG_LOCAL"; then
|
|
die "cluster is not healthy — refuse to start cp-1 update"
|
|
fi
|
|
else
|
|
log "DRY-RUN would run: $HEALTH_SCRIPT"
|
|
fi
|
|
|
|
log "-- swap on cp-1"
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
local swap_total
|
|
swap_total=$(ssh $SSH_OPTS "root@$CP1_HOST" "awk '/^SwapTotal:/{print \$2}' /proc/meminfo")
|
|
[ "${swap_total:-0}" -ge $((2 * 1024 * 1024)) ] \
|
|
|| die "cp-1 SwapTotal=${swap_total} KiB below 2 GiB — run --add-swap first"
|
|
log " cp-1 SwapTotal=$(( swap_total / 1024 )) MiB"
|
|
fi
|
|
|
|
log "-- fsn1 workers Ready and have >=${MIN_WORKER_MEM_MIB} MiB MemAvailable"
|
|
# Worker private IPs (worker-4 nbg1 excluded — cannot host fsn1 RWO PVs).
|
|
local ips=("10.42.1.2" "10.42.1.3" "10.42.1.5" "10.42.1.7")
|
|
local names=("k3s-worker-1" "k3s-worker-2" "k3s-worker-3" "k3s-worker-5")
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
for i in "${!ips[@]}"; do
|
|
local nm="${names[$i]}"
|
|
local ip="${ips[$i]}"
|
|
local ready
|
|
ready=$(kubectl get node "$nm" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo Unknown)
|
|
[ "$ready" = "True" ] || die "worker $nm not Ready (Ready=$ready)"
|
|
local mem
|
|
mem=$(ssh $SSH_OPTS -J "root@$CP1_HOST" "root@$ip" \
|
|
"awk '/^MemAvailable:/{printf \"%d\n\", \$2/1024}' /proc/meminfo" 2>/dev/null || echo 0)
|
|
[ "$mem" -ge "$MIN_WORKER_MEM_MIB" ] \
|
|
|| die "worker $nm MemAvailable=${mem} MiB below ${MIN_WORKER_MEM_MIB} MiB — HALT"
|
|
log " $nm ok: MemAvailable=${mem} MiB"
|
|
done
|
|
fi
|
|
|
|
log "-- kine latency probe"
|
|
if [ "$DRY_RUN" != "1" ]; then guard_kine_healthy; fi
|
|
|
|
log "-- k3s SQLite datastore snapshot"
|
|
# k3s etcd-snapshot in embedded-SQLite mode copies the sqlite file to
|
|
# /var/lib/rancher/k3s/server/db/snapshots/.
|
|
ssh_run "k3s etcd-snapshot save --name pre-cp1-os-update-${TS}"
|
|
ssh_run "ls -la /var/lib/rancher/k3s/server/db/snapshots/ | tail -10"
|
|
|
|
log "-- persist eviction plan"
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
local plan="/tmp/cp1-stateful-plan-${TS}.txt"
|
|
ssh $SSH_OPTS "root@$CP1_HOST" "cat > $plan" <<EOF
|
|
# cp-1 stateful eviction plan generated at $TS
|
|
# Order: idle -> active, small -> heavy. Do NOT reorder without design review.
|
|
$(for e in "${STATEFUL_ORDER[@]}"; do echo "$e"; done)
|
|
EOF
|
|
log " wrote $plan on cp-1"
|
|
else
|
|
log "DRY-RUN would write /tmp/cp1-stateful-plan-${TS}.txt with STATEFUL_ORDER"
|
|
fi
|
|
|
|
log "=== Phase B: preflight OK ==="
|
|
}
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Phase C — Cordon + move stateful pods off cp-1, one at a time
|
|
# --------------------------------------------------------------------------- #
|
|
phase_drain_stateful() {
|
|
log "=== Phase C: cordon + batched stateful move ==="
|
|
|
|
log "-- cordon $NODE (prevents rescheduled pods from landing back on cp-1)"
|
|
run kubectl cordon "$NODE"
|
|
|
|
local i=0
|
|
for entry in "${STATEFUL_ORDER[@]}"; do
|
|
i=$((i + 1))
|
|
local ns="${entry%%:*}"
|
|
local pod="${entry##*:}"
|
|
|
|
log "--- [${i}/${#STATEFUL_ORDER[@]}] moving $ns/$pod"
|
|
|
|
# Confirm the pod actually IS on cp-1 before touching it.
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
local on
|
|
on=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.nodeName}' 2>/dev/null || echo "")
|
|
if [ -z "$on" ]; then
|
|
log " $ns/$pod not found — sts may have changed; skipping"
|
|
continue
|
|
fi
|
|
if [ "$on" != "$NODE" ]; then
|
|
log " $ns/$pod already on $on (not cp-1) — skipping"
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
# Delete the pod. StatefulSet controller will re-create it; scheduler will
|
|
# pick a fsn1 worker because cp-1 is cordoned and worker-4 is nbg1.
|
|
run kubectl -n "$ns" delete pod "$pod" --wait=false
|
|
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
log " DRY-RUN skipping wait-for-ready"
|
|
continue
|
|
fi
|
|
|
|
log " waiting for $ns/$pod to be Ready on a non-cp-1 node (max 300s)"
|
|
local deadline=$(( $(date +%s) + 300 ))
|
|
local new_node="" ready=""
|
|
while [ $(date +%s) -lt $deadline ]; do
|
|
new_node=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.nodeName}' 2>/dev/null || echo "")
|
|
ready=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "")
|
|
if [ -n "$new_node" ] && [ "$new_node" != "$NODE" ] && [ "$ready" = "True" ]; then
|
|
log " $ns/$pod -> $new_node OK"
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
[ "$ready" = "True" ] && [ "$new_node" != "$NODE" ] \
|
|
|| die "$ns/$pod did not reach Ready on a non-cp-1 node in 300s — HALT (leave cp-1 cordoned)"
|
|
|
|
log " guardrails after $ns/$pod"
|
|
guard_kine_healthy
|
|
guard_cp1_memory
|
|
|
|
if [ "$i" -lt "${#STATEFUL_ORDER[@]}" ]; then
|
|
log " settle pause ${STATEFUL_SETTLE_SECONDS}s"
|
|
sleep "$STATEFUL_SETTLE_SECONDS"
|
|
fi
|
|
done
|
|
|
|
log "=== Phase C: all stateful pods evicted from cp-1 ==="
|
|
}
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Phase D — Drain remaining pods + apt on cp-1
|
|
# --------------------------------------------------------------------------- #
|
|
phase_apt() {
|
|
log "=== Phase D: drain (remaining deployments) + apt on $NODE ==="
|
|
|
|
# Guardrail: refuse to run if any StatefulSet pod is still on cp-1.
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
local sts_on_cp1
|
|
sts_on_cp1=$(kubectl get pods -A -o json --field-selector spec.nodeName=$NODE \
|
|
| jq -r '.items[] | select(.metadata.ownerReferences[0].kind=="StatefulSet") | "\(.metadata.namespace)/\(.metadata.name)"' \
|
|
| wc -l)
|
|
[ "$sts_on_cp1" -eq 0 ] \
|
|
|| die "$sts_on_cp1 StatefulSet pod(s) still on cp-1 — Phase C incomplete; refuse Phase D"
|
|
fi
|
|
|
|
log "-- drain (timeout ${DRAIN_TIMEOUT_SECONDS}s)"
|
|
set +e
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
log "DRY-RUN would run: kubectl drain $NODE --ignore-daemonsets --delete-emptydir-data --timeout=${DRAIN_TIMEOUT_SECONDS}s"
|
|
local rc=0
|
|
else
|
|
kubectl drain "$NODE" \
|
|
--ignore-daemonsets \
|
|
--delete-emptydir-data \
|
|
--timeout="${DRAIN_TIMEOUT_SECONDS}s" 2>&1 | tee -a "$LOG_LOCAL"
|
|
local rc=${PIPESTATUS[0]}
|
|
fi
|
|
set -e
|
|
|
|
if [ "$rc" -ne 0 ]; then
|
|
log "drain FAILED (rc=$rc). Never force. Uncordoning."
|
|
run kubectl uncordon "$NODE"
|
|
die "drain failed on $NODE — investigate PDB / orphan pods; do NOT proceed"
|
|
fi
|
|
|
|
log "-- apt on cp-1"
|
|
cat <<'REMOTE' | ssh_run_stdin
|
|
set -euo pipefail
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
APT_OPTS='-y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold'
|
|
|
|
# Rollback references (Phase-A didn't need this, but Phase D does).
|
|
uname -r > /root/pre-apt-kernel
|
|
dpkg-query -W -f='${Package}\t${Version}\n' > /root/pre-apt-packages.tsv
|
|
echo "pre-apt kernel: $(cat /root/pre-apt-kernel)"
|
|
|
|
# Recover from any half-finished dpkg state before touching apt.
|
|
if dpkg --audit | grep -qE .; then
|
|
echo "dpkg audit reported issues, running dpkg --configure -a"
|
|
dpkg --configure -a || true
|
|
fi
|
|
|
|
apt-get update
|
|
|
|
if ! apt-get $APT_OPTS upgrade; then
|
|
echo "upgrade failed, attempting apt-get -f install"
|
|
apt-get $APT_OPTS -f install
|
|
apt-get $APT_OPTS upgrade
|
|
fi
|
|
|
|
apt-get $APT_OPTS dist-upgrade
|
|
apt-get $APT_OPTS autoremove --purge
|
|
apt-get clean
|
|
|
|
if [ -f /var/run/reboot-required ]; then
|
|
echo "REBOOT_REQUIRED=yes"
|
|
echo "REBOOT_REASON<<EOF"
|
|
cat /var/run/reboot-required.pkgs 2>/dev/null || echo "(no package list)"
|
|
echo "EOF"
|
|
else
|
|
echo "REBOOT_REQUIRED=no"
|
|
fi
|
|
REMOTE
|
|
|
|
log "=== Phase D: apt complete (check REBOOT_REQUIRED in the log) ==="
|
|
}
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Phase E — Reboot cp-1 and wait for api-server /livez
|
|
# --------------------------------------------------------------------------- #
|
|
phase_reboot() {
|
|
log "=== Phase E: reboot $NODE ==="
|
|
|
|
# Tell cp-1 to reboot. The ssh command will hang up mid-command — that's fine.
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
log "issuing 'systemctl reboot' on $NODE (ssh will drop; expected)"
|
|
ssh $SSH_OPTS "root@$CP1_HOST" 'systemctl reboot' 2>&1 | tee -a "$LOG_LOCAL" || true
|
|
log "waiting 15s for ssh to fully drop before polling api-server"
|
|
sleep 15
|
|
else
|
|
log "DRY-RUN would ssh root@$CP1_HOST 'systemctl reboot'"
|
|
fi
|
|
|
|
log "-- poll api-server /livez (timeout ${REBOOT_MAX_WAIT_SECONDS}s)"
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
local deadline=$(( $(date +%s) + REBOOT_MAX_WAIT_SECONDS ))
|
|
local code=000
|
|
while [ $(date +%s) -lt $deadline ]; do
|
|
code=$(curl -sk -o /dev/null -w '%{http_code}' "https://$CP1_HOST:6443/livez" 2>/dev/null || echo 000)
|
|
if [ "$code" = "200" ]; then
|
|
log " api-server /livez=200 (elapsed $(( REBOOT_MAX_WAIT_SECONDS - (deadline - $(date +%s)) ))s)"
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
[ "$code" = "200" ] || die "api-server did not return within ${REBOOT_MAX_WAIT_SECONDS}s — escalate; check 'hcloud server describe k3s-cp-1' and Hetzner console"
|
|
fi
|
|
|
|
log "-- wait for kubelet Ready on $NODE (max 300s)"
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
local deadline=$(( $(date +%s) + 300 ))
|
|
local ready=Unknown
|
|
while [ $(date +%s) -lt $deadline ]; do
|
|
ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo Unknown)
|
|
[ "$ready" = "True" ] && break
|
|
sleep 5
|
|
done
|
|
[ "$ready" = "True" ] || die "kubelet on $NODE never returned Ready — escalate (do NOT change k3s config)"
|
|
log " kubelet Ready=True"
|
|
fi
|
|
|
|
log "=== Phase E: cp-1 is back ==="
|
|
}
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Phase F — Uncordon + verify + finalize
|
|
# --------------------------------------------------------------------------- #
|
|
phase_finalize() {
|
|
log "=== Phase F: uncordon + verify ==="
|
|
|
|
run kubectl uncordon "$NODE"
|
|
|
|
log "-- settle wait ${POST_UNCORDON_WAIT_SECONDS}s"
|
|
[ "$DRY_RUN" = "1" ] || sleep "$POST_UNCORDON_WAIT_SECONDS"
|
|
|
|
log "-- cluster health"
|
|
if [ "$DRY_RUN" != "1" ]; then
|
|
if ! RETRY_ON_TRANSIENT=1 "$HEALTH_SCRIPT" 2>&1 | tee -a "$LOG_LOCAL"; then
|
|
die "cluster health failed after cp-1 update — escalate, do NOT touch k3s"
|
|
fi
|
|
fi
|
|
|
|
log "-- apt history summary (audit)"
|
|
ssh_run 'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -60'
|
|
|
|
log "-- old snapshots (>30d) — listing only, review manually"
|
|
ssh_run 'find /var/lib/rancher/k3s/server/db/snapshots/ -type f -mtime +30 -name "pre-*" -print 2>/dev/null || true'
|
|
|
|
log "=== cp-1 OS update complete — attach $LOG_LOCAL to the execution ticket ==="
|
|
}
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# --run — orchestrate all phases with confirmations
|
|
# --------------------------------------------------------------------------- #
|
|
phase_run_all() {
|
|
log "=== full cp-1 update run (log: $LOG_LOCAL) ==="
|
|
|
|
confirm "Phase A (add swap) — proceed?"
|
|
phase_add_swap
|
|
|
|
confirm "Phase B (preflight) — proceed?"
|
|
phase_preflight
|
|
|
|
confirm "Phase C (cordon + batched stateful move) — proceed?"
|
|
phase_drain_stateful
|
|
|
|
confirm "Phase D (drain remaining + apt) — proceed?"
|
|
phase_apt
|
|
|
|
confirm "Phase E (reboot cp-1; api-server unavailable ~90-180s) — proceed?"
|
|
phase_reboot
|
|
|
|
confirm "Phase F (uncordon + verify) — proceed?"
|
|
phase_finalize
|
|
|
|
log "=== FULL RUN COMPLETE ==="
|
|
}
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Arg parse
|
|
# --------------------------------------------------------------------------- #
|
|
usage() { grep -E '^# ' "$0" | sed 's/^# \{0,1\}//'; exit 2; }
|
|
|
|
[ $# -ge 1 ] || usage
|
|
|
|
# Init the local log file up front so tee always has a target.
|
|
: > "$LOG_LOCAL"
|
|
log "update-cp-1.sh started (DRY_RUN=$DRY_RUN)"
|
|
log "log file: $LOG_LOCAL"
|
|
|
|
case "$1" in
|
|
--dry-run)
|
|
DRY_RUN=1
|
|
export DRY_RUN
|
|
log "DRY_RUN=1 — walking Phases A..F without touching state"
|
|
phase_add_swap
|
|
phase_preflight
|
|
phase_drain_stateful
|
|
phase_apt
|
|
phase_reboot
|
|
phase_finalize
|
|
;;
|
|
--add-swap) phase_add_swap ;;
|
|
--preflight) phase_preflight ;;
|
|
--drain-stateful) phase_drain_stateful ;;
|
|
--apt) phase_apt ;;
|
|
--reboot) phase_reboot ;;
|
|
--finalize) phase_finalize ;;
|
|
--run) phase_run_all ;;
|
|
-h|--help) usage ;;
|
|
*) echo "unknown arg: $1" >&2; usage ;;
|
|
esac
|