#!/bin/bash # update-cp-1.sh — controlled OS update for the k3s control-plane node. # # See ../CP1_UPDATE_PROCEDURE.md for the full design rationale. # # Usage: # update-cp-1.sh --dry-run # print what would be done, touch nothing # update-cp-1.sh --add-swap # Phase A only (idempotent, safe standalone) # update-cp-1.sh --preflight # Phase B only # update-cp-1.sh --drain-stateful # Phase C only (cordon + batched sts moves) # update-cp-1.sh --apt # Phase D only (requires cp-1 fully drained) # update-cp-1.sh --reboot # Phase E only (requires --apt reported REBOOT_REQUIRED=yes) # update-cp-1.sh --finalize # Phase F only (uncordon + verify) # update-cp-1.sh --run # all phases with a confirmation between each (or ASSUME_YES=1) # # Environment overrides: # CP1_HOST default 178.105.17.239 # SWAP_SIZE_MB default 4096 (>=2048 required) # SWAP_PATH default /swapfile # DRAIN_TIMEOUT_SECONDS default 600 # REBOOT_MAX_WAIT_SECONDS default 600 # POST_UNCORDON_WAIT_SECONDS default 180 # STATEFUL_SETTLE_SECONDS default 60 (pause between batched sts moves) # MIN_WORKER_MEM_MIB default 500 (per-worker MemAvailable floor at preflight) # MIN_CP1_MEM_MIB default 200 (cp-1 MemAvailable floor mid-drain) # MAX_KUBECTL_SECONDS default 5 (kine-health guardrail) # SSH_OPTS default "-o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new" # ASSUME_YES=1 skip interactive confirmations in --run # DRY_RUN=1 do not execute state-mutating commands; log-only # # Log directory contract (matches update-node.sh): # Every command's stdout+stderr is tee'd to /tmp/os-update-cp-1-.log on # the operator machine AND to /tmp/os-update-cp-1-.log on cp-1 (via the # ssh command wrappers). Attach both to the execution ticket at the end. # # NEVER touches k3s config, k3s services, containerd, or any manifest. Only # fixes it will attempt: apt/dpkg recovery on cp-1 (see Phase D). Any other # problem → escalate and STOP. set -euo pipefail # --------------------------------------------------------------------------- # # Config # --------------------------------------------------------------------------- # CP1_HOST="${CP1_HOST:-178.105.17.239}" SWAP_SIZE_MB="${SWAP_SIZE_MB:-4096}" SWAP_PATH="${SWAP_PATH:-/swapfile}" DRAIN_TIMEOUT_SECONDS="${DRAIN_TIMEOUT_SECONDS:-600}" REBOOT_MAX_WAIT_SECONDS="${REBOOT_MAX_WAIT_SECONDS:-600}" POST_UNCORDON_WAIT_SECONDS="${POST_UNCORDON_WAIT_SECONDS:-180}" STATEFUL_SETTLE_SECONDS="${STATEFUL_SETTLE_SECONDS:-60}" MIN_WORKER_MEM_MIB="${MIN_WORKER_MEM_MIB:-500}" MIN_CP1_MEM_MIB="${MIN_CP1_MEM_MIB:-200}" MAX_KUBECTL_SECONDS="${MAX_KUBECTL_SECONDS:-5}" SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new}" DRY_RUN="${DRY_RUN:-0}" ASSUME_YES="${ASSUME_YES:-0}" NODE="k3s-cp-1" TS="$(date -u +%Y%m%dT%H%M%SZ)" LOG_LOCAL="/tmp/os-update-cp-1-${TS}.log" LOG_REMOTE="/tmp/os-update-cp-1-${TS}.log" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" HEALTH_SCRIPT="$SCRIPT_DIR/cluster-health.sh" # --------------------------------------------------------------------------- # # Stateful eviction ledger — order matters (idle -> active, small -> heavy). # --------------------------------------------------------------------------- # # The list is derived live from the cluster in preflight and stored to # /tmp/cp1-stateful-plan-.txt. This ORDER is the authoritative fallback. STATEFUL_ORDER=( "harbor:harbor-redis-0" "nextcloud:nextcloud-redis-replicas-0" "harbor:harbor-database-0" "stalwart:stalwart-postgres-0" "stalwart:stalwart-0" ) # --------------------------------------------------------------------------- # # Logging + safe-run helpers # --------------------------------------------------------------------------- # log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_LOCAL"; } die() { log "FATAL: $*"; exit 1; } warn() { log "WARN: $*"; } # run/ssh_run/kubectl_run: honour DRY_RUN — print, do not execute state changes. run() { if [ "$DRY_RUN" = "1" ]; then log "DRY-RUN would exec: $*" return 0 fi log "exec: $*" "$@" 2>&1 | tee -a "$LOG_LOCAL" } ssh_run() { local target="root@${CP1_HOST}" if [ "$DRY_RUN" = "1" ]; then log "DRY-RUN would ssh $target: $*" return 0 fi log "ssh $target: $*" ssh $SSH_OPTS "$target" "$@" 2>&1 | tee -a "$LOG_LOCAL" } # ssh_run_stdin: pipe a heredoc through bash -s on cp-1; used for multi-line remote # blocks (swap add, apt). ssh_run_stdin() { local target="root@${CP1_HOST}" if [ "$DRY_RUN" = "1" ]; then log "DRY-RUN would ssh $target with stdin script:" sed 's/^/ | /' | tee -a "$LOG_LOCAL" return 0 fi log "ssh $target (heredoc)" ssh $SSH_OPTS "$target" "bash -s" 2>&1 | tee -a "$LOG_LOCAL" } confirm() { local prompt="$1" if [ "$ASSUME_YES" = "1" ]; then log "confirm SKIPPED (ASSUME_YES=1): $prompt" return 0 fi echo -n " >>> $prompt Continue? [y/N] " read -r a case "$a" in y|Y|yes|YES) return 0 ;; *) die "aborted by operator" ;; esac } # --------------------------------------------------------------------------- # # Phase A — Add swap on cp-1 (idempotent) # --------------------------------------------------------------------------- # phase_add_swap() { log "=== Phase A: add swap on $NODE ($SWAP_SIZE_MB MiB at $SWAP_PATH) ===" [ "$SWAP_SIZE_MB" -ge 2048 ] || die "SWAP_SIZE_MB=$SWAP_SIZE_MB below 2048 MiB guardrail" cat </dev/null | grep -qx "\$SWAP_PATH"; then echo "swap already on at \$SWAP_PATH — skipping" free -h exit 0 fi # Root filesystem free space check — abort if less than 2*swap free. avail_mb=\$(df -m --output=avail / | tail -1 | tr -d ' ') need_mb=\$(( SIZE_MB * 2 )) if [ "\$avail_mb" -lt "\$need_mb" ]; then echo "ERROR: only \${avail_mb} MiB free on /, need \${need_mb} MiB (2x swap for safety)" exit 1 fi # Create swapfile. fallocate is fast; dd is the fallback. if ! fallocate -l "\${SIZE_MB}M" "\$SWAP_PATH" 2>/dev/null; then dd if=/dev/zero of="\$SWAP_PATH" bs=1M count="\$SIZE_MB" status=progress fi chmod 600 "\$SWAP_PATH" mkswap "\$SWAP_PATH" swapon "\$SWAP_PATH" # Persist via fstab (dedup). if ! grep -q "^\$SWAP_PATH " /etc/fstab; then echo "\$SWAP_PATH none swap sw 0 0" >> /etc/fstab fi # Moderate swappiness — swap as safety net, not aggressive paging. sysctl -w vm.swappiness=10 if [ ! -f /etc/sysctl.d/99-k3s-swap.conf ] || ! grep -q '^vm.swappiness' /etc/sysctl.d/99-k3s-swap.conf; then echo 'vm.swappiness=10' > /etc/sysctl.d/99-k3s-swap.conf fi echo "--- swap after ---" free -h swapon --show sysctl vm.swappiness REMOTE # Verify from operator view. if [ "$DRY_RUN" != "1" ]; then log "verifying kubelet still Ready after swap add" local ready ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') [ "$ready" = "True" ] || die "kubelet on $NODE not Ready after swap add — halt" log "kubelet Ready=True — Phase A complete" fi } # --------------------------------------------------------------------------- # # Guardrail probes # --------------------------------------------------------------------------- # kubectl_latency_s() { # returns integer seconds elapsed for `kubectl get nodes >/dev/null`. local start end start=$(date +%s) kubectl get nodes >/dev/null 2>&1 || echo "kubectl-error" >&2 end=$(date +%s) echo $(( end - start )) } cp1_mem_available_mib() { # returns integer MiB `MemAvailable` on cp-1. ssh $SSH_OPTS "root@$CP1_HOST" "awk '/^MemAvailable:/{printf \"%d\n\", \$2/1024}' /proc/meminfo" 2>/dev/null || echo 0 } guard_kine_healthy() { local s s=$(kubectl_latency_s) if [ "$s" -gt "$MAX_KUBECTL_SECONDS" ]; then die "kubectl get nodes took ${s}s (>${MAX_KUBECTL_SECONDS}s) — kine cascade risk, HALT" fi log " kine ok: kubectl get nodes took ${s}s" } guard_cp1_memory() { local m m=$(cp1_mem_available_mib) if [ "$m" -lt "$MIN_CP1_MEM_MIB" ]; then die "cp-1 MemAvailable=${m} MiB below ${MIN_CP1_MEM_MIB} MiB floor — HALT" fi log " cp-1 mem ok: MemAvailable=${m} MiB" } # --------------------------------------------------------------------------- # # Phase B — Preflight # --------------------------------------------------------------------------- # phase_preflight() { log "=== Phase B: preflight ===" log "-- cluster health" if [ "$DRY_RUN" != "1" ]; then if ! RETRY_ON_TRANSIENT=1 "$HEALTH_SCRIPT" 2>&1 | tee -a "$LOG_LOCAL"; then die "cluster is not healthy — refuse to start cp-1 update" fi else log "DRY-RUN would run: $HEALTH_SCRIPT" fi log "-- swap on cp-1" if [ "$DRY_RUN" != "1" ]; then local swap_total swap_total=$(ssh $SSH_OPTS "root@$CP1_HOST" "awk '/^SwapTotal:/{print \$2}' /proc/meminfo") [ "${swap_total:-0}" -ge $((2 * 1024 * 1024)) ] \ || die "cp-1 SwapTotal=${swap_total} KiB below 2 GiB — run --add-swap first" log " cp-1 SwapTotal=$(( swap_total / 1024 )) MiB" fi log "-- fsn1 workers Ready and have >=${MIN_WORKER_MEM_MIB} MiB MemAvailable" # Worker private IPs (worker-4 nbg1 excluded — cannot host fsn1 RWO PVs). local ips=("10.42.1.2" "10.42.1.3" "10.42.1.5" "10.42.1.7") local names=("k3s-worker-1" "k3s-worker-2" "k3s-worker-3" "k3s-worker-5") if [ "$DRY_RUN" != "1" ]; then for i in "${!ips[@]}"; do local nm="${names[$i]}" local ip="${ips[$i]}" local ready ready=$(kubectl get node "$nm" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo Unknown) [ "$ready" = "True" ] || die "worker $nm not Ready (Ready=$ready)" local mem mem=$(ssh $SSH_OPTS -J "root@$CP1_HOST" "root@$ip" \ "awk '/^MemAvailable:/{printf \"%d\n\", \$2/1024}' /proc/meminfo" 2>/dev/null || echo 0) [ "$mem" -ge "$MIN_WORKER_MEM_MIB" ] \ || die "worker $nm MemAvailable=${mem} MiB below ${MIN_WORKER_MEM_MIB} MiB — HALT" log " $nm ok: MemAvailable=${mem} MiB" done fi log "-- kine latency probe" if [ "$DRY_RUN" != "1" ]; then guard_kine_healthy; fi log "-- k3s SQLite datastore snapshot" # k3s etcd-snapshot in embedded-SQLite mode copies the sqlite file to # /var/lib/rancher/k3s/server/db/snapshots/. ssh_run "k3s etcd-snapshot save --name pre-cp1-os-update-${TS}" ssh_run "ls -la /var/lib/rancher/k3s/server/db/snapshots/ | tail -10" log "-- persist eviction plan" if [ "$DRY_RUN" != "1" ]; then local plan="/tmp/cp1-stateful-plan-${TS}.txt" ssh $SSH_OPTS "root@$CP1_HOST" "cat > $plan" < active, small -> heavy. Do NOT reorder without design review. $(for e in "${STATEFUL_ORDER[@]}"; do echo "$e"; done) EOF log " wrote $plan on cp-1" else log "DRY-RUN would write /tmp/cp1-stateful-plan-${TS}.txt with STATEFUL_ORDER" fi log "=== Phase B: preflight OK ===" } # --------------------------------------------------------------------------- # # Phase C — Cordon + move stateful pods off cp-1, one at a time # --------------------------------------------------------------------------- # phase_drain_stateful() { log "=== Phase C: cordon + batched stateful move ===" log "-- cordon $NODE (prevents rescheduled pods from landing back on cp-1)" run kubectl cordon "$NODE" local i=0 for entry in "${STATEFUL_ORDER[@]}"; do i=$((i + 1)) local ns="${entry%%:*}" local pod="${entry##*:}" log "--- [${i}/${#STATEFUL_ORDER[@]}] moving $ns/$pod" # Confirm the pod actually IS on cp-1 before touching it. if [ "$DRY_RUN" != "1" ]; then local on on=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.nodeName}' 2>/dev/null || echo "") if [ -z "$on" ]; then log " $ns/$pod not found — sts may have changed; skipping" continue fi if [ "$on" != "$NODE" ]; then log " $ns/$pod already on $on (not cp-1) — skipping" continue fi fi # Delete the pod. StatefulSet controller will re-create it; scheduler will # pick a fsn1 worker because cp-1 is cordoned and worker-4 is nbg1. run kubectl -n "$ns" delete pod "$pod" --wait=false if [ "$DRY_RUN" = "1" ]; then log " DRY-RUN skipping wait-for-ready" continue fi log " waiting for $ns/$pod to be Ready on a non-cp-1 node (max 300s)" local deadline=$(( $(date +%s) + 300 )) local new_node="" ready="" while [ $(date +%s) -lt $deadline ]; do new_node=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.nodeName}' 2>/dev/null || echo "") ready=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "") if [ -n "$new_node" ] && [ "$new_node" != "$NODE" ] && [ "$ready" = "True" ]; then log " $ns/$pod -> $new_node OK" break fi sleep 5 done [ "$ready" = "True" ] && [ "$new_node" != "$NODE" ] \ || die "$ns/$pod did not reach Ready on a non-cp-1 node in 300s — HALT (leave cp-1 cordoned)" log " guardrails after $ns/$pod" guard_kine_healthy guard_cp1_memory if [ "$i" -lt "${#STATEFUL_ORDER[@]}" ]; then log " settle pause ${STATEFUL_SETTLE_SECONDS}s" sleep "$STATEFUL_SETTLE_SECONDS" fi done log "=== Phase C: all stateful pods evicted from cp-1 ===" } # --------------------------------------------------------------------------- # # Phase D — Drain remaining pods + apt on cp-1 # --------------------------------------------------------------------------- # phase_apt() { log "=== Phase D: drain (remaining deployments) + apt on $NODE ===" # Guardrail: refuse to run if any StatefulSet pod is still on cp-1. if [ "$DRY_RUN" != "1" ]; then local sts_on_cp1 sts_on_cp1=$(kubectl get pods -A -o json --field-selector spec.nodeName=$NODE \ | jq -r '.items[] | select(.metadata.ownerReferences[0].kind=="StatefulSet") | "\(.metadata.namespace)/\(.metadata.name)"' \ | wc -l) [ "$sts_on_cp1" -eq 0 ] \ || die "$sts_on_cp1 StatefulSet pod(s) still on cp-1 — Phase C incomplete; refuse Phase D" fi log "-- drain (timeout ${DRAIN_TIMEOUT_SECONDS}s)" set +e if [ "$DRY_RUN" = "1" ]; then log "DRY-RUN would run: kubectl drain $NODE --ignore-daemonsets --delete-emptydir-data --timeout=${DRAIN_TIMEOUT_SECONDS}s" local rc=0 else kubectl drain "$NODE" \ --ignore-daemonsets \ --delete-emptydir-data \ --timeout="${DRAIN_TIMEOUT_SECONDS}s" 2>&1 | tee -a "$LOG_LOCAL" local rc=${PIPESTATUS[0]} fi set -e if [ "$rc" -ne 0 ]; then log "drain FAILED (rc=$rc). Never force. Uncordoning." run kubectl uncordon "$NODE" die "drain failed on $NODE — investigate PDB / orphan pods; do NOT proceed" fi log "-- apt on cp-1" cat <<'REMOTE' | ssh_run_stdin set -euo pipefail export DEBIAN_FRONTEND=noninteractive APT_OPTS='-y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold' # Rollback references (Phase-A didn't need this, but Phase D does). uname -r > /root/pre-apt-kernel dpkg-query -W -f='${Package}\t${Version}\n' > /root/pre-apt-packages.tsv echo "pre-apt kernel: $(cat /root/pre-apt-kernel)" # Recover from any half-finished dpkg state before touching apt. if dpkg --audit | grep -qE .; then echo "dpkg audit reported issues, running dpkg --configure -a" dpkg --configure -a || true fi apt-get update if ! apt-get $APT_OPTS upgrade; then echo "upgrade failed, attempting apt-get -f install" apt-get $APT_OPTS -f install apt-get $APT_OPTS upgrade fi apt-get $APT_OPTS dist-upgrade apt-get $APT_OPTS autoremove --purge apt-get clean if [ -f /var/run/reboot-required ]; then echo "REBOOT_REQUIRED=yes" echo "REBOOT_REASON</dev/null || echo "(no package list)" echo "EOF" else echo "REBOOT_REQUIRED=no" fi REMOTE log "=== Phase D: apt complete (check REBOOT_REQUIRED in the log) ===" } # --------------------------------------------------------------------------- # # Phase E — Reboot cp-1 and wait for api-server /livez # --------------------------------------------------------------------------- # phase_reboot() { log "=== Phase E: reboot $NODE ===" # Tell cp-1 to reboot. The ssh command will hang up mid-command — that's fine. if [ "$DRY_RUN" != "1" ]; then log "issuing 'systemctl reboot' on $NODE (ssh will drop; expected)" ssh $SSH_OPTS "root@$CP1_HOST" 'systemctl reboot' 2>&1 | tee -a "$LOG_LOCAL" || true log "waiting 15s for ssh to fully drop before polling api-server" sleep 15 else log "DRY-RUN would ssh root@$CP1_HOST 'systemctl reboot'" fi log "-- poll api-server /livez (timeout ${REBOOT_MAX_WAIT_SECONDS}s)" if [ "$DRY_RUN" != "1" ]; then local deadline=$(( $(date +%s) + REBOOT_MAX_WAIT_SECONDS )) local code=000 while [ $(date +%s) -lt $deadline ]; do code=$(curl -sk -o /dev/null -w '%{http_code}' "https://$CP1_HOST:6443/livez" 2>/dev/null || echo 000) if [ "$code" = "200" ]; then log " api-server /livez=200 (elapsed $(( REBOOT_MAX_WAIT_SECONDS - (deadline - $(date +%s)) ))s)" break fi sleep 5 done [ "$code" = "200" ] || die "api-server did not return within ${REBOOT_MAX_WAIT_SECONDS}s — escalate; check 'hcloud server describe k3s-cp-1' and Hetzner console" fi log "-- wait for kubelet Ready on $NODE (max 300s)" if [ "$DRY_RUN" != "1" ]; then local deadline=$(( $(date +%s) + 300 )) local ready=Unknown while [ $(date +%s) -lt $deadline ]; do ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo Unknown) [ "$ready" = "True" ] && break sleep 5 done [ "$ready" = "True" ] || die "kubelet on $NODE never returned Ready — escalate (do NOT change k3s config)" log " kubelet Ready=True" fi log "=== Phase E: cp-1 is back ===" } # --------------------------------------------------------------------------- # # Phase F — Uncordon + verify + finalize # --------------------------------------------------------------------------- # phase_finalize() { log "=== Phase F: uncordon + verify ===" run kubectl uncordon "$NODE" log "-- settle wait ${POST_UNCORDON_WAIT_SECONDS}s" [ "$DRY_RUN" = "1" ] || sleep "$POST_UNCORDON_WAIT_SECONDS" log "-- cluster health" if [ "$DRY_RUN" != "1" ]; then if ! RETRY_ON_TRANSIENT=1 "$HEALTH_SCRIPT" 2>&1 | tee -a "$LOG_LOCAL"; then die "cluster health failed after cp-1 update — escalate, do NOT touch k3s" fi fi log "-- apt history summary (audit)" ssh_run 'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -60' log "-- old snapshots (>30d) — listing only, review manually" ssh_run 'find /var/lib/rancher/k3s/server/db/snapshots/ -type f -mtime +30 -name "pre-*" -print 2>/dev/null || true' log "=== cp-1 OS update complete — attach $LOG_LOCAL to the execution ticket ===" } # --------------------------------------------------------------------------- # # --run — orchestrate all phases with confirmations # --------------------------------------------------------------------------- # phase_run_all() { log "=== full cp-1 update run (log: $LOG_LOCAL) ===" confirm "Phase A (add swap) — proceed?" phase_add_swap confirm "Phase B (preflight) — proceed?" phase_preflight confirm "Phase C (cordon + batched stateful move) — proceed?" phase_drain_stateful confirm "Phase D (drain remaining + apt) — proceed?" phase_apt confirm "Phase E (reboot cp-1; api-server unavailable ~90-180s) — proceed?" phase_reboot confirm "Phase F (uncordon + verify) — proceed?" phase_finalize log "=== FULL RUN COMPLETE ===" } # --------------------------------------------------------------------------- # # Arg parse # --------------------------------------------------------------------------- # usage() { grep -E '^# ' "$0" | sed 's/^# \{0,1\}//'; exit 2; } [ $# -ge 1 ] || usage # Init the local log file up front so tee always has a target. : > "$LOG_LOCAL" log "update-cp-1.sh started (DRY_RUN=$DRY_RUN)" log "log file: $LOG_LOCAL" case "$1" in --dry-run) DRY_RUN=1 export DRY_RUN log "DRY_RUN=1 — walking Phases A..F without touching state" phase_add_swap phase_preflight phase_drain_stateful phase_apt phase_reboot phase_finalize ;; --add-swap) phase_add_swap ;; --preflight) phase_preflight ;; --drain-stateful) phase_drain_stateful ;; --apt) phase_apt ;; --reboot) phase_reboot ;; --finalize) phase_finalize ;; --run) phase_run_all ;; -h|--help) usage ;; *) echo "unknown arg: $1" >&2; usage ;; esac