stack.basicstack.de/infrastructure/scripts/os-update/update-cp-node.sh

#!/bin/bash
# update-cp-node.sh — controlled OS update for any HA k3s control-plane node.
#
# See ../CP_UPDATE_PROCEDURE.md for the full design rationale.
#
# Usage:
#   update-cp-node.sh <node> --dry-run          # print what would be done, touch nothing
#   update-cp-node.sh <node> --add-swap         # Phase A only (idempotent, safe standalone)
#   update-cp-node.sh <node> --preflight        # Phase B only
#   update-cp-node.sh <node> --drain            # Phase C only (cordon + drain)
#   update-cp-node.sh <node> --apt              # Phase D only (requires <node> already fully drained)
#   update-cp-node.sh <node> --reboot           # Phase E only (requires --apt reported REBOOT_REQUIRED=yes)
#   update-cp-node.sh <node> --finalize         # Phase F only (uncordon + verify)
#   update-cp-node.sh <node> --run              # all phases with confirmation between each (or ASSUME_YES=1)
#
# <node> must be one of: k3s-cp-1, k3s-cp-2, k3s-cp-3.
#
# Environment overrides:
#   SWAP_SIZE_MB                 default 4096       (>=2048 required)
#   SWAP_PATH                    default /swapfile
#   DRAIN_TIMEOUT_SECONDS        default 600
#   REBOOT_MAX_WAIT_SECONDS      default 600
#   POST_UNCORDON_WAIT_SECONDS   default 180
#   MIN_TARGET_MEM_MIB           default 200        (target-CP MemAvailable floor mid-drain)
#   MAX_KUBECTL_SECONDS          default 5          (kine-latency guardrail; softened for HA etcd)
#   SSH_OPTS                     default "-o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
#   ASSUME_YES=1                 skip interactive confirmations in --run
#   DRY_RUN=1                    do not execute state-mutating commands; log-only
#
# Log directory contract (matches update-node.sh):
#   Every command's stdout+stderr is tee'd to /tmp/os-update-<node>-<UTC-ts>.log on
#   the operator machine. Attach it to the execution ticket at the end.
#
# NEVER touches k3s config, k3s services, containerd, or any manifest. Only
# fixes it will attempt: apt/dpkg recovery on the target CP (Phase D). Any other
# problem -> escalate and STOP.

set -euo pipefail

# --------------------------------------------------------------------------- #
# CP topology — public/routable IPs used to reach each CP via ssh + /livez.
# Kept out-of-cluster on purpose: during a full-cluster incident the operator
# machine must be able to reach each CP without going through k3s.
# --------------------------------------------------------------------------- #
declare -A CP_HOST=(
  [k3s-cp-1]="178.105.17.239"
  [k3s-cp-2]="188.245.85.199"
  [k3s-cp-3]="49.13.92.162"
)

# --------------------------------------------------------------------------- #
# Arg parse — expect <node> as $1
# --------------------------------------------------------------------------- #
usage() { grep -E '^# ' "$0" | sed 's/^# \{0,1\}//'; exit 2; }
[ $# -ge 2 ] || usage

NODE="$1"; shift
TARGET_HOST="${CP_HOST[$NODE]:-}"
if [ -z "$TARGET_HOST" ]; then
  echo "unknown CP node: $NODE (allowed: ${!CP_HOST[*]})" >&2
  exit 2
fi

# --------------------------------------------------------------------------- #
# Config
# --------------------------------------------------------------------------- #
SWAP_SIZE_MB="${SWAP_SIZE_MB:-4096}"
SWAP_PATH="${SWAP_PATH:-/swapfile}"
DRAIN_TIMEOUT_SECONDS="${DRAIN_TIMEOUT_SECONDS:-600}"
REBOOT_MAX_WAIT_SECONDS="${REBOOT_MAX_WAIT_SECONDS:-600}"
POST_UNCORDON_WAIT_SECONDS="${POST_UNCORDON_WAIT_SECONDS:-180}"
MIN_TARGET_MEM_MIB="${MIN_TARGET_MEM_MIB:-200}"
MAX_KUBECTL_SECONDS="${MAX_KUBECTL_SECONDS:-5}"
SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new}"
DRY_RUN="${DRY_RUN:-0}"
ASSUME_YES="${ASSUME_YES:-0}"

TS="$(date -u +%Y%m%dT%H%M%SZ)"
LOG_LOCAL="/tmp/os-update-${NODE}-${TS}.log"

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
HEALTH_SCRIPT="$SCRIPT_DIR/cluster-health.sh"

# Pick a peer CP for etcd status probes. Requirements:
#   - not the target
#   - etcdctl available (via non-login ssh PATH)
# We probe each candidate at parse time. Order preference: cp-1 first (it has
# etcdctl installed since the pre-HA era), then cp-2, then cp-3. The Phase A
# `--add-swap` step also installs `etcd-client` via apt so cp-2/cp-3 pick up
# etcdctl on their first pass.
peer_host_for() {
  local target="$1"
  local order=(k3s-cp-1 k3s-cp-2 k3s-cp-3)
  for n in "${order[@]}"; do
    [ "$n" = "$target" ] && continue
    local h="${CP_HOST[$n]}"
    if ssh $SSH_OPTS -o BatchMode=yes "root@$h" 'command -v etcdctl >/dev/null 2>&1' 2>/dev/null; then
      echo "$h"; return 0
    fi
  done
  # In DRY_RUN we don't need a real etcdctl-capable peer.
  if [ "${DRY_RUN:-0}" = "1" ]; then
    for n in "${order[@]}"; do
      [ "$n" != "$target" ] && { echo "${CP_HOST[$n]}"; return 0; }
    done
  fi
  # No non-target CP has etcdctl. Return empty so Phase B/E can fail
  # explicitly with a targeted "install etcd-client on cp-X first" message.
  echo ""
  return 0
}
PEER_HOST="$(peer_host_for "$NODE")"

# --------------------------------------------------------------------------- #
# Logging + safe-run helpers
# --------------------------------------------------------------------------- #
log()  { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_LOCAL"; }
die()  { log "FATAL: $*"; exit 1; }
warn() { log "WARN:  $*"; }

run() {
  if [ "$DRY_RUN" = "1" ]; then
    log "DRY-RUN would exec: $*"
    return 0
  fi
  log "exec: $*"
  "$@" 2>&1 | tee -a "$LOG_LOCAL"
}

ssh_run() {
  local target="${1:-root@$TARGET_HOST}"; shift || true
  if [ "$DRY_RUN" = "1" ]; then
    log "DRY-RUN would ssh $target: $*"
    return 0
  fi
  log "ssh $target: $*"
  ssh $SSH_OPTS "$target" "$@" 2>&1 | tee -a "$LOG_LOCAL"
}

# ssh_run_stdin: pipe a heredoc through bash -s on the target; used for multi-line remote blocks.
ssh_run_stdin() {
  local target="root@${TARGET_HOST}"
  if [ "$DRY_RUN" = "1" ]; then
    log "DRY-RUN would ssh $target with stdin script:"
    sed 's/^/    | /' | tee -a "$LOG_LOCAL"
    return 0
  fi
  log "ssh $target (heredoc)"
  ssh $SSH_OPTS "$target" "bash -s" 2>&1 | tee -a "$LOG_LOCAL"
}

confirm() {
  local prompt="$1"
  if [ "$ASSUME_YES" = "1" ]; then
    log "confirm SKIPPED (ASSUME_YES=1): $prompt"
    return 0
  fi
  echo -n "  >>> $prompt  Continue? [y/N] "
  read -r a
  case "$a" in y|Y|yes|YES) return 0 ;; *) die "aborted by operator" ;; esac
}

# --------------------------------------------------------------------------- #
# etcd helpers — run etcdctl on the target OR on a peer CP.
# --------------------------------------------------------------------------- #
ETCDCTL_ENV='ETCDCTL_API=3 etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
  --cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
  --key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key'

etcd_status_via() {
  # $1 = host (IP)
  local host="$1"
  ssh $SSH_OPTS "root@$host" "$ETCDCTL_ENV endpoint status --cluster -w table" 2>&1
}

etcd_leader_name() {
  # returns the etcd member NAME (e.g. k3s-cp-3-9d305472) whose row shows IS LEADER=true.
  # Uses simple-format output (no -w table) for reliable parsing.
  local host="$1"
  ssh $SSH_OPTS "root@$host" "$ETCDCTL_ENV endpoint status --cluster -w simple" 2>/dev/null \
    | awk -F, '$5 ~ /true/ {print $1}'
}

target_is_leader() {
  # returns 0 if the target IP is the current etcd leader, 1 otherwise.
  local host="$1"     # peer to query from
  local target="$2"   # target IP to compare
  local leader_ep
  leader_ep=$(ssh $SSH_OPTS "root@$host" "$ETCDCTL_ENV endpoint status --cluster -w simple" 2>/dev/null \
    | awk -F, '$5 ~ /true/ {print $1}')
  # leader_ep looks like https://49.13.92.162:2379
  echo "$leader_ep" | grep -q "://${target}:" && return 0 || return 1
}

# --------------------------------------------------------------------------- #
# Phase A — Add swap on the target CP (idempotent)
# --------------------------------------------------------------------------- #
phase_add_swap() {
  log "=== Phase A: add swap on $NODE ($SWAP_SIZE_MB MiB at $SWAP_PATH) ==="
  [ "$SWAP_SIZE_MB" -ge 2048 ] || die "SWAP_SIZE_MB=$SWAP_SIZE_MB below 2048 MiB guardrail"

  cat <<REMOTE | ssh_run_stdin
set -euo pipefail
SWAP_PATH="$SWAP_PATH"
SIZE_MB="$SWAP_SIZE_MB"

# Skip if a swapfile at this path is already active.
if swapon --show=NAME 2>/dev/null | grep -qx "\$SWAP_PATH"; then
  echo "swap already on at \$SWAP_PATH -- skipping"
  free -h
  exit 0
fi

# Root filesystem free space check -- abort if less than 2*swap free.
avail_mb=\$(df -m --output=avail / | tail -1 | tr -d ' ')
need_mb=\$(( SIZE_MB * 2 ))
if [ "\$avail_mb" -lt "\$need_mb" ]; then
  echo "ERROR: only \${avail_mb} MiB free on /, need \${need_mb} MiB (2x swap for safety)"
  exit 1
fi

# Create swapfile. fallocate is fast; dd is the fallback.
if ! fallocate -l "\${SIZE_MB}M" "\$SWAP_PATH" 2>/dev/null; then
  dd if=/dev/zero of="\$SWAP_PATH" bs=1M count="\$SIZE_MB" status=progress
fi
chmod 600 "\$SWAP_PATH"
mkswap "\$SWAP_PATH"
swapon "\$SWAP_PATH"

# Persist via fstab (dedup).
if ! grep -q "^\$SWAP_PATH " /etc/fstab; then
  echo "\$SWAP_PATH none swap sw 0 0" >> /etc/fstab
fi

# Moderate swappiness -- swap as safety net, not aggressive paging.
sysctl -w vm.swappiness=10
if [ ! -f /etc/sysctl.d/99-k3s-swap.conf ] || ! grep -q '^vm.swappiness' /etc/sysctl.d/99-k3s-swap.conf; then
  echo 'vm.swappiness=10' > /etc/sysctl.d/99-k3s-swap.conf
fi

echo "--- swap after ---"
free -h
swapon --show
sysctl vm.swappiness

# Ensure etcdctl is available for etcd-quorum probes (idempotent apt install).
# Needed because when this CP is the target of a later update, another CP
# must probe etcd cluster status; if cp-1 is the target, one of cp-2/cp-3
# is the probing peer and must have etcdctl.
if ! command -v etcdctl >/dev/null 2>&1; then
  echo "--- installing etcd-client (provides etcdctl) ---"
  export DEBIAN_FRONTEND=noninteractive
  apt-get update -y >/dev/null
  apt-get install -y etcd-client
  command -v etcdctl && etcdctl version
fi
REMOTE

  if [ "$DRY_RUN" != "1" ]; then
    log "verifying kubelet still Ready after swap add"
    local ready
    ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo Unknown)
    [ "$ready" = "True" ] || die "kubelet on $NODE not Ready after swap add -- halt"
    log "kubelet Ready=True -- Phase A complete"
  fi
}

# --------------------------------------------------------------------------- #
# Guardrail probes
# --------------------------------------------------------------------------- #
kubectl_latency_s() {
  local start end
  start=$(date +%s)
  kubectl get nodes >/dev/null 2>&1 || echo "kubectl-error" >&2
  end=$(date +%s)
  echo $(( end - start ))
}

target_mem_available_mib() {
  ssh $SSH_OPTS "root@$TARGET_HOST" "awk '/^MemAvailable:/{printf \"%d\n\", \$2/1024}' /proc/meminfo" 2>/dev/null || echo 0
}

guard_kine_healthy() {
  local s
  s=$(kubectl_latency_s)
  if [ "$s" -gt "$MAX_KUBECTL_SECONDS" ]; then
    die "kubectl get nodes took ${s}s (>${MAX_KUBECTL_SECONDS}s) -- etcd slow, HALT"
  fi
  log "  kubectl-latency ok: ${s}s"
}

guard_target_memory() {
  local m
  m=$(target_mem_available_mib)
  if [ "$m" -lt "$MIN_TARGET_MEM_MIB" ]; then
    die "$NODE MemAvailable=${m} MiB below ${MIN_TARGET_MEM_MIB} MiB floor -- HALT"
  fi
  log "  $NODE mem ok: MemAvailable=${m} MiB"
}

# --------------------------------------------------------------------------- #
# Phase B — Preflight
# --------------------------------------------------------------------------- #
phase_preflight() {
  log "=== Phase B: preflight for $NODE ==="

  log "-- cluster health"
  if [ "$DRY_RUN" != "1" ]; then
    if ! RETRY_ON_TRANSIENT=1 "$HEALTH_SCRIPT" 2>&1 | tee -a "$LOG_LOCAL"; then
      die "cluster is not healthy -- refuse to start $NODE update"
    fi
  else
    log "DRY-RUN would run: $HEALTH_SCRIPT"
  fi

  log "-- confirm $NODE is a control-plane node"
  if [ "$DRY_RUN" != "1" ]; then
    local is_cp
    is_cp=$(kubectl get node "$NODE" -o jsonpath='{.metadata.labels.node-role\.kubernetes\.io/control-plane}' 2>/dev/null || echo "")
    [ "$is_cp" = "true" ] || die "$NODE is not labelled control-plane -- refuse (use update-node.sh for workers)"
  fi

  log "-- verify no OTHER CP is currently cordoned"
  if [ "$DRY_RUN" != "1" ]; then
    local other_cordoned
    other_cordoned=$(kubectl get nodes -l node-role.kubernetes.io/control-plane=true \
      -o json | jq -r --arg n "$NODE" '.items[] | select(.metadata.name != $n) | select(.spec.unschedulable == true) | .metadata.name' \
      | tr '\n' ' ')
    if [ -n "${other_cordoned// /}" ]; then
      die "another CP is already cordoned: $other_cordoned -- refuse (one CP at a time)"
    fi
    log "  no other CP cordoned -- proceeding"
  fi

  log "-- etcd cluster status (all members must be started)"
  if [ "$DRY_RUN" != "1" ]; then
    if [ -z "$PEER_HOST" ]; then
      die "no non-target CP has etcdctl installed -- run \`update-cp-node.sh <other-cp> --add-swap\` first on one of the OTHER CPs (that step installs etcd-client), then retry"
    fi
    etcd_status_via "$PEER_HOST" | tee -a "$LOG_LOCAL"
  fi

  log "-- swap on $NODE"
  if [ "$DRY_RUN" != "1" ]; then
    local swap_total
    swap_total=$(ssh $SSH_OPTS "root@$TARGET_HOST" "awk '/^SwapTotal:/{print \$2}' /proc/meminfo")
    [ "${swap_total:-0}" -ge $((2 * 1024 * 1024)) ] \
      || die "$NODE SwapTotal=${swap_total} KiB below 2 GiB -- run --add-swap first"
    log "  $NODE SwapTotal=$(( swap_total / 1024 )) MiB"
  fi

  log "-- kubectl-latency probe"
  if [ "$DRY_RUN" != "1" ]; then guard_kine_healthy; fi

  log "-- record current etcd leader"
  if [ "$DRY_RUN" != "1" ] && [ -n "$PEER_HOST" ]; then
    if target_is_leader "$PEER_HOST" "$TARGET_HOST"; then
      warn "$NODE IS the current etcd leader. Per CP ordering rule, prefer updating a follower first."
      warn "  Not aborting -- operator/agent must confirm this is intentional."
    else
      log "  $NODE is a FOLLOWER -- safe to proceed."
    fi
  fi

  log "-- k3s etcd snapshot"
  ssh_run "root@$TARGET_HOST" "k3s etcd-snapshot save --name pre-cp-os-update-${NODE}-${TS}"
  ssh_run "root@$TARGET_HOST" "ls -la /var/lib/rancher/k3s/server/db/snapshots/ | tail -10"

  log "=== Phase B: preflight OK ==="
}

# --------------------------------------------------------------------------- #
# Phase C — Cordon + drain
# --------------------------------------------------------------------------- #
phase_drain() {
  log "=== Phase C: cordon + drain $NODE ==="

  log "-- cordon $NODE"
  run kubectl cordon "$NODE"

  log "-- drain $NODE (timeout ${DRAIN_TIMEOUT_SECONDS}s)"
  set +e
  if [ "$DRY_RUN" = "1" ]; then
    log "DRY-RUN would run: kubectl drain $NODE --ignore-daemonsets --delete-emptydir-data --timeout=${DRAIN_TIMEOUT_SECONDS}s"
    local rc=0
  else
    kubectl drain "$NODE" \
      --ignore-daemonsets \
      --delete-emptydir-data \
      --timeout="${DRAIN_TIMEOUT_SECONDS}s" 2>&1 | tee -a "$LOG_LOCAL"
    local rc=${PIPESTATUS[0]}
  fi
  set -e

  if [ "$rc" -ne 0 ]; then
    log "drain FAILED (rc=$rc). Never force. Uncordoning."
    run kubectl uncordon "$NODE"
    die "drain failed on $NODE -- investigate PDB / orphan pods; do NOT proceed"
  fi

  log "-- post-drain guardrails"
  if [ "$DRY_RUN" != "1" ]; then
    guard_kine_healthy
    guard_target_memory
  fi

  log "=== Phase C: $NODE drained ==="
}

# --------------------------------------------------------------------------- #
# Phase D — apt on the target
# --------------------------------------------------------------------------- #
phase_apt() {
  log "=== Phase D: apt on $NODE ==="

  cat <<'REMOTE' | ssh_run_stdin
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
APT_OPTS='-y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold'

uname -r > /root/pre-apt-kernel
dpkg-query -W -f='${Package}\t${Version}\n' > /root/pre-apt-packages.tsv
echo "pre-apt kernel: $(cat /root/pre-apt-kernel)"

if dpkg --audit | grep -qE .; then
  echo "dpkg audit reported issues, running dpkg --configure -a"
  dpkg --configure -a || true
fi

apt-get update

if ! apt-get $APT_OPTS upgrade; then
  echo "upgrade failed, attempting apt-get -f install"
  apt-get $APT_OPTS -f install
  apt-get $APT_OPTS upgrade
fi

apt-get $APT_OPTS dist-upgrade
apt-get $APT_OPTS autoremove --purge
apt-get clean

if [ -f /var/run/reboot-required ]; then
  echo "REBOOT_REQUIRED=yes"
  echo "REBOOT_REASON<<EOF"
  cat /var/run/reboot-required.pkgs 2>/dev/null || echo "(no package list)"
  echo "EOF"
else
  echo "REBOOT_REQUIRED=no"
fi
REMOTE

  log "=== Phase D: apt complete (check REBOOT_REQUIRED in the log) ==="
}

# --------------------------------------------------------------------------- #
# Phase E — Reboot and wait for target /livez + kubelet Ready + etcd rejoin
# --------------------------------------------------------------------------- #
phase_reboot() {
  log "=== Phase E: reboot $NODE ==="

  if [ "$DRY_RUN" != "1" ]; then
    log "issuing 'systemctl reboot' on $NODE (ssh will drop; expected)"
    ssh $SSH_OPTS "root@$TARGET_HOST" 'systemctl reboot' 2>&1 | tee -a "$LOG_LOCAL" || true
    log "waiting 15s for ssh to fully drop before polling"
    sleep 15
  else
    log "DRY-RUN would ssh root@$TARGET_HOST 'systemctl reboot'"
  fi

  log "-- poll $NODE api-server /livez (timeout ${REBOOT_MAX_WAIT_SECONDS}s)"
  if [ "$DRY_RUN" != "1" ]; then
    local deadline=$(( $(date +%s) + REBOOT_MAX_WAIT_SECONDS ))
    local code=000
    # /livez is auth-gated on this cluster (returns 401 without a bearer token).
    # Any HTTP response from the api-server (200/401/403) means it is up and
    # serving; only network-level failures (000, connection refused) count as
    # "not yet back". curl --max-time keeps the poll cycle snappy after reboot.
    while [ $(date +%s) -lt $deadline ]; do
      code=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "https://$TARGET_HOST:6443/livez" 2>/dev/null || echo 000)
      case "$code" in
        200|401|403)
          log "  $NODE api-server responding (/livez HTTP $code)"
          break
          ;;
      esac
      sleep 5
    done
    case "$code" in
      200|401|403) : ;;
      *) die "$NODE api-server did not return within ${REBOOT_MAX_WAIT_SECONDS}s (last code=$code) -- escalate; check 'hcloud server describe $NODE' and Hetzner console" ;;
    esac
  fi

  log "-- verify etcd cluster status from peer ($PEER_HOST) -- $NODE should be 'started'"
  if [ "$DRY_RUN" != "1" ] && [ -n "$PEER_HOST" ]; then
    etcd_status_via "$PEER_HOST" | tee -a "$LOG_LOCAL"
  fi

  log "-- wait for kubelet Ready on $NODE (max 300s)"
  if [ "$DRY_RUN" != "1" ]; then
    local deadline=$(( $(date +%s) + 300 ))
    local ready=Unknown
    while [ $(date +%s) -lt $deadline ]; do
      ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo Unknown)
      [ "$ready" = "True" ] && break
      sleep 5
    done
    [ "$ready" = "True" ] || die "kubelet on $NODE never returned Ready -- escalate (do NOT change k3s config)"
    log "  kubelet Ready=True"
  fi

  log "=== Phase E: $NODE is back ==="
}

# --------------------------------------------------------------------------- #
# Phase F — Uncordon + verify + finalize
# --------------------------------------------------------------------------- #
phase_finalize() {
  log "=== Phase F: uncordon + verify $NODE ==="

  run kubectl uncordon "$NODE"

  log "-- settle wait ${POST_UNCORDON_WAIT_SECONDS}s"
  [ "$DRY_RUN" = "1" ] || sleep "$POST_UNCORDON_WAIT_SECONDS"

  log "-- cluster health"
  if [ "$DRY_RUN" != "1" ]; then
    if ! RETRY_ON_TRANSIENT=1 "$HEALTH_SCRIPT" 2>&1 | tee -a "$LOG_LOCAL"; then
      die "cluster health failed after $NODE update -- escalate, do NOT touch k3s"
    fi
  fi

  log "-- etcd cluster status (all three should be started)"
  if [ "$DRY_RUN" != "1" ] && [ -n "$PEER_HOST" ]; then
    etcd_status_via "$PEER_HOST" | tee -a "$LOG_LOCAL"
  fi

  log "-- apt history summary (audit)"
  ssh_run "root@$TARGET_HOST" 'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -60'

  log "-- old snapshots (>30d) -- listing only, review manually"
  ssh_run "root@$TARGET_HOST" 'find /var/lib/rancher/k3s/server/db/snapshots/ -type f -mtime +30 -name "pre-*" -print 2>/dev/null || true'

  log "=== $NODE OS update complete -- attach $LOG_LOCAL to the execution ticket ==="
}

# --------------------------------------------------------------------------- #
# --run — orchestrate all phases with confirmations
# --------------------------------------------------------------------------- #
phase_run_all() {
  log "=== full $NODE update run (log: $LOG_LOCAL) ==="

  confirm "Phase A (add swap) -- proceed?"
  phase_add_swap

  confirm "Phase B (preflight) -- proceed?"
  phase_preflight

  confirm "Phase C (cordon + drain) -- proceed?"
  phase_drain

  confirm "Phase D (apt) -- proceed?"
  phase_apt

  confirm "Phase E (reboot $NODE; api-server on THIS node unavailable ~90-180s, other 2 CPs keep serving) -- proceed?"
  phase_reboot

  confirm "Phase F (uncordon + verify) -- proceed?"
  phase_finalize

  log "=== FULL RUN COMPLETE for $NODE ==="
}

# --------------------------------------------------------------------------- #
# Phase dispatch
# --------------------------------------------------------------------------- #
: > "$LOG_LOCAL"
log "update-cp-node.sh started (NODE=$NODE, TARGET_HOST=$TARGET_HOST, PEER_HOST=$PEER_HOST, DRY_RUN=$DRY_RUN)"
log "log file: $LOG_LOCAL"

case "$1" in
  --dry-run)
    DRY_RUN=1
    export DRY_RUN
    log "DRY_RUN=1 -- walking Phases A..F without touching state"
    phase_add_swap
    phase_preflight
    phase_drain
    phase_apt
    phase_reboot
    phase_finalize
    ;;
  --add-swap)       phase_add_swap ;;
  --preflight)      phase_preflight ;;
  --drain)          phase_drain ;;
  --apt)            phase_apt ;;
  --reboot)         phase_reboot ;;
  --finalize)       phase_finalize ;;
  --run)            phase_run_all ;;
  -h|--help)        usage ;;
  *)                echo "unknown arg: $1" >&2; usage ;;
esac