stack.basicstack.de/infrastructure/scripts/os-update/os-update.sh

#!/bin/bash
# os-update.sh — full weekly rolling OS-update cycle.
#
# Behavior:
#   1. preflight cluster health (fail-closed)
#   2. take an etcd snapshot
#   3. compute node order:
#        workers (non-Stalwart first, Stalwart-hosting fsn1 workers last)
#        then control-plane nodes (etcd followers first, leader last).
#   4. call update-node.sh for each worker and update-cp-node.sh for each CP,
#      halting on any failure
#   5. finalization: health snapshot + apt history digest
#
# Usage:
#   os-update.sh [--dry-run] [--start-from <node>] [--only <node>]
#                [--skip-cps] [--only-cps]
#
# Flags:
#   --skip-cps   run workers only (legacy behavior, useful for narrow patches).
#   --only-cps   skip workers, only walk the CP phase.
#   --only <n>   update a single node (worker or CP, routed automatically).
#
# Environment:
#   CONTROL_PLANE_HOST (default 178.105.17.239)
#   POST_CP_WAIT_SECONDS (default 180) — settle between CPs after uncordon.
#   All env vars honored by update-node.sh / update-cp-node.sh are honored here.
#
# Read OS_UPDATE_PROCEDURE.md alongside this script; the script mirrors it
# step-for-step and the doc is the authoritative reference. CP handling is
# delegated to update-cp-node.sh; see CP_UPDATE_PROCEDURE.md.
set -euo pipefail

CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}"
POST_CP_WAIT_SECONDS="${POST_CP_WAIT_SECONDS:-180}"
DRY_RUN=0
START_FROM=""
ONLY=""
SKIP_CPS=0
ONLY_CPS=0

while [ $# -gt 0 ]; do
  case "$1" in
    --dry-run)     DRY_RUN=1; shift ;;
    --start-from)  START_FROM="$2"; shift 2 ;;
    --only)        ONLY="$2"; shift 2 ;;
    --skip-cps)    SKIP_CPS=1; shift ;;
    --only-cps)    ONLY_CPS=1; shift ;;
    -h|--help)     grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
    *)             echo "unknown arg: $1" >&2; exit 2 ;;
  esac
done

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
STAMP=$(date -u +%Y%m%dT%H%M%SZ)
LOG_DIR="/tmp/os-update-${STAMP}"
mkdir -p "$LOG_DIR"

log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_DIR/main.log"; }
die() { log "FATAL: $*"; exit 1; }

log "=== os-update.sh cycle $STAMP ==="
log "log dir: $LOG_DIR"

# --- 0. sanity checks ---------------------------------------------------------
command -v kubectl >/dev/null || die "kubectl not on PATH"
command -v jq >/dev/null      || die "jq not on PATH (needed for health checks)"

# --- 1. preflight -------------------------------------------------------------
log "[preflight] cluster health"
if ! RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh" | tee "$LOG_DIR/preflight.log"; then
  die "cluster is not healthy at preflight — refuse to start OS updates"
fi

# --- 2. etcd snapshot ---------------------------------------------------------
if [ "$DRY_RUN" -eq 0 ]; then
  log "[preflight] taking k3s etcd snapshot"
  ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@"$CONTROL_PLANE_HOST" \
    "k3s etcd-snapshot save --name pre-os-update-$(date +%Y%m%d)" | tee "$LOG_DIR/etcd-snapshot.log" || \
      log "WARN: etcd snapshot failed — continuing but this is a risk on CP reboot"
else
  log "[preflight] DRY-RUN — skipping etcd snapshot"
fi

# --- 3. node ordering ---------------------------------------------------------
# Ordering rule:
#   - workers first (non-Stalwart, then Stalwart-hosting fsn1 last)
#   - control-plane nodes AFTER workers, one at a time, etcd followers first,
#     leader last. Rationale (see CP_UPDATE_PROCEDURE.md):
#     * two of three CPs down = etcd quorum loss = write outage. Only one CP
#       cordoned/rebooting at a time.
#     * draining a follower is a no-op for raft leadership; draining the leader
#       forces a re-election, so doing followers first minimises leader flapping.
#   Since DEV-510 (2026-08-22) the cluster runs HA embedded etcd across
#   cp-1/cp-2/cp-3 with 3-node quorum. This script delegates all CP steps to
#   update-cp-node.sh, which handles swap (idempotent), preflight, drain, apt,
#   reboot with /livez polling, etcd rejoin check, and finalize.
STALWART_NODE=$(kubectl -n stalwart get pod -l app=stalwart -o jsonpath='{.items[*].spec.nodeName}' 2>/dev/null | tr ' ' '\n' | sort -u || true)
# If the pod's not currently up (e.g. Pending) we still want to protect fsn1 workers.

CP_NODES=$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' || true)
ALL_NODES=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n')

is_cp() {
  local n="$1"
  for c in $CP_NODES; do [ "$c" = "$n" ] && return 0; done
  return 1
}

workers=()
stalwart_workers=()
for n in $ALL_NODES; do
  is_cp "$n" && continue
  if [ -n "$STALWART_NODE" ] && [ "$n" = "$STALWART_NODE" ]; then
    stalwart_workers+=("$n")
    continue
  fi
  # Any fsn1 worker is a potential Stalwart host — schedule after non-Stalwart nodes.
  loc=$(kubectl get node "$n" -o jsonpath='{.metadata.labels.csi\.hetzner\.cloud/location}' 2>/dev/null || echo "")
  if [ "$loc" = "fsn1" ]; then
    stalwart_workers+=("$n")
  else
    workers+=("$n")
  fi
done

WORKER_ORDER=("${workers[@]}" "${stalwart_workers[@]}")

# --- 3b. control-plane ordering (leader last) --------------------------------
# CP-node → public IP map. Kept out-of-cluster so a full-cluster incident
# doesn't hide etcd from the operator machine.
declare -A CP_HOST=(
  [k3s-cp-1]="178.105.17.239"
  [k3s-cp-2]="188.245.85.199"
  [k3s-cp-3]="49.13.92.162"
)

etcd_leader_ip() {
  # Query etcd leader endpoint from the operator's control-plane host. Uses
  # the same TLS material as update-cp-node.sh. Prints "IP" on success,
  # nothing on failure.
  ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -o BatchMode=yes \
    root@"$CONTROL_PLANE_HOST" \
    'ETCDCTL_API=3 etcdctl \
      --endpoints=https://127.0.0.1:2379 \
      --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
      --cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
      --key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \
      endpoint status --cluster -w simple 2>/dev/null' \
    | awk -F, '$5 ~ /true/ {print $1}' \
    | sed -E 's|https?://([^:]+):.*|\1|'
}

CP_ORDER=()
if [ -n "$CP_NODES" ]; then
  LEADER_IP="$(etcd_leader_ip || true)"
  leader_node=""
  followers=()
  for n in $CP_NODES; do
    ip="${CP_HOST[$n]:-}"
    if [ -n "$ip" ] && [ "$ip" = "$LEADER_IP" ]; then
      leader_node="$n"
    else
      followers+=("$n")
    fi
  done
  if [ -n "$leader_node" ]; then
    CP_ORDER=("${followers[@]}" "$leader_node")
    log "[plan] etcd leader is $leader_node ($LEADER_IP) — updating followers first, leader last"
  else
    # Couldn't identify leader — refuse to guess. Fall back to no CPs.
    CP_ORDER=()
    log "[plan] WARN could not identify etcd leader — skipping CP phase this cycle"
  fi
fi

# Route --only <node> automatically to worker or CP list.
if [ -n "$ONLY" ]; then
  if is_cp "$ONLY"; then
    CP_ORDER=("$ONLY")
    WORKER_ORDER=()
  else
    WORKER_ORDER=("$ONLY")
    CP_ORDER=()
  fi
elif [ -n "$START_FROM" ]; then
  new=()
  skip=1
  for n in "${WORKER_ORDER[@]}"; do
    [ "$n" = "$START_FROM" ] && skip=0
    [ $skip -eq 0 ] && new+=("$n")
  done
  WORKER_ORDER=("${new[@]}")
  # If START_FROM is a CP, trim CP_ORDER instead.
  if is_cp "$START_FROM"; then
    new_cp=()
    skip=1
    for n in "${CP_ORDER[@]:-}"; do
      [ -z "$n" ] && continue
      [ "$n" = "$START_FROM" ] && skip=0
      [ $skip -eq 0 ] && new_cp+=("$n")
    done
    CP_ORDER=("${new_cp[@]}")
    WORKER_ORDER=()
  fi
fi

[ "$SKIP_CPS" -eq 1 ] && CP_ORDER=()
[ "$ONLY_CPS" -eq 1 ] && WORKER_ORDER=()

log "[plan] workers (${#WORKER_ORDER[@]}): ${WORKER_ORDER[*]:-none}"
log "[plan] control-planes (${#CP_ORDER[@]}): ${CP_ORDER[*]:-none}"
{
  echo "# workers"
  printf '%s\n' "${WORKER_ORDER[@]:-}"
  echo "# control-planes"
  printf '%s\n' "${CP_ORDER[@]:-}"
} > "$LOG_DIR/plan.txt"

if [ "$DRY_RUN" -eq 1 ]; then
  log "DRY-RUN — plan written, no node touched. Exiting."
  exit 0
fi

# --- 4. worker loop ----------------------------------------------------------
updated=()
skipped=()
for n in "${WORKER_ORDER[@]}"; do
  [ -z "$n" ] && continue
  log "===================================================================="
  log "==> updating worker $n"
  log "===================================================================="
  NODE_LOG="$LOG_DIR/${n}.log"
  set +e
  "$SCRIPT_DIR/update-node.sh" "$n" 2>&1 | tee "$NODE_LOG"
  rc=${PIPESTATUS[0]}
  set -e
  case $rc in
    0) updated+=("$n") ;;
    3) skipped+=("$n:drain-blocked") ;;
    *) die "update-node.sh failed for $n (rc=$rc). Cycle halted. See $NODE_LOG" ;;
  esac
done

# --- 4b. control-plane loop --------------------------------------------------
# One CP at a time, followers first, leader last. Delegates to update-cp-node.sh
# --run (Phase A add-swap is idempotent, Phase B..F run with ASSUME_YES=1).
# Between CPs we run cluster-health.sh and re-verify etcd cluster status; halt
# on any yellow signal.
if [ ${#CP_ORDER[@]} -gt 0 ]; then
  log "===================================================================="
  log "==> control-plane phase (${#CP_ORDER[@]} nodes): ${CP_ORDER[*]}"
  log "===================================================================="
  first_cp=1
  for cp in "${CP_ORDER[@]}"; do
    [ -z "$cp" ] && continue
    if [ $first_cp -eq 0 ]; then
      log "-- health gate between CPs (settle ${POST_CP_WAIT_SECONDS}s, then re-check)"
      sleep "$POST_CP_WAIT_SECONDS"
      if ! RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh" 2>&1 | tee "$LOG_DIR/inter-cp-health-before-${cp}.log"; then
        die "cluster health degraded between CPs — halting before $cp. See $LOG_DIR/"
      fi
      # Re-check etcd cluster status; every member must still be started.
      ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@"$CONTROL_PLANE_HOST" \
        'ETCDCTL_API=3 etcdctl \
          --endpoints=https://127.0.0.1:2379 \
          --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
          --cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
          --key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \
          endpoint status --cluster -w table' 2>&1 | tee -a "$LOG_DIR/inter-cp-health-before-${cp}.log" || \
        die "etcd endpoint status failed between CPs — halting before $cp"
    fi
    log "===================================================================="
    log "==> updating control-plane $cp"
    log "===================================================================="
    CP_LOG="$LOG_DIR/${cp}.log"
    set +e
    ASSUME_YES=1 "$SCRIPT_DIR/update-cp-node.sh" "$cp" --run 2>&1 | tee "$CP_LOG"
    rc=${PIPESTATUS[0]}
    set -e
    case $rc in
      0) updated+=("$cp") ;;
      *) die "update-cp-node.sh failed for $cp (rc=$rc). Cycle halted. See $CP_LOG" ;;
    esac
    first_cp=0
  done
fi

# --- 5. finalization ----------------------------------------------------------
log "===================================================================="
log "==> finalization"
log "===================================================================="

log "[final] cluster health"
"$SCRIPT_DIR/cluster-health.sh" | tee "$LOG_DIR/final-health.log" || \
  die "final cluster health failed after cycle. Do NOT declare success."

log "[final] apt history digest"
{
  for n in "${updated[@]}"; do
    echo "=== $n ==="
    # resolve ssh target via a mini-eval of node_ssh_target-equivalent
    case "$n" in
      k3s-cp-1)           t="root@178.105.17.239" ;;
      k3s-cp-2)           t="root@188.245.85.199" ;;
      k3s-cp-3)           t="root@49.13.92.162" ;;
      k3s-worker-1)       t="-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
      k3s-worker-2)       t="-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
      k3s-worker-3)       t="root@167.233.121.121" ;;
      k3s-worker-4)       t="root@128.140.3.80" ;;
      k3s-worker-5)       t="root@167.233.192.86" ;;
      k3s-worker-6)       t="root@49.12.223.133" ;;
      k3s-update-runner)  t="root@167.233.79.65" ;;
      *)                  echo "  (unknown ssh target)"; continue ;;
    esac
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new $t \
      'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -40' 2>/dev/null \
      || echo "  (could not read apt history)"
  done
} | tee "$LOG_DIR/apt-history.log"

log "=== summary ==="
log "updated (${#updated[@]}): ${updated[*]:-none}"
log "skipped (${#skipped[@]}): ${skipped[*]:-none}"
log "logs:   $LOG_DIR"

if [ ${#skipped[@]} -gt 0 ]; then
  log "cycle finished with skipped nodes — return code 4 so the caller can escalate"
  exit 4
fi

log "cycle complete."