#!/bin/bash # ensure-node-docker.sh — ensure docker.io is installed, enabled, and running on # a node that carries (or should carry) the `basicstack.de/docker=true` label. # # Runs from the operator machine (or the control plane); needs kubectl and # ssh access to root@. # # Behavior: # - If the node has label `basicstack.de/docker=true`, ensure docker.io is # installed, marked `apt-mark manual`, systemd `docker` is enabled+active, # and /var/run/docker.sock exists. # - Re-apply the label (idempotent) so that returning nodes always end the # step in a known state. # - If the node does NOT carry the label, this is a no-op — do NOT install # docker on nodes that were not designated to run the Forgejo runner. # # Usage: # ensure-node-docker.sh # # Exit codes: # 0 — node is either not designated (no label) or Docker is confirmed healthy # 1 — hard failure: label present but docker could not be brought up # # Motivated by DEV-499: the weekly rolling OS update was purging docker.io from # workers, which broke the Forgejo runner (DEV-498). Making Docker part of the # post-reboot reconciliation removes the manual "apt-get install docker.io && # systemctl enable --now docker && kubectl label node" step. set -euo pipefail NODE="${1:-}" if [ -z "$NODE" ]; then echo "usage: $0 " >&2 exit 2 fi CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}" SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new}" DOCKER_LABEL_KEY="basicstack.de/docker" log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] ensure-docker/$NODE: $*"; } die() { log "FATAL: $*"; exit 1; } # node -> ssh target. Keep in sync with update-node.sh::node_ssh_target. node_ssh_target() { case "$1" in k3s-cp-1) echo "root@178.105.17.239" ;; k3s-worker-1) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;; k3s-worker-2) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;; k3s-worker-3) echo "root@167.233.121.121" ;; k3s-worker-4) echo "root@128.140.3.80" ;; k3s-worker-5) echo "root@167.233.192.86" ;; k3s-update-runner) echo "root@167.233.79.65" ;; *) die "unknown node $1 — update node_ssh_target() in $0" ;; esac } # --- 1. is this node designated to run Docker? -------------------------------- kubectl get node "$NODE" >/dev/null || die "node $NODE not found in cluster" LABEL_VAL=$(kubectl get node "$NODE" \ -o jsonpath="{.metadata.labels.${DOCKER_LABEL_KEY//./\\.}}" 2>/dev/null || echo "") if [ "$LABEL_VAL" != "true" ]; then log "no ${DOCKER_LABEL_KEY}=true label — skipping Docker reconciliation" exit 0 fi SSH_TARGET=$(node_ssh_target "$NODE") log "label ${DOCKER_LABEL_KEY}=true present — reconciling docker.io via $SSH_TARGET" # --- 2. ensure docker.io on the node ----------------------------------------- REMOTE=$(cat <<'REMOTE' set -euo pipefail export DEBIAN_FRONTEND=noninteractive APT_OPTS='-y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold' # Recover from any half-finished dpkg state before touching apt. dpkg --configure -a >/dev/null 2>&1 || true if ! dpkg -s docker.io >/dev/null 2>&1; then echo "docker.io not installed — installing" apt-get update apt-get $APT_OPTS install docker.io else echo "docker.io already installed" fi # Keep docker.io out of the reach of apt-get autoremove --purge, which is what # the weekly OS-update cycle runs. Idempotent. apt-mark manual docker.io >/dev/null systemctl enable --now docker # Wait for the docker socket to appear so the forgejo-runner pod can bind it. for _ in $(seq 1 30); do if [ -S /var/run/docker.sock ]; then break; fi sleep 1 done if [ ! -S /var/run/docker.sock ]; then echo "docker.sock missing after enable" >&2 systemctl status docker --no-pager | tail -20 >&2 exit 1 fi echo "docker OK: $(docker version --format '{{.Server.Version}}')" REMOTE ) ssh $SSH_OPTS $SSH_TARGET "bash -s" <<< "$REMOTE" | sed 's/^/ /' # --- 3. re-apply the label (idempotent, in case someone stripped it) --------- kubectl label node "$NODE" "${DOCKER_LABEL_KEY}=true" --overwrite >/dev/null log "docker + label reconciled OK"