113 lines
4.1 KiB
Bash
113 lines
4.1 KiB
Bash
|
#!/bin/bash
|
||
|
|
# ensure-node-docker.sh — ensure docker.io is installed, enabled, and running on
|
||
|
|
# a node that carries (or should carry) the `basicstack.de/docker=true` label.
|
||
|
|
#
|
||
|
|
# Runs from the operator machine (or the control plane); needs kubectl and
|
||
|
|
# ssh access to root@<node-ssh-target>.
|
||
|
|
#
|
||
|
|
# Behavior:
|
||
|
|
# - If the node has label `basicstack.de/docker=true`, ensure docker.io is
|
||
|
|
# installed, marked `apt-mark manual`, systemd `docker` is enabled+active,
|
||
|
|
# and /var/run/docker.sock exists.
|
||
|
|
# - Re-apply the label (idempotent) so that returning nodes always end the
|
||
|
|
# step in a known state.
|
||
|
|
# - If the node does NOT carry the label, this is a no-op — do NOT install
|
||
|
|
# docker on nodes that were not designated to run the Forgejo runner.
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# ensure-node-docker.sh <node-name>
|
||
|
|
#
|
||
|
|
# Exit codes:
|
||
|
|
# 0 — node is either not designated (no label) or Docker is confirmed healthy
|
||
|
|
# 1 — hard failure: label present but docker could not be brought up
|
||
|
|
#
|
||
|
|
# Motivated by DEV-499: the weekly rolling OS update was purging docker.io from
|
||
|
|
# workers, which broke the Forgejo runner (DEV-498). Making Docker part of the
|
||
|
|
# post-reboot reconciliation removes the manual "apt-get install docker.io &&
|
||
|
|
# systemctl enable --now docker && kubectl label node" step.
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
NODE="${1:-}"
|
||
|
|
if [ -z "$NODE" ]; then
|
||
|
|
echo "usage: $0 <node-name>" >&2
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
|
||
|
|
CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}"
|
||
|
|
SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new}"
|
||
|
|
DOCKER_LABEL_KEY="basicstack.de/docker"
|
||
|
|
|
||
|
|
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] ensure-docker/$NODE: $*"; }
|
||
|
|
die() { log "FATAL: $*"; exit 1; }
|
||
|
|
|
||
|
|
# node -> ssh target. Keep in sync with update-node.sh::node_ssh_target.
|
||
|
|
node_ssh_target() {
|
||
|
|
case "$1" in
|
||
|
|
k3s-cp-1) echo "root@178.105.17.239" ;;
|
||
|
|
k3s-worker-1) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
|
||
|
|
k3s-worker-2) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
|
||
|
|
k3s-worker-3) echo "root@167.233.121.121" ;;
|
||
|
|
k3s-worker-4) echo "root@128.140.3.80" ;;
|
||
|
|
k3s-worker-5) echo "root@167.233.192.86" ;;
|
||
|
|
k3s-update-runner) echo "root@167.233.79.65" ;;
|
||
|
|
*) die "unknown node $1 — update node_ssh_target() in $0" ;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
# --- 1. is this node designated to run Docker? --------------------------------
|
||
|
|
kubectl get node "$NODE" >/dev/null || die "node $NODE not found in cluster"
|
||
|
|
LABEL_VAL=$(kubectl get node "$NODE" \
|
||
|
|
-o jsonpath="{.metadata.labels.${DOCKER_LABEL_KEY//./\\.}}" 2>/dev/null || echo "")
|
||
|
|
|
||
|
|
if [ "$LABEL_VAL" != "true" ]; then
|
||
|
|
log "no ${DOCKER_LABEL_KEY}=true label — skipping Docker reconciliation"
|
||
|
|
exit 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
SSH_TARGET=$(node_ssh_target "$NODE")
|
||
|
|
log "label ${DOCKER_LABEL_KEY}=true present — reconciling docker.io via $SSH_TARGET"
|
||
|
|
|
||
|
|
# --- 2. ensure docker.io on the node -----------------------------------------
|
||
|
|
REMOTE=$(cat <<'REMOTE'
|
||
|
|
set -euo pipefail
|
||
|
|
export DEBIAN_FRONTEND=noninteractive
|
||
|
|
APT_OPTS='-y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold'
|
||
|
|
|
||
|
|
# Recover from any half-finished dpkg state before touching apt.
|
||
|
|
dpkg --configure -a >/dev/null 2>&1 || true
|
||
|
|
|
||
|
|
if ! dpkg -s docker.io >/dev/null 2>&1; then
|
||
|
|
echo "docker.io not installed — installing"
|
||
|
|
apt-get update
|
||
|
|
apt-get $APT_OPTS install docker.io
|
||
|
|
else
|
||
|
|
echo "docker.io already installed"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Keep docker.io out of the reach of apt-get autoremove --purge, which is what
|
||
|
|
# the weekly OS-update cycle runs. Idempotent.
|
||
|
|
apt-mark manual docker.io >/dev/null
|
||
|
|
|
||
|
|
systemctl enable --now docker
|
||
|
|
|
||
|
|
# Wait for the docker socket to appear so the forgejo-runner pod can bind it.
|
||
|
|
for _ in $(seq 1 30); do
|
||
|
|
if [ -S /var/run/docker.sock ]; then break; fi
|
||
|
|
sleep 1
|
||
|
|
done
|
||
|
|
if [ ! -S /var/run/docker.sock ]; then
|
||
|
|
echo "docker.sock missing after enable" >&2
|
||
|
|
systemctl status docker --no-pager | tail -20 >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "docker OK: $(docker version --format '{{.Server.Version}}')"
|
||
|
|
REMOTE
|
||
|
|
)
|
||
|
|
|
||
|
|
ssh $SSH_OPTS $SSH_TARGET "bash -s" <<< "$REMOTE" | sed 's/^/ /'
|
||
|
|
|
||
|
|
# --- 3. re-apply the label (idempotent, in case someone stripped it) ---------
|
||
|
|
kubectl label node "$NODE" "${DOCKER_LABEL_KEY}=true" --overwrite >/dev/null
|
||
|
|
log "docker + label reconciled OK"
|