From e4d86deb8a8ed18c011a733cdd05c1ad4b0c78e2 Mon Sep 17 00:00:00 2001
From: John Coffey <john@coffeylabs.org>
Date: Fri, 30 Oct 2026 20:20:19 -0700
Subject: [PATCH] Managed nodes for Ansible labs, and the EX294 3.1 lab

---
 docs/writing-labs.md                          |  63 ++++++
 .../3-1-preparing-managed-nodes/README.md     |  39 ++++
 .../3-1-preparing-managed-nodes/check.sh      |  28 +++
 .../3-1-preparing-managed-nodes/lab.conf      |   3 +
 .../3-1-preparing-managed-nodes/reset.sh      |   4 +
 .../3-1-preparing-managed-nodes/setup.sh      |  19 ++
 le-lab                                        |   2 +-
 lib/lab.sh                                    | 189 ++++++++++++++++++
 8 files changed, 346 insertions(+), 1 deletion(-)
 create mode 100644 labs/ex294/3-1-preparing-managed-nodes/README.md
 create mode 100644 labs/ex294/3-1-preparing-managed-nodes/check.sh
 create mode 100644 labs/ex294/3-1-preparing-managed-nodes/lab.conf
 create mode 100644 labs/ex294/3-1-preparing-managed-nodes/reset.sh
 create mode 100644 labs/ex294/3-1-preparing-managed-nodes/setup.sh

diff --git a/docs/writing-labs.md b/docs/writing-labs.md
index 64a5507..0dc03af 100644
--- a/docs/writing-labs.md
+++ b/docs/writing-labs.md
@@ -44,6 +44,8 @@ with `set -e`. Read `lib/lab.sh` for the helpers; the ones most labs use:
   boot; reset removes it and anything on it
 - `peer_up`, `in_peer CMD`: a second machine as a network namespace, at
   `10.99.0.2` (name `peer`), linked to the VM's `10.99.0.1`
+- `nodes_up N`, `in_node NAME CMD`: managed nodes `node1`..`nodeN` for
+  Ansible labs (see "Managed nodes" below)
 - `hosts_add LINE`: an `/etc/hosts` line reset takes out again
 - `state_set KEY VALUE`, `state_get KEY`: notes for check and reset
 - `sshd_restart`: restart sshd after reset puts its settings back, and
@@ -58,6 +60,67 @@ should run in the peer gets `NetworkNamespacePath=/run/netns/lepeer` and
 `After=le-lab-peer.service` in its unit; the peer comes back at every
 boot.
 
+### Managed nodes
+
+Where a guide has a control node and managed nodes (EX294), the VM is the
+control node and `nodes_up N` (1 to 9) in setup.sh adds `node1`..`nodeN`:
+podman containers from AlmaLinux 10 (`quay.io/almalinuxorg/10-init`, no
+login needed), at `10.99.1.11`, `.12`, ... with names in `/etc/hosts`.
+Each has systemd as PID 1, its own files, packages, users, services and
+cron, and sshd on port 22. `root` and `ansible` (in `wheel`, so sudo asks
+for the password) have password `Lab-Pass-1`, and root may log in with it.
+dnf uses the real AlmaLinux repositories; the node image starts with a
+copy of the VM's dnf metadata, so the first `dnf` in a node is quick.
+The image is about 200 MB to pull (seconds from quay.io) and 300 MB
+built; `nodes_up` takes about 20 seconds once podman is installed. In the
+nodes, dnf moves on from a mirror slower than 100 kB/s after 15 seconds.
+Tested on the RHEL family; EX294 labs are `SYSTEMS="rhel"`.
+
+- `in_node NAME CMD...` runs a command as root in a node (one argument
+  runs as shell code); `in_node -i NAME 'cat >FILE' <<EOF` passes standard
+  input. Use it in check.sh to grade a node: `need "devops exists on node1"
+  'in_node node1 id devops'`.
+- `nodes_names` lists the nodes, `node_addr NAME` prints an address,
+  `nodes_wait` waits until every node's sshd answers (check.sh can start
+  with `nodes_wait 60 || true`).
+- For a root shell on a node, students use `sudo podman exec -it node1 bash`.
+- Reset calls `nodes_down` for you: it removes the containers, their
+  network, `le-lab-nodes.service`, the images setup pulled or built, the
+  `/etc/hosts` lines, and podman itself (with all its containers and
+  images) if `nodes_up` installed it.
+
+The containers are kept, not recreated, so whatever a student does in a
+node survives a reboot of the VM: `le-lab-nodes.service` starts them again
+at boot and waits for their sshd before `multi-user.target`.
+`systemctl reboot` inside a node restarts just that container, also with
+its state kept.
+
+What works in a node: packages (`dnf`), users, groups, passwords and
+sudo, files and templates, services and targets (`systemctl enable`,
+`mask`, `set-default`), cron, the journal, and firewalld once it's
+installed (each node has its own network stack, so a node's firewall
+really blocks the control node). The image has only sshd, sudo, Python,
+cron and a few tools; a lab installs anything else it needs with
+`in_node NAME dnf -y -q install ...`.
+
+What doesn't, because the nodes share the VM's kernel:
+
+- **SELinux** is off inside a node, so `sefcontext`, `seboolean`,
+  `selinux` and `restorecon` tasks have nothing to act on.
+- **Storage**: a node has no block devices and may not mount; partitions,
+  LVM, file systems and fstab mounts fail there.
+- **Time**: `chronyd` is skipped (no permission to set the clock), so it
+  can be enabled but never active.
+- **Rebooting**: the boot ID and uptime are the VM's, so Ansible's
+  `reboot` module waits in vain; reboot the VM instead.
+- **Host names**: `hostnamectl` and `hostname` are refused (podman sets
+  each node's name), and kernel settings (sysctl, boot loader, kernel
+  modules) belong to the VM.
+
+Tasks of those kinds target the control node itself: put `localhost
+ansible_connection=local` in the inventory (a group such as `[storage]`),
+give the VM spare disks with `disk_add`, and grade the VM as usual.
+
 **setup.sh** makes the starting point. Every graded task must fail right
 after setup: if a fresh system already passes a task, set up the opposite.
 Install what the scenario needs, but never what a task asks the student to
diff --git a/labs/ex294/3-1-preparing-managed-nodes/README.md b/labs/ex294/3-1-preparing-managed-nodes/README.md
new file mode 100644
index 0000000..d91654e
--- /dev/null
+++ b/labs/ex294/3-1-preparing-managed-nodes/README.md
@@ -0,0 +1,39 @@
+# EX294 3.1: Preparing managed nodes
+
+Bootstrap an automation user on every managed node with one playbook, run
+it again without changes, and reach a node by address.
+
+## Before you start
+
+This VM is the control node. The lab creates the user `greg` (password
+`Lab-Pass-1`), who has no SSH key yet, and installs `ansible-core`. Work as
+greg, in `/home/greg/ansible`: its `ansible.cfg` reads the inventory
+`inventory`, logs in as `devops` and becomes root.
+
+The managed nodes `node1` (10.99.1.11) and `node2` (10.99.1.12) are
+AlmaLinux 10 containers with their own users and packages, reached over
+SSH by name. On each, `root` and `ansible` have the password `Lab-Pass-1`;
+`ansible` is in `wheel`, so sudo asks for that password. Nothing else is
+set up there. For a root shell on a node, `sudo podman exec -it node1 bash`.
+
+## Tasks
+
+1. As greg, write `/home/greg/ansible/bootstrap.yml` and run it once with
+   `-k`, so that every managed node has the user `devops` with a hashed
+   password, passwordless sudo, and greg's public key. Then
+   `ansible all -m ping` works without `-k`, and `ansible all -a id` shows
+   `uid=0(root)`.
+2. Make a second run of `bootstrap.yml` report `changed=0` on every node.
+3. Point one inventory host at an address instead of a name with
+   `ansible_host`, so `ansible HOST -m ping` reaches it.
+
+## Hints
+
+- `ssh-keygen -t ed25519`; `-k` needs `sshpass` on the control node.
+- `remote_user: ansible` with `become: true` needs `-K` too; with
+  `remote_user: root`, leave out `become`.
+- `password_hash('sha512')`, `update_password: on_create`, a sudoers
+  drop-in with `validate: /usr/sbin/visudo -cf %s`.
+- `ansible.posix.authorized_key` needs `ansible-galaxy collection install
+  ansible.posix`; `ansible.builtin.file` and `lineinfile` do the same job.
+- `ansible-inventory --list` shows what Ansible makes of the inventory.
diff --git a/labs/ex294/3-1-preparing-managed-nodes/check.sh b/labs/ex294/3-1-preparing-managed-nodes/check.sh
new file mode 100644
index 0000000..974f223
--- /dev/null
+++ b/labs/ex294/3-1-preparing-managed-nodes/check.sh
@@ -0,0 +1,28 @@
+nodes_wait 60 || true
+# ans CMD: run CMD as greg in his project directory.
+ans() { timeout 180 runuser -l greg -c "cd ~/ansible && $*" </dev/null; }
+OUT=$LAB_STATE/check.out
+# hashed NODE: devops has a password hash there, not a lock or nothing.
+hashed() { in_node "$1" getent shadow devops | cut -d: -f2 | grep -q '^\$'; }
+
+task 1 "devops on every node: hashed password, passwordless sudo, greg's key"
+for n in $(nodes_names); do
+	need "user devops exists on $n" "in_node $n id devops"
+	need "devops has a hashed password on $n" hashed $n
+	need "devops may run any command with sudo and no password on $n" "in_node $n 'sudo -l -U devops' | grep -q 'NOPASSWD: ALL'"
+	need "greg logs in to $n as devops with his key" "timeout 20 runuser -u greg -- ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 devops@$n true"
+done
+need "ansible all -m ping works without -k (as greg, in ~/ansible)" 'ans ansible all -m ping'
+need "ansible all -a id shows uid=0(root) on every host" 'ans ansible all -o -a id >"$OUT" && grep -q . "$OUT" && ! grep -v "uid=0(root)" "$OUT"'
+
+task 2 "A second run of bootstrap.yml reports changed=0 on every node"
+need "/home/greg/ansible/bootstrap.yml exists" '[ -f /home/greg/ansible/bootstrap.yml ]'
+need "ansible-playbook bootstrap.yml reports changed=0 and no failures" 'ans ansible-playbook bootstrap.yml -e ansible_password=Lab-Pass-1 -e ansible_become_password=Lab-Pass-1 >"$OUT"; grep -Eq "^node[0-9]+ +: ok=" "$OUT" && ! grep -E " : ok=" "$OUT" | grep -Eq "(changed|unreachable|failed)=[1-9]"'
+
+task 3 "One inventory host is reached by address, with ansible_host"
+H=$(ans ansible-inventory --list 2>/dev/null | python3 -c 'import sys, json, re
+d = json.load(sys.stdin)
+print(next((h for h, v in d.get("_meta", {}).get("hostvars", {}).items() if re.fullmatch(r"\d+(\.\d+){3}", str(v.get("ansible_host", "")))), ""))' 2>/dev/null)
+need "an inventory host has ansible_host set to an IPv4 address" '[ -n "$H" ]'
+need "ansible $H -m ping works" 'ans ansible "$H" -m ping'
+rm -f "$OUT"
diff --git a/labs/ex294/3-1-preparing-managed-nodes/lab.conf b/labs/ex294/3-1-preparing-managed-nodes/lab.conf
new file mode 100644
index 0000000..632c10e
--- /dev/null
+++ b/labs/ex294/3-1-preparing-managed-nodes/lab.conf
@@ -0,0 +1,3 @@
+TITLE="3.1 Preparing managed nodes"
+CHAPTER=/ex294/3-managed-nodes/3-1-preparing-managed-nodes/
+SYSTEMS="rhel"
diff --git a/labs/ex294/3-1-preparing-managed-nodes/reset.sh b/labs/ex294/3-1-preparing-managed-nodes/reset.sh
new file mode 100644
index 0000000..bcee485
--- /dev/null
+++ b/labs/ex294/3-1-preparing-managed-nodes/reset.sh
@@ -0,0 +1,4 @@
+# The nodes go with nodes_down and greg (with his project) with
+# users_remove; the packages the lab or its tasks installed here.
+while read -r p; do dnf -y -q remove "$p" >/dev/null 2>&1; done <"$LAB_STATE/missing" 2>/dev/null
+true
diff --git a/labs/ex294/3-1-preparing-managed-nodes/setup.sh b/labs/ex294/3-1-preparing-managed-nodes/setup.sh
new file mode 100644
index 0000000..dae71fa
--- /dev/null
+++ b/labs/ex294/3-1-preparing-managed-nodes/setup.sh
@@ -0,0 +1,19 @@
+# The control node: ansible-core and greg's project, but not sshpass (task 1).
+for p in ansible-core sshpass; do pkg_installed $p || echo $p >>"$LAB_STATE/missing"; done
+pkg_installed ansible-core || pkg_install ansible-core
+user_add greg
+install -d -m 755 -o greg -g greg /home/greg/ansible
+cat >/home/greg/ansible/ansible.cfg <<'X'
+[defaults]
+inventory = inventory
+remote_user = devops
+host_key_checking = false
+
+[privilege_escalation]
+become = true
+X
+printf '[nodes]\nnode1\nnode2\n' >/home/greg/ansible/inventory
+chown greg: /home/greg/ansible/ansible.cfg /home/greg/ansible/inventory
+
+# The managed nodes.
+nodes_up 2
diff --git a/le-lab b/le-lab
index cae0adf..989703d 100755
--- a/le-lab
+++ b/le-lab
@@ -175,7 +175,7 @@ cmd_reset() {
 	if [ -f "$dir/reset.sh" ]; then
 		bash -c '. "$1/lib/lab.sh"; . "$2/reset.sh"' _ "$ROOT" "$dir" || warn "Some of the reset failed; see above."
 	fi
-	bash -c '. "$1/lib/lab.sh"; users_remove; disks_remove; peer_down' _ "$ROOT"
+	bash -c '. "$1/lib/lab.sh"; users_remove; disks_remove; peer_down; nodes_down' _ "$ROOT"
 	rm -rf "$LAB_STATE" "$CURRENT" /root/lab-answers
 	say "Done. Anything you changed outside the lab's own files is still there;"
 	say "roll back to your snapshot for a completely clean VM."
diff --git a/lib/lab.sh b/lib/lab.sh
index a08de77..0df8f7d 100644
--- a/lib/lab.sh
+++ b/lib/lab.sh
@@ -7,6 +7,11 @@ LE_DISKS=$LE_STATE/disks
 LE_PEER_NS=lepeer
 LE_HOST_ADDR=10.99.0.1
 LE_PEER_ADDR=10.99.0.2
+LE_NODES=$LE_STATE/nodes
+LE_NODE_NET=le-lab-nodes
+LE_NODE_PREFIX=10.99.1
+LE_NODE_BASE=quay.io/almalinuxorg/10-init:10
+LE_NODE_IMAGE=localhost/le-lab-node:10
 
 # ---- output -----------------------------------------------------------------
 
@@ -237,6 +242,190 @@ sshd_restart() {
 	return 1
 }
 
+# ---- managed nodes ----------------------------------------------------------
+
+# Tasks written for "managed nodes" (a control node and several machines it
+# manages over SSH) use podman containers: node1..nodeN at
+# $LE_NODE_PREFIX.11, .12, ... (names in /etc/hosts), each an AlmaLinux 10
+# system with systemd as PID 1, its own files, packages, users, services
+# and cron, sshd, and dnf against the real AlmaLinux repositories. root and
+# user "ansible" (in wheel) have password Lab-Pass-1. The containers are
+# kept, not recreated, so everything done in a node survives a reboot of
+# the VM; le-lab-nodes.service starts them again at boot.
+
+# nodes_up N: set up node1..nodeN (1-9) and wait until their sshd answers.
+nodes_up() {
+	local n=${1:-3} i name
+	case $n in [1-9]) ;; *) die "nodes_up takes 1 to 9 nodes" ;; esac
+	mkdir -p "$LE_NODES"
+	if ! command -v podman >/dev/null; then
+		pkg_install podman
+		printf 'podman\n' >"$LE_NODES/installed-podman"
+	fi
+	if ! podman image exists "$LE_NODE_BASE"; then
+		say "${C_DIM}Pulling the node image...${C_0}"
+		printf '%s\n' "$LE_NODE_BASE" >"$LE_NODES/pulled"
+		for i in 1 2 3; do podman pull -q "$LE_NODE_BASE" >/dev/null && break; sleep 5; done
+		podman image exists "$LE_NODE_BASE" || die "could not pull $LE_NODE_BASE"
+	fi
+	if ! podman image exists "$LE_NODE_IMAGE"; then
+		say "${C_DIM}Building the node image...${C_0}"
+		_node_build || die "could not build the node image"
+	fi
+	if ! podman network exists "$LE_NODE_NET"; then
+		podman network create --subnet "$LE_NODE_PREFIX.0/24" --gateway "$LE_NODE_PREFIX.1" \
+			--label le-lab=node "$LE_NODE_NET" >/dev/null
+	fi
+	for i in $(seq 1 "$n"); do
+		name=node$i
+		if ! podman container exists "$name"; then
+			podman create --name "$name" --hostname "$name" --label le-lab=node \
+				--network "$LE_NODE_NET" --ip "$LE_NODE_PREFIX.$((10 + i))" \
+				--systemd always --restart always \
+				--cap-add AUDIT_WRITE,AUDIT_CONTROL,NET_ADMIN,NET_RAW \
+				"$LE_NODE_IMAGE" >/dev/null
+		fi
+		grep -qxF "$name" "$LE_NODES/names" 2>/dev/null || printf '%s\n' "$name" >>"$LE_NODES/names"
+		hosts_add "$LE_NODE_PREFIX.$((10 + i)) $name.lab.example $name"
+	done
+	if [ ! -f /etc/systemd/system/le-lab-nodes.service ]; then
+		cat >/etc/systemd/system/le-lab-nodes.service <<EOF
+[Unit]
+Description=LINUXexpert practice lab managed nodes
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+TimeoutStartSec=300
+ExecStart=/bin/bash -c '. $LE_LIB && nodes_start'
+ExecStop=/bin/bash -c '. $LE_LIB && nodes_stop'
+
+[Install]
+WantedBy=multi-user.target
+EOF
+		systemctl daemon-reload
+		systemctl enable --quiet le-lab-nodes.service
+	fi
+	say "${C_DIM}Starting ${n} managed node(s)...${C_0}"
+	systemctl start le-lab-nodes.service
+	nodes_start
+}
+
+# The node image: the AlmaLinux 10 init image with sshd, sudo, Python and
+# cron, and the two lab logins. The image starts with a copy of this VM's
+# dnf metadata (the same AlmaLinux repositories), so neither the build nor
+# the first dnf run in each node has to download it again. dnf in the build
+# and in the nodes moves on from a mirror slower than 100 kB/s after 15 s.
+_node_build() {
+	local dir cache=()
+	dir=$(mktemp -d)
+	cat >"$dir/Containerfile" <<'EOF'
+FROM base
+RUN if [ -d /le-cache ]; then cp -a /le-cache/. /var/cache/dnf/; fi \
+    && dnf -y -q --setopt=install_weak_deps=False --setopt=minrate=100k --setopt=timeout=15 install \
+        openssh-server openssh-clients sudo python3 cronie procps-ng \
+        iproute iputils hostname tar findutils less \
+    && dnf clean packages
+RUN useradd -m -G wheel ansible \
+    && printf 'root:Lab-Pass-1\nansible:Lab-Pass-1\n' | chpasswd \
+    && printf 'PermitRootLogin yes\n' >/etc/ssh/sshd_config.d/01-le-lab.conf \
+    && printf '# Practice lab: move on from slow mirrors.\nminrate=100k\ntimeout=15\n' >>/etc/dnf/dnf.conf \
+    && systemctl enable sshd crond
+EOF
+	is_rhel && [ -d /var/cache/dnf ] &&
+		cache=(--security-opt label=disable -v /var/cache/dnf:/le-cache:ro)
+	podman build -q --layers=false --from "$LE_NODE_BASE" "${cache[@]}" \
+		-t "$LE_NODE_IMAGE" "$dir" >/dev/null
+	local r=$?
+	rm -rf "$dir"
+	return $r
+}
+
+# nodes_names: the node names, one a line.
+nodes_names() { cat "$LE_NODES/names" 2>/dev/null; }
+
+# node_addr NAME: a node's address.
+node_addr() { printf '%s.%d\n' "$LE_NODE_PREFIX" $((10 + ${1#node})); }
+
+# nodes_start: start every node and wait for its sshd (le-lab-nodes.service,
+# at boot).
+nodes_start() {
+	local name
+	for name in $(nodes_names); do
+		podman start "$name" >/dev/null || warn "le-lab: could not start $name"
+	done
+	nodes_wait
+}
+
+nodes_stop() {
+	local names; names=$(nodes_names)
+	[ -n "$names" ] && podman stop -t 15 $names >/dev/null 2>&1
+	true
+}
+
+# nodes_wait [SECONDS]: wait until every node's sshd accepts connections.
+nodes_wait() {
+	local limit=${1:-120} t=0 name addr
+	for name in $(nodes_names); do
+		addr=$(node_addr "$name")
+		until timeout 2 bash -c "exec 3<>/dev/tcp/$addr/22" 2>/dev/null; do
+			sleep 1; t=$((t + 1))
+			[ "$t" -lt "$limit" ] || { warn "le-lab: $name doesn't answer on port 22"; return 1; }
+		done
+	done
+}
+
+# in_node [-i] NAME CMD...: run a command as root in a node; one argument
+# runs as shell code. With -i it reads standard input (for here-documents);
+# without, it leaves it alone, so it is safe in a while-read loop.
+in_node() {
+	local i=()
+	[ "$1" = -i ] && { i=(-i); shift; }
+	local name=$1; shift
+	if [ $# -eq 1 ]; then
+		podman exec "${i[@]}" "$name" /bin/bash -c "$1"
+	else
+		podman exec "${i[@]}" "$name" "$@"
+	fi
+}
+
+# nodes_down: remove the nodes, their unit, network and images, and podman
+# itself if nodes_up installed it.
+nodes_down() {
+	local unit=/etc/systemd/system/le-lab-nodes.service c
+	if [ -f "$unit" ]; then
+		systemctl disable --quiet le-lab-nodes.service 2>/dev/null
+		systemctl stop le-lab-nodes.service 2>/dev/null
+		rm -f "$unit"
+		systemctl daemon-reload
+	fi
+	if command -v podman >/dev/null; then
+		for c in $(podman ps -a --filter label=le-lab=node --format '{{.Names}}'); do
+			podman rm -f -t 5 "$c" >/dev/null 2>&1
+		done
+		podman network rm -f "$LE_NODE_NET" >/dev/null 2>&1
+		podman rmi -f "$LE_NODE_IMAGE" >/dev/null 2>&1
+		if [ -f "$LE_NODES/pulled" ]; then
+			podman rmi -f "$(cat "$LE_NODES/pulled")" >/dev/null 2>&1
+		fi
+		if [ -f "$LE_NODES/installed-podman" ]; then
+			podman system reset -f >/dev/null 2>&1
+			if is_rhel; then dnf -y -q remove podman >/dev/null 2>&1
+			else apt-get -y -qq purge podman >/dev/null 2>&1 && apt-get -y -qq autoremove >/dev/null 2>&1; fi
+			# What podman left behind; there was none of it before.
+			if ! pkg_installed podman; then
+				rm -rf /var/lib/containers /etc/containers/networks
+				rmdir /etc/containers 2>/dev/null
+			fi
+		fi
+	fi
+	[ -d "$LE_NODES" ] && hosts_remove
+	rm -rf "$LE_NODES"
+	true
+}
+
 # ---- users ------------------------------------------------------------------
 
 # user_add NAME [useradd options]: create a lab user if missing, with bash
-- 
2.54.0