2026-10-31 01:07:58 +00:00
145 changed files with 4220 additions and 0 deletions
@@ -0,0 +1,34 @@
+# LFCA 1.1: Linux operating system
+
+Finding out what system you're on, processes and PIDs, installing and
+removing a package, and looking at the hardware.
+
+## Before you start
+
+Work as root (`sudo -i`). Answers go in `/root/lab-answers/<task>`, which
+the lab creates. The lab starts a `sleep 6000` process in the background
+for task 3. It notes whether `tree` is installed, and reset puts that back.
+
+## Tasks
+
+1. Find the distribution, its version and the kernel version. Write them
+   to `/root/lab-answers/1`, one per line: the `ID` and `VERSION_ID` from
+   `/etc/os-release`, then the output of `uname -r`.
+2. Find the program running as PID 1 with `ps -ef | head`, and write its
+   name to `/root/lab-answers/2`.
+3. Find the PID of the lab's `sleep 6000` process with `ps`, write it to
+   `/root/lab-answers/3`, and end the process with `kill`.
+4. Install the package `tree`, run it, and list the files the package
+   installed. Write the full path of the `tree` program from that list to
+   `/root/lab-answers/4`. Then remove the package.
+5. Look at the hardware with `lscpu`, `free -h`, `lsblk` and `lspci`.
+   Write the number of CPUs that `lscpu` shows to `/root/lab-answers/5`.
+
+## Hints
+
+- `cat /etc/os-release`, `uname -r`.
+- `ps -ef | head -n 3`; the `CMD` column.
+- `ps -ef | grep sleep`, `pgrep -a sleep`, `kill PID`.
+- `dnf install tree` / `apt install tree`, `rpm -ql tree` or
+  `dpkg -L tree`, `dnf remove` / `apt remove`.
+- The `CPU(s):` line of `lscpu`.
@@ -0,0 +1,25 @@
+. /etc/os-release
+A=/root/lab-answers
+PID=$(state_get sleep-pid)
+cpus() { lscpu | awk -F: '/^CPU\(s\):/ {gsub(/ /, "", $2); print $2}'; }
+# has FILE VALUE: a line of FILE is VALUE (case and spaces ignored).
+has() { sed 's/^[[:space:]]*//; s/[[:space:]]*$//' "$1" 2>/dev/null | grep -qixF -- "$2"; }
+
+task 1 "Distribution, version and kernel in /root/lab-answers/1"
+need "the answer has the ID, $ID" 'has $A/1 "$ID"'
+need "the answer has the VERSION_ID, $VERSION_ID" 'has $A/1 "$VERSION_ID"'
+need "the answer has the kernel version from uname -r" 'has $A/1 "$(uname -r)"'
+
+task 2 "PID 1's program in /root/lab-answers/2"
+need "the answer names systemd (or init, its other name)" '[[ "$(answer 2)" =~ (systemd|init) ]]'
+
+task 3 "The sleep process's PID found and the process ended"
+need "/root/lab-answers/3 has the sleep process's PID" '[ "$(answer 3)" = "$PID" ]'
+need "the process has ended" '! ps -o args= -p "$PID" | grep -q "sleep 6000"'
+
+task 4 "tree installed, its files listed, and removed again"
+need "/root/lab-answers/4 has the path of the tree program" '[ "$(answer 4)" = /usr/bin/tree ]'
+need "the tree package is removed" '! pkg_installed tree'
+
+task 5 "The number of CPUs in /root/lab-answers/5"
+need "the answer is the CPU count lscpu shows" '[ "$(answer 5)" = "$(cpus)" ]'
@@ -0,0 +1,3 @@
+TITLE="1.1 Linux operating system"
+CHAPTER=/lfca/1-linux-fundamentals/1-1-linux-operating-system/
+SYSTEMS="rhel debian"
@@ -0,0 +1,9 @@
+systemctl stop le-lab-sleep.service 2>/dev/null
+systemctl reset-failed le-lab-sleep.service 2>/dev/null
+# tree as it was before the lab.
+if [ -n "$(state_get had-tree)" ]; then
+	pkg_installed tree || pkg_install tree
+elif pkg_installed tree; then
+	if is_rhel; then dnf -y -q remove tree >/dev/null; else DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge tree >/dev/null 2>&1; fi
+fi
+true
@@ -0,0 +1,11 @@
+pkg_installed tree && state_set had-tree 1
+# The process for task 3, as if started with "sleep 6000 &".
+systemctl stop le-lab-sleep.service 2>/dev/null || true
+systemctl reset-failed le-lab-sleep.service 2>/dev/null || true
+systemd-run --quiet --unit=le-lab-sleep --description="Practice lab sleep" /usr/bin/sleep 6000
+for i in 1 2 3 4 5; do
+	pid=$(systemctl show -p MainPID --value le-lab-sleep.service)
+	[ "${pid:-0}" != 0 ] && break
+	sleep 1
+done
+state_set sleep-pid "$pid"
@@ -0,0 +1,52 @@
+# LFCA 1.2: Command line
+
+Directories, finding files, reading the SSH log, permissions, archives,
+two editors, and a script of your own on the `PATH`.
+
+## Before you start
+
+Work as root (`sudo -i`), so `~` is `/root`. Answers go in
+`/root/lab-answers/<task>`. The lab creates `~/editme.txt` for task 6 and
+saves your `~/.bashrc` and sshd's settings so reset can put them back.
+Files in `/tmp` don't survive a reboot, so do the tasks in one go and run
+`le-lab check` before you reboot.
+
+## Tasks
+
+1. Make `~/practice/a/b` with one command, move into `b`, and get back home
+   three different ways.
+2. Find every file under `/etc` whose name ends in `.conf`, save the list to
+   `~/conf-files.txt`, count its lines, and write the count to
+   `/root/lab-answers/2`.
+3. Make sure the SSH server is running and allows password logins (turn
+   them on if not), run `ssh localhost` and type a wrong password. Count
+   the failures in sshd's journal and write the count to
+   `/root/lab-answers/3`.
+4. Make a file `/tmp/private.txt` that only you can read and write, check
+   it with `ls -l`, then create user `bob` with a home directory and try
+   `sudo -u bob cat /tmp/private.txt`.
+5. Archive `~/practice` to `~/practice.tar.gz` with `tar -czf`, delete
+   `~/practice`, and restore it from the archive.
+6. Open `~/editme.txt` in vi, change `before` to `after` on the first line,
+   save and quit. Then do the same on the second line in nano (install it
+   if it's missing).
+7. Add `~/scripts` to `PATH` in `~/.bashrc`, put a script `hello` in it
+   with a `#!/bin/bash` first line, make it executable with `chmod u+x`,
+   and run it as `hello` in a new shell.
+
+## Hints
+
+- `mkdir -p`, `cd`, `cd ~`, `cd ../../..`, `cd -` and `pwd`.
+- `find /etc -name '*.conf' > ~/conf-files.txt`, `wc -l`.
+- `systemctl status sshd` (`ssh` on Ubuntu). `sshd -T | grep passwordauth`
+  should say `yes`; many cloud images set it to `no`, on Ubuntu in
+  `/etc/ssh/sshd_config.d/50-cloud-init.conf`. Change it there and
+  `systemctl restart sshd` (`ssh` on Ubuntu).
+- `journalctl -u sshd | grep -c "Failed password"` (`-u ssh` on Ubuntu).
+- `chmod 600`, `useradd -m bob`; the `cat` should fail.
+- `tar -czf ~/practice.tar.gz -C ~ practice`, `rm -r`, `tar -xzf`,
+  `tar -tzf` to look inside.
+- vi: `i` to insert, Esc, `:wq`. nano: Ctrl-O to save, Ctrl-X to quit.
+  `dnf install nano` on the RHEL family.
+- `export PATH="$HOME/scripts:$PATH"` in `~/.bashrc`, `bash` or a new login
+  to get a new shell, `type hello`.
@@ -0,0 +1,47 @@
+A=/root/lab-answers
+# A wrong SSH password since the lab started, in the journal or the log files.
+failed_logged() {
+	journalctl --since "@$(state_get start)" 2>/dev/null | grep -q "Failed password" && return 0
+	local f
+	for f in /var/log/secure /var/log/auth.log; do
+		[ -f "$f" ] || continue
+		[ "$(grep -c 'Failed password' "$f")" -gt "$(state_get "failed-${f##*/}" || echo 0)" ] && return 0
+	done
+	return 1
+}
+new_shell() { env -i HOME=/root TERM=dumb PATH=/usr/sbin:/usr/bin:/sbin:/bin bash -ic "$1" 2>/dev/null; }
+
+task 1 "~/practice/a/b made"
+need "/root/practice/a/b is a directory" '[ -d /root/practice/a/b ]'
+
+task 2 "The .conf files under /etc listed in ~/conf-files.txt and counted"
+need "/root/conf-files.txt lists files under /etc ending in .conf" '[ "$(grep -c "^/etc/.*\.conf$" /root/conf-files.txt)" -gt 20 ] && ! grep -v "\.conf$" /root/conf-files.txt | grep -q .'
+need "it lists most of them" '[ $(( $(wc -l </root/conf-files.txt) * 10 )) -ge $(( $(find /etc -name "*.conf" | wc -l) * 9 )) ]'
+need "/root/lab-answers/2 has its line count" '[ "$(answer 2)" = "$(wc -l </root/conf-files.txt)" ]'
+
+task 3 "A failed SSH password logged and counted"
+need "sshd logged a wrong password since the lab started" failed_logged
+need "/root/lab-answers/3 has the count, a number above 0" '[[ "$(answer 3)" =~ ^[0-9]+$ ]] && [ "$(answer 3)" -gt 0 ]'
+
+task 4 "/tmp/private.txt only for you; bob can't read it"
+need "/tmp/private.txt exists" '[ -f /tmp/private.txt ]'
+need "its permissions are rw------- (600)" '[ "$(stat -c %a /tmp/private.txt)" = 600 ]'
+need "user bob exists, with a home directory" 'id bob && [ -d "$(getent passwd bob | cut -d: -f6)" ]'
+need "bob can't read it" '! runuser -u bob -- cat /tmp/private.txt'
+
+task 5 "~/practice archived to ~/practice.tar.gz and restored"
+need "/root/practice.tar.gz is a gzip-compressed tar archive" 'tar -tzf /root/practice.tar.gz'
+need "the archive has practice/a/b" 'tar -tzf /root/practice.tar.gz | grep -Eq "(^|/)practice/a/b/?$"'
+need "/root/practice/a/b is back" '[ -d /root/practice/a/b ]'
+
+task 6 "~/editme.txt edited in vi and nano"
+need "the first line says after" '[ "$(sed -n 1p /root/editme.txt)" = "This line is for vi: after" ]'
+need "the second line says after" '[ "$(sed -n 2p /root/editme.txt)" = "This line is for nano: after" ]'
+need "nano is installed" 'command -v nano'
+
+task 7 "~/scripts on PATH from ~/.bashrc, with a script hello"
+need "/root/scripts/hello starts with #!/bin/bash" '[ "$(head -n1 /root/scripts/hello)" = "#!/bin/bash" ]'
+need "it's executable by its owner" '[ -x /root/scripts/hello ] && stat -c %A /root/scripts/hello | grep -q "^...x"'
+need "~/.bashrc puts ~/scripts on PATH" 'grep -q "scripts" /root/.bashrc && grep "scripts" /root/.bashrc | grep -q PATH'
+need "a new shell finds hello in ~/scripts" '[ "$(new_shell "type -P hello")" = /root/scripts/hello ]'
+need "hello runs in a new shell" 'new_shell hello'
@@ -0,0 +1,3 @@
+TITLE="1.2 Command line"
+CHAPTER=/lfca/1-linux-fundamentals/1-2-command-line/
+SYSTEMS="rhel debian"
@@ -0,0 +1,22 @@
+rm -rf /root/practice /root/practice.tar.gz /root/conf-files.txt /root/editme.txt /root/scripts /tmp/private.txt
+if id bob >/dev/null 2>&1; then
+	pkill -KILL -u bob 2>/dev/null
+	userdel -r bob 2>/dev/null
+fi
+[ -f "$LAB_STATE/bashrc" ] && cp -a "$LAB_STATE/bashrc" /root/.bashrc
+if [ -z "$(state_get had-nano)" ] && pkg_installed nano; then
+	if is_rhel; then dnf -y -q remove nano >/dev/null; else DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge nano >/dev/null 2>&1; fi
+fi
+# sshd as it was: removed if task 3 installed it, else its settings back.
+if [ -z "$(state_get had-sshd)" ]; then
+	if pkg_installed openssh-server; then
+		if is_rhel; then dnf -y -q remove openssh-server >/dev/null 2>&1; else DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge openssh-server >/dev/null 2>&1; fi
+	fi
+elif [ -f "$LAB_STATE/sshd.tar" ] &&
+	[ "$(find /etc/ssh/sshd_config /etc/ssh/sshd_config.d -type f -exec sha256sum {} + 2>/dev/null | sort)" != "$(cat "$LAB_STATE/sshd.sums")" ]; then
+	rm -rf /etc/ssh/sshd_config.d
+	tar -C /etc -xf "$LAB_STATE/sshd.tar"
+	command -v restorecon >/dev/null && restorecon -R /etc/ssh
+	if is_rhel; then systemctl try-restart sshd; else systemctl try-restart ssh.service; fi
+fi
+true
@@ -0,0 +1,23 @@
+for p in /root/practice /root/practice.tar.gz /root/scripts /tmp/private.txt; do
+	[ ! -e "$p" ] || refuse "$p already exists; move it away first"
+done
+id bob >/dev/null 2>&1 && refuse "user bob already exists; remove it first (userdel -r bob)"
+
+pkg_installed nano && state_set had-nano 1
+pkg_installed openssh-server && state_set had-sshd 1
+# sshd's settings, which task 3 may change: reset puts them back.
+if [ -f /etc/ssh/sshd_config ]; then
+	files=ssh/sshd_config
+	[ -d /etc/ssh/sshd_config.d ] && files="$files ssh/sshd_config.d"
+	tar -C /etc -cf "$LAB_STATE/sshd.tar" $files
+	find /etc/ssh/sshd_config /etc/ssh/sshd_config.d -type f -exec sha256sum {} + 2>/dev/null | sort >"$LAB_STATE/sshd.sums" || true
+fi
+if [ -f /root/.bashrc ]; then cp -a /root/.bashrc "$LAB_STATE/bashrc"; fi
+printf 'This line is for vi: before\nThis line is for nano: before\n' >/root/editme.txt
+
+# Where the SSH log stands now, so check counts only new failures.
+state_set start "$(date +%s)"
+for f in /var/log/secure /var/log/auth.log; do
+	[ -f "$f" ] && state_set "failed-${f##*/}" "$(grep -c 'Failed password' "$f")"
+done
+true
@@ -0,0 +1,37 @@
+# LFCA 2.1: System administration
+
+A sudo user, a web server as a service, a new disk mounted at boot, and a
+cron job.
+
+## Before you start
+
+Work as root (`sudo -i`). The lab adds a 1 GiB spare disk, `/dev/loop40`,
+standing in for a second virtual disk; it comes back at every boot. It
+notes whether nginx is installed, and reset puts that back.
+
+## Tasks
+
+1. Create a user, add them to the `wheel` group (RHEL family) or `sudo`
+   group (Ubuntu), log in as them, and run `sudo -l` to see what they may
+   do. Then delete the user. Not graded.
+2. Install nginx, enable and start it with `systemctl enable --now`, check
+   `systemctl status`, fetch the page with `curl localhost`, and read its
+   log with `journalctl -u nginx`.
+3. Partition `/dev/loop40` (one partition is enough), make an ext4
+   filesystem on the partition, mount it at `/data`, add it to
+   `/etc/fstab`, and reboot to prove it comes back.
+4. Add a root crontab entry that writes the date to `/root/cron-date.txt`
+   every two minutes. Check the file once the job has run, then remove the
+   entry, leaving the file.
+
+## Hints
+
+- `useradd -m -G wheel NAME` (or `-G sudo`), `passwd NAME`,
+  `su - NAME`, `sudo -l`, `userdel -r NAME`.
+- `dnf install nginx` / `apt install nginx`, `systemctl enable --now
+  nginx`, `curl -s localhost | head`.
+- `fdisk /dev/loop40` (or `parted`), then `lsblk /dev/loop40`;
+  `mkfs.ext4 /dev/loop40p1`, `blkid`, a `UUID=` line in `/etc/fstab`,
+  `systemctl daemon-reload`, `mount -a`, `findmnt /data`.
+- `crontab -e` with `*/2 * * * * date >> /root/cron-date.txt`;
+  `crontab -l`.
@@ -0,0 +1,29 @@
+# The fstab line for /data, comments left out.
+line() { awk '$1 !~ /^#/ && $2 == "/data"' /etc/fstab; }
+page() {
+	local i
+	for i in 1 2 3 4 5; do
+		[ "$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 http://localhost/)" = 200 ] && return 0
+		sleep 2
+	done
+	return 1
+}
+
+task 1 "A sudo user created, checked and deleted" self
+
+task 2 "nginx installed, running, enabled, serving localhost"
+need "nginx is installed" 'command -v nginx'
+need "nginx starts at boot" enabled nginx
+need "nginx is running" active nginx
+need "curl localhost gets a page" page
+
+task 3 "/dev/loop40 partitioned, ext4 mounted at /data, in /etc/fstab"
+need "/dev/loop40 has a partition" 'lsblk -nro TYPE /dev/loop40 | grep -qx part'
+need "/etc/fstab has one line for /data" '[ "$(line | wc -l)" = 1 ]'
+need "the line says ext4" '[ "$(line | awk "{print \$3}")" = ext4 ]'
+need "/data is mounted from a partition of /dev/loop40" '[[ "$(findmnt -n -o SOURCE /data)" == /dev/loop40p* ]]'
+need "/data is ext4" '[ "$(findmnt -n -o FSTYPE /data)" = ext4 ]'
+
+task 4 "A cron job wrote /root/cron-date.txt, and the entry is gone"
+need "/root/cron-date.txt has a date in it" 'grep -q "20[0-9][0-9]" /root/cron-date.txt'
+need "no crontab entry writes to cron-date.txt any more" '! grep -rqs cron-date /var/spool/cron /etc/crontab /etc/cron.d'
@@ -0,0 +1,3 @@
+TITLE="2.1 System administration"
+CHAPTER=/lfca/2-system-administration/2-1-system-administration/
+SYSTEMS="rhel debian"
@@ -0,0 +1,23 @@
+# Cron entries and their file.
+if crontab -l 2>/dev/null | grep -q cron-date; then
+	crontab -l | grep -v cron-date | crontab -
+fi
+grep -rls cron-date /etc/cron.d 2>/dev/null | xargs -r rm -f
+rm -f /root/cron-date.txt
+
+# /data, before reset removes the disk.
+sed -i -E '/^[^#]*[[:space:]]\/data[[:space:]]/d' /etc/fstab
+systemctl daemon-reload
+findmnt /data >/dev/null && umount -l /data
+[ -n "$(state_get had-data)" ] || rmdir /data 2>/dev/null
+
+# nginx as it was.
+if [ -n "$(state_get had-nginx)" ]; then
+	[ -n "$(state_get nginx-was-enabled)" ] && systemctl enable --quiet nginx
+	[ -n "$(state_get nginx-was-enabled)" ] || systemctl disable --quiet nginx 2>/dev/null
+	if [ -n "$(state_get nginx-was-active)" ]; then systemctl start nginx; else systemctl stop nginx 2>/dev/null; fi
+elif pkg_installed nginx; then
+	systemctl disable --now --quiet nginx 2>/dev/null
+	if is_rhel; then dnf -y -q remove nginx >/dev/null; else DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge nginx nginx-common >/dev/null 2>&1; fi
+fi
+true
@@ -0,0 +1,12 @@
+findmnt /data >/dev/null && refuse "something is mounted at /data; unmount it first"
+grep -Eq '^[^#]*[[:space:]]/data[[:space:]]' /etc/fstab && refuse "/etc/fstab already has a line for /data; remove it first"
+[ ! -e /root/cron-date.txt ] || refuse "/root/cron-date.txt already exists; move it away first"
+[ -d /data ] && state_set had-data 1
+if pkg_installed nginx; then
+	state_set had-nginx 1
+	enabled nginx && state_set nginx-was-enabled 1
+	active nginx && state_set nginx-was-active 1
+	# Task 2 starts from nginx stopped and off at boot.
+	systemctl disable --now --quiet nginx
+fi
+disk_add 0 1G >/dev/null
@@ -0,0 +1,41 @@
+# LFCA 2.2: Best practices
+
+Turning off a service nobody needs, a runbook, a script for a manual task,
+and a first Ansible playbook.
+
+## Before you start
+
+Work as root (`sudo -i`), so `~` is `/root`. The lab starts a service this
+VM doesn't need, listening on TCP port 7070, and creates group
+`developers`. It notes whether nginx and Ansible are installed, and reset
+puts that back. The VM needs internet access to install packages.
+
+## Tasks
+
+1. Check which services listen on the network with `ss -tulpn`. Find the
+   one on port 7070, which this VM doesn't need, and turn it off now and at
+   boot.
+2. Write a runbook for restarting nginx and checking it came back, then
+   have someone else follow it. Not graded.
+3. Turn a three-step manual task into a shell script: write
+   `/root/add-dev.sh` that takes a user name, creates that user, adds them
+   to group `developers`, and creates a directory `/srv/NAME` they own.
+   Make it executable and run it for user `erin`.
+4. Install Ansible (the `ansible` package on Ubuntu, `ansible-core` on the
+   RHEL family). In `/root/ansible`, make an inventory file `inventory`
+   with a `[webservers]` line followed by
+   `localhost ansible_connection=local`, and save the chapter's nginx
+   playbook as `playbook.yml`. Run it against the VM with
+   `ansible-playbook -i inventory -K playbook.yml`, twice, and compare the
+   output.
+
+## Hints
+
+- `ss -tulpn` shows the process; `systemctl status PID` shows its unit.
+  `systemctl disable --now UNIT`.
+- `#!/bin/bash` first, `$1` is the name; `useradd -m`, `usermod -aG` (or
+  `useradd -G`), `mkdir`, `chown`; `chmod +x`.
+- `dnf install ansible-core` / `apt install ansible`. The playbook's
+  `hosts: webservers` matches the inventory's group; `-K` asks for the sudo
+  password (as root, just press Enter). The second run should show
+  `changed=0`.
@@ -0,0 +1,22 @@
+inv=/root/ansible/inventory
+
+task 1 "The service on port 7070 is off, now and at boot"
+need "nothing listens on port 7070" '! listening 7070 tcp'
+need "le-lab-oldapp.service doesn't start at boot" '! enabled le-lab-oldapp.service'
+
+task 2 "A runbook for restarting nginx" self
+
+task 3 "/root/add-dev.sh run for erin"
+need "/root/add-dev.sh is an executable script" '[ -x /root/add-dev.sh ] && head -n1 /root/add-dev.sh | grep -q "^#!"'
+need "user erin exists" 'id erin'
+need "erin is in group developers" 'id -nG erin | grep -qw developers'
+need "/srv/erin is a directory owned by erin" '[ -d /srv/erin ] && [ "$(stat -c %U /srv/erin)" = erin ]'
+
+task 4 "Ansible installed nginx from /root/ansible/playbook.yml"
+need "ansible-playbook is installed" 'command -v ansible-playbook'
+need "/root/ansible/inventory has a [webservers] group" 'grep -qx "[[:space:]]*\[webservers\][[:space:]]*" $inv'
+need "the inventory has localhost ansible_connection=local" 'grep -Eq "^[[:space:]]*localhost[[:space:]]+ansible_connection=local" $inv'
+need "/root/ansible/playbook.yml runs on hosts webservers" 'grep -Eq "hosts:[[:space:]]*\"?webservers" /root/ansible/playbook.yml'
+need "nginx is installed" 'command -v nginx'
+need "nginx starts at boot" enabled nginx
+need "nginx is running" active nginx
@@ -0,0 +1,3 @@
+TITLE="2.2 Best practices"
+CHAPTER=/lfca/2-system-administration/2-2-best-practices/
+SYSTEMS="rhel debian"
@@ -0,0 +1,31 @@
+systemctl disable --now --quiet le-lab-oldapp.service 2>/dev/null
+systemctl unmask --quiet le-lab-oldapp.service 2>/dev/null
+rm -f /etc/systemd/system/le-lab-oldapp.service
+systemctl daemon-reload
+rm -rf /srv/le-oldapp /usr/local/lib/le-lab/httpd.py
+rmdir /usr/local/lib/le-lab 2>/dev/null
+
+if id erin >/dev/null 2>&1; then
+	pkill -KILL -u erin 2>/dev/null
+	userdel -r erin 2>/dev/null
+fi
+rm -rf /srv/erin /root/add-dev.sh /root/ansible
+[ -n "$(state_get had-dot-ansible)" ] || rm -rf /root/.ansible
+[ -n "$(state_get made-developers)" ] && groupdel developers 2>/dev/null
+
+# Packages and nginx as they were.
+if grep -qx nginx "$LAB_STATE/not-installed" 2>/dev/null; then
+	systemctl disable --now --quiet nginx 2>/dev/null
+else
+	[ -n "$(state_get nginx-was-enabled)" ] || systemctl disable --quiet nginx 2>/dev/null
+	[ -n "$(state_get nginx-was-active)" ] || systemctl stop nginx 2>/dev/null
+fi
+if [ -s "$LAB_STATE/not-installed" ]; then
+	if is_rhel; then
+		dnf -y -q remove $(cat "$LAB_STATE/not-installed") >/dev/null 2>&1
+	else
+		grep -qx nginx "$LAB_STATE/not-installed" && echo nginx-common >>"$LAB_STATE/not-installed"
+		DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge $(cat "$LAB_STATE/not-installed") >/dev/null 2>&1
+	fi
+fi
+true
@@ -0,0 +1,53 @@
+id erin >/dev/null 2>&1 && refuse "user erin already exists; remove it first (userdel -r erin)"
+for p in /root/add-dev.sh /root/ansible /srv/erin; do
+	[ ! -e "$p" ] || refuse "$p already exists; move it away first"
+done
+getent group developers >/dev/null || { groupadd developers; state_set made-developers 1; }
+[ -e /root/.ansible ] && state_set had-dot-ansible 1
+
+# Packages the tasks install: reset removes the ones that weren't here.
+for p in $(pick ansible-core ansible) nginx; do
+	pkg_installed "$p" || echo "$p" >>"$LAB_STATE/not-installed"
+done
+if pkg_installed nginx; then
+	enabled nginx && state_set nginx-was-enabled 1
+	active nginx && state_set nginx-was-active 1
+fi
+
+# The service nobody needs.
+mkdir -p /usr/local/lib/le-lab /srv/le-oldapp
+echo 'Old intranet app' >/srv/le-oldapp/index.html
+cat >/usr/local/lib/le-lab/httpd.py <<'X'
+# A static web server like "python3 -m http.server PORT --directory DIR",
+# without its reverse DNS lookup of the address, which can take seconds.
+import argparse, functools, http.server, socketserver
+
+ap = argparse.ArgumentParser()
+ap.add_argument("port", type=int)
+ap.add_argument("--bind", default="")
+ap.add_argument("--directory", default=".")
+a = ap.parse_args()
+
+
+class Server(http.server.ThreadingHTTPServer):
+    def server_bind(self):
+        socketserver.TCPServer.server_bind(self)
+        self.server_name, self.server_port = self.server_address[:2]
+
+
+handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=a.directory)
+Server((a.bind, a.port), handler).serve_forever()
+X
+cat >/etc/systemd/system/le-lab-oldapp.service <<'X'
+[Unit]
+Description=Old intranet app (practice lab)
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/python3 /usr/local/lib/le-lab/httpd.py 7070 --directory /srv/le-oldapp
+
+[Install]
+WantedBy=multi-user.target
+X
+systemctl daemon-reload
+systemctl enable --now --quiet le-lab-oldapp.service
@@ -0,0 +1,51 @@
+# LFCA 2.3: Networking
+
+Addresses and routes, a subnet worked out by hand, listening ports, DNS
+lookups, SSH keys, and a firewall that allows only SSH.
+
+## Before you start
+
+Work as root (`sudo -i`), so `~` is `/root`. Answers go in
+`/root/lab-answers/<task>`. The VM needs internet access for task 4.
+
+The lab runs a web server on port 8080 and creates user `alice` (password
+`Lab-Pass-1`). The peer, at `10.99.0.2` (name `peer`), stands in for a
+computer outside the VM; this VM is `10.99.0.1` there. Get a shell there
+with `sudo ip netns exec lepeer bash`. The lab notes the firewall's
+settings, and reset puts them back. Keep a console open for task 6: a
+firewall mistake can cut off SSH.
+
+## Tasks
+
+1. Find the VM's IPv4 address with its prefix, its default gateway and its
+   DNS servers. Write the address with its prefix (like `192.0.2.10/24`)
+   and the gateway to `/root/lab-answers/1`, one per line.
+2. Work out the network address, broadcast address and usable range of
+   `10.20.30.77/26`. Write them to `/root/lab-answers/2`, one per line:
+   network, broadcast, first usable, last usable. Check your answer with
+   `ipcalc` if it's installed.
+3. List the listening ports with `ss -tulpn` and say which service each
+   belongs to. Write the name of the program listening on TCP port 8080 to
+   `/root/lab-answers/3`.
+4. Look up the `A`, `AAAA` and `MX` records of a domain you know with `dig`
+   (in `bind9-dnsutils` on Ubuntu, `bind-utils` on the RHEL family). Write
+   the domain to `/root/lab-answers/4`.
+5. Set up SSH key login from root to `alice` on this VM: make a key pair
+   for root, so that `ssh alice@localhost` logs in without a password.
+6. Turn on the VM's firewall (firewalld on the RHEL family, ufw on Ubuntu),
+   allow only SSH, and check that the web server on port 8080 can no longer
+   be reached from the peer, while SSH still can.
+
+## Hints
+
+- `ip addr`, `ip route`, `/etc/resolv.conf` or `resolvectl status`.
+- A /26 has 64 addresses; the network address is a multiple of 64.
+  `ipcalc 10.20.30.77/26`.
+- `ss -tulpn`, the `users:(("NAME",...))` column.
+- `dig example.org A`, `dig example.org AAAA +short`, `dig example.org MX`.
+- `ssh-keygen`, `ssh-copy-id alice@localhost`, then
+  `ssh alice@localhost`.
+- `dnf install firewalld`, `systemctl enable --now firewalld`,
+  `firewall-cmd --list-all`, `--remove-service`, `--remove-port`,
+  `--permanent`, `--reload`. `ufw allow OpenSSH`, `ufw enable`,
+  `ufw status`. From the peer: `curl --max-time 5 http://10.99.0.1:8080/`.
@@ -0,0 +1,59 @@
+IF=$(state_get if)
+A=/root/lab-answers
+# has N VALUE: a line of answer N is VALUE (spaces ignored).
+has() { sed 's/^[[:space:]]*//; s/[[:space:]]*$//' "$A/$1" 2>/dev/null | grep -qxF -- "$2"; }
+hasre() { sed 's/^[[:space:]]*//; s/[[:space:]]*$//' "$A/$1" 2>/dev/null | grep -qxE -- "$2"; }
+ADDR=$(ip -4 -o addr show dev "$IF" scope global | awk '{print $4; exit}')
+GW=$(ip -4 route show default | awk '{print $3; exit}')
+SSHOPT=(-o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o PreferredAuthentications=publickey)
+
+# The firewall: firewalld if it's running, otherwise ufw.
+fwd=''; active firewalld && fwd=1
+ufw_on() { command -v ufw >/dev/null && ufw status | grep -q 'Status: active'; }
+zones() { { firewall-cmd --get-zone-of-interface="$IF" 2>/dev/null || firewall-cmd --get-default-zone; firewall-cmd --get-default-zone; } | sort -u; }
+only_ssh() {
+	local z p
+	if [ -n "$fwd" ]; then
+		for z in $(zones); do
+			for p in "" --permanent; do
+				firewall-cmd $p --zone="$z" --list-services | tr ' ' '\n' | grep -Evx 'ssh|dhcpv6-client|' && return 1
+				[ -z "$(firewall-cmd $p --zone="$z" --list-ports)" ] || return 1
+				[ "$(firewall-cmd $p --zone="$z" --get-target)" != ACCEPT ] || return 1
+			done
+		done
+		return 0
+	fi
+	! ufw status | grep ALLOW | grep -v FWD | grep -Evq '^(22(/tcp)?|OpenSSH)[[:space:]]' &&
+		ufw status verbose | grep -Eq 'Default: (deny|reject) \(incoming\)'
+}
+peer_ssh() { in_peer timeout 5 bash -c 'exec 3<>/dev/tcp/10.99.0.1/22; head -c 4 <&3' 2>/dev/null | grep -q SSH; }
+peer_web() { in_peer curl -s --max-time 5 http://10.99.0.1:8080/ | grep -q "Lab web server"; }
+
+task 1 "Address with prefix and default gateway in /root/lab-answers/1"
+need "the answer has the address with its prefix, $ADDR" 'has 1 "$ADDR"'
+need "the answer has the default gateway, $GW" 'has 1 "$GW"'
+
+task 2 "10.20.30.77/26 worked out in /root/lab-answers/2"
+need "the answer has the network address" 'hasre 2 "10\.20\.30\.64(/26)?"'
+need "the answer has the broadcast address" 'hasre 2 "10\.20\.30\.127(/26)?"'
+need "the answer has the first usable address" 'hasre 2 "10\.20\.30\.65(/26)?"'
+need "the answer has the last usable address" 'hasre 2 "10\.20\.30\.126(/26)?"'
+
+task 3 "The program on TCP port 8080 in /root/lab-answers/3"
+need "the answer names the program listening on 8080" '[ -n "$(answer 3)" ] && ss -Htlpn "sport = :8080" | grep -qF "\"$(answer 3)"'
+
+task 4 "dig installed; a domain looked up"
+need "dig is installed" 'command -v dig'
+need "/root/lab-answers/4 is a domain with an A record" '[ -n "$(answer 4)" ] && dig +short +time=3 +tries=2 A "$(answer 4)" | grep -Eq "^[0-9.]+$"'
+
+task 5 "ssh alice@localhost logs root in with a key"
+need "root has a key pair" 'ls /root/.ssh/id_*.pub'
+need "ssh alice@localhost logs in with the key, without a password" 'timeout 15 ssh "${SSHOPT[@]}" alice@localhost id -un </dev/null | grep -qx alice'
+
+task 6 "The firewall is on and allows only SSH"
+need "firewalld or ufw is on" '[ -n "$fwd" ] || ufw_on'
+need "it's on at boot" 'if [ -n "$fwd" ]; then enabled firewalld; else grep -qx "ENABLED=yes" /etc/ufw/ufw.conf; fi'
+need "it allows only SSH in, now and permanently" only_ssh
+need "the web server on 8080 is still running" active le-lab-app.service
+need "the peer can't reach port 8080" '! peer_web'
+need "the peer can still reach SSH" peer_ssh
@@ -0,0 +1,3 @@
+TITLE="2.3 Networking"
+CHAPTER=/lfca/2-system-administration/2-3-networking/
+SYSTEMS="rhel debian"
@@ -0,0 +1,47 @@
+systemctl disable --now --quiet le-lab-app.service 2>/dev/null
+rm -f /etc/systemd/system/le-lab-app.service
+systemctl daemon-reload
+rm -rf /srv/le-app /usr/local/lib/le-lab/httpd.py
+rmdir /usr/local/lib/le-lab 2>/dev/null
+
+# Root's new keys.
+for k in /root/.ssh/id_*; do
+	[ -e "$k" ] || continue
+	grep -qxF "$k" "$LAB_STATE/root-keys" 2>/dev/null || rm -f "$k"
+done
+
+# The firewall as it was.
+if is_rhel; then
+	if grep -qx firewalld "$LAB_STATE/not-installed" 2>/dev/null; then
+		systemctl disable --now --quiet firewalld 2>/dev/null
+	else
+		if [ -f "$LAB_STATE/firewalld.tar" ]; then
+			rm -rf /etc/firewalld
+			tar -C /etc -xf "$LAB_STATE/firewalld.tar"
+			command -v restorecon >/dev/null && restorecon -R /etc/firewalld
+		fi
+		if [ -n "$(state_get firewalld-was-active)" ]; then
+			systemctl start firewalld && firewall-cmd -q --reload
+		else
+			systemctl stop firewalld 2>/dev/null
+		fi
+		if [ -n "$(state_get firewalld-was-enabled)" ]; then systemctl enable --quiet firewalld; else systemctl disable --quiet firewalld 2>/dev/null; fi
+	fi
+else
+	[ -n "$(state_get ufw-was-active)" ] || ufw --force disable >/dev/null
+	if [ -f "$LAB_STATE/ufw.tar" ]; then
+		rm -rf /etc/ufw
+		tar -C /etc -xf "$LAB_STATE/ufw.tar"
+	fi
+	[ -n "$(state_get ufw-was-active)" ] && ufw reload >/dev/null
+fi
+
+if [ -s "$LAB_STATE/not-installed" ]; then
+	if is_rhel; then
+		dnf -y -q remove $(cat "$LAB_STATE/not-installed") >/dev/null 2>&1
+		grep -qx firewalld "$LAB_STATE/not-installed" && rm -rf /etc/firewalld
+	else
+		DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge $(cat "$LAB_STATE/not-installed") >/dev/null 2>&1
+	fi
+fi
+true
@@ -0,0 +1,69 @@
+IF=$(ip -4 route show default | awk '{for (i = 1; i < NF; i++) if ($i == "dev") { print $(i + 1); exit }}')
+[ -n "$IF" ] || refuse "no default route; this lab needs a working network"
+state_set if "$IF"
+user_add alice
+ls -d /root/.ssh/id_* >"$LAB_STATE/root-keys" 2>/dev/null || true
+
+# Packages the tasks install: reset removes the ones that weren't here.
+for p in $(pick bind-utils bind9-dnsutils) $(pick firewalld ''); do
+	pkg_installed "$p" || echo "$p" >>"$LAB_STATE/not-installed"
+done
+
+# The firewall as it is, and a starting point with port 8080 open.
+if is_rhel; then
+	if pkg_installed firewalld; then
+		tar -C /etc -cf "$LAB_STATE/firewalld.tar" firewalld
+		enabled firewalld && state_set firewalld-was-enabled 1
+	fi
+	if active firewalld; then
+		state_set firewalld-was-active 1
+		z=$(firewall-cmd --get-zone-of-interface="$IF" 2>/dev/null || firewall-cmd --get-default-zone)
+		firewall-cmd -q --permanent --zone="$z" --add-port=8080/tcp
+		firewall-cmd -q --reload
+	fi
+else
+	tar -C /etc -cf "$LAB_STATE/ufw.tar" ufw default/ufw
+	if ufw status | grep -q 'Status: active'; then
+		state_set ufw-was-active 1
+		ufw allow 8080/tcp >/dev/null
+	fi
+fi
+
+# The web server on 8080, and the peer.
+peer_up
+mkdir -p /usr/local/lib/le-lab /srv/le-app
+echo 'Lab web server on port 8080' >/srv/le-app/index.html
+cat >/usr/local/lib/le-lab/httpd.py <<'X'
+# A static web server like "python3 -m http.server PORT --directory DIR",
+# without its reverse DNS lookup of the address, which can take seconds.
+import argparse, functools, http.server, socketserver
+
+ap = argparse.ArgumentParser()
+ap.add_argument("port", type=int)
+ap.add_argument("--bind", default="")
+ap.add_argument("--directory", default=".")
+a = ap.parse_args()
+
+
+class Server(http.server.ThreadingHTTPServer):
+    def server_bind(self):
+        socketserver.TCPServer.server_bind(self)
+        self.server_name, self.server_port = self.server_address[:2]
+
+
+handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=a.directory)
+Server((a.bind, a.port), handler).serve_forever()
+X
+cat >/etc/systemd/system/le-lab-app.service <<'X'
+[Unit]
+Description=Practice lab web server on port 8080
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/python3 /usr/local/lib/le-lab/httpd.py 8080 --directory /srv/le-app
+
+[Install]
+WantedBy=multi-user.target
+X
+systemctl daemon-reload
+systemctl enable --now --quiet le-lab-app.service
@@ -0,0 +1,43 @@
+# LFCA 2.4: Troubleshooting
+
+A full filesystem, a web server that won't start, DNS, load, and a boot
+that stops in emergency mode.
+
+## Before you start
+
+Work as root (`sudo -i`). Answers go in `/root/lab-answers/<task>`.
+
+The lab adds a 1 GiB spare disk with a filesystem mounted at `/data` and
+fills most of it with one large file, as if you had run `fallocate` there.
+The mount lasts until the next reboot. It installs nginx if it's missing
+and puts a typo in `/etc/nginx/nginx.conf`, so nginx fails to start. Reset
+puts nginx back as it was.
+
+## Tasks
+
+1. `/data` is nearly full. Find what fills it with `df` and `du`, as if you
+   didn't know, and write the full path of the large file to
+   `/root/lab-answers/1`.
+2. nginx won't start. Find the error with `systemctl status`,
+   `journalctl -u nginx` and `nginx -t`, fix it, and get nginx running
+   (and starting at boot) again.
+3. Put a wrong DNS server in the VM's network settings and diagnose the
+   symptoms with `ping` and `dig`. Put it back. Not graded.
+4. Install `stress-ng`, run `stress-ng --cpu 2 --timeout 60` (or
+   `yes > /dev/null`) and watch the load average in `top`. Not graded.
+5. Give root a password, add a bad line to `/etc/fstab`, reboot, and
+   recover from emergency mode. Do this only at the VM's console, and take
+   a snapshot first. Not graded: the VM booting normally again is your
+   check.
+
+## Hints
+
+- `df -h`, then `du -xh --max-depth=1 /data | sort -h`, one level deeper
+  each time; `du -a` also shows files, and `ls -la` shows hidden ones.
+- `nginx -t` names the file and line. `systemctl restart nginx`,
+  `systemctl enable nginx`, `curl -s localhost | head`.
+- `resolvectl dns` on Ubuntu; `nmcli connection modify ... ipv4.dns` on
+  the RHEL family.
+- `stress-ng` is in the RHEL family's AppStream and Ubuntu's universe.
+- In emergency mode, `mount -o remount,rw /`, fix `/etc/fstab`,
+  `systemctl daemon-reload`, `mount -a`, then `reboot`.
@@ -0,0 +1,21 @@
+page() {
+	local i
+	for i in 1 2 3 4 5; do
+		[ "$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 http://localhost/)" = 200 ] && return 0
+		sleep 2
+	done
+	return 1
+}
+
+task 1 "The large file in /data found"
+need "/root/lab-answers/1 has the large file's full path" '[ "$(head -n1 /root/lab-answers/1 | sed "s/^[[:space:]]*//; s/[[:space:]]*$//")" = "$(state_get big)" ]'
+
+task 2 "nginx's configuration fixed; nginx running"
+need "nginx -t finds no errors" 'nginx -t'
+need "nginx starts at boot" enabled nginx
+need "nginx is running" active nginx
+need "curl localhost gets a page" page
+
+task 3 "A wrong DNS server diagnosed and put back" self
+task 4 "Load watched in top" self
+task 5 "Recovered from emergency mode" self
@@ -0,0 +1,3 @@
+TITLE="2.4 Troubleshooting"
+CHAPTER=/lfca/2-system-administration/2-4-troubleshooting/
+SYSTEMS="rhel debian"
@@ -0,0 +1,21 @@
+findmnt /data >/dev/null && umount -l /data
+[ -n "$(state_get had-data)" ] || rmdir /data 2>/dev/null
+
+# nginx as it was.
+if [ -n "$(state_get had-nginx)" ]; then
+	[ -f "$LAB_STATE/nginx.conf" ] && cp -a "$LAB_STATE/nginx.conf" /etc/nginx/nginx.conf
+	if [ -n "$(state_get nginx-was-enabled)" ]; then systemctl enable --quiet nginx; else systemctl disable --quiet nginx 2>/dev/null; fi
+	if [ -n "$(state_get nginx-was-active)" ]; then systemctl restart nginx; else systemctl stop nginx 2>/dev/null; fi
+elif pkg_installed nginx; then
+	systemctl disable --now --quiet nginx 2>/dev/null
+	if is_rhel; then
+		dnf -y -q remove nginx >/dev/null 2>&1
+	else
+		DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge nginx nginx-common >/dev/null 2>&1
+	fi
+	rm -rf /etc/nginx
+fi
+if [ -n "$(state_get no-stress-ng)" ] && pkg_installed stress-ng; then
+	if is_rhel; then dnf -y -q remove stress-ng >/dev/null 2>&1; else DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge stress-ng >/dev/null 2>&1; fi
+fi
+true
@@ -0,0 +1,30 @@
+findmnt /data >/dev/null && refuse "something is mounted at /data; unmount it first"
+[ -d /data ] && state_set had-data 1
+if pkg_installed nginx; then
+	state_set had-nginx 1
+	enabled nginx && state_set nginx-was-enabled 1
+	active nginx && state_set nginx-was-active 1
+	grep -Eq '^[[:space:]]*worker_processes[[:space:]]+[^;]+;' /etc/nginx/nginx.conf ||
+		refuse "/etc/nginx/nginx.conf has no worker_processes line for the lab's typo; reinstall nginx's default configuration first"
+else
+	pkg_install nginx
+fi
+pkg_installed stress-ng || state_set no-stress-ng 1
+
+# Task 1: a nearly full /data.
+dev=$(disk_add 0 1G)
+mkfs.ext4 -q "$dev"
+mkdir -p /data
+mount "$dev" /data
+mkdir -p /data/reports/2026 /data/home/sam/.cache/old /data/www
+for m in 01 02 03 04 05 06 07 08 09 10; do seq 1 2000 >/data/reports/2026/sales-$m.csv; done
+echo '<h1>Intranet</h1>' >/data/www/index.html
+fallocate -l 820M /data/home/sam/.cache/old/render-tmp.bin
+state_set big /data/home/sam/.cache/old/render-tmp.bin
+
+# Task 2: a typo in nginx.conf.
+cp -a /etc/nginx/nginx.conf "$LAB_STATE/nginx.conf"
+sed -i -E '0,/^([[:space:]]*worker_processes[[:space:]]+[^;]+);/s//\1/' /etc/nginx/nginx.conf
+nginx -t 2>/dev/null && die "the typo didn't take; nginx -t still passes"
+systemctl enable --quiet nginx
+systemctl restart nginx 2>/dev/null || true
@@ -0,0 +1,37 @@
+# LFCA 2.5: Disaster recovery
+
+A backup with tar and a restore, and a mirror to another machine with
+rsync over SSH.
+
+## Before you start
+
+Work as root (`sudo -i`). The lab creates the directories `/srv/projects`
+and `/srv/site` with a few files, and user `alice` (password
+`Lab-Pass-1`).
+
+The peer, at `10.99.0.2` (name `peer`), stands in for a second VM. It runs
+its own SSH server. It shares this VM's files and users, so what you copy
+to `alice@peer:mirror/` lands in `/home/alice/mirror` here too, where you
+can compare it. Get a shell there with `sudo ip netns exec lepeer bash`.
+
+## Tasks
+
+1. Back up `/srv/projects` to `/root/projects.tar.gz` with `tar`, delete
+   `/srv/projects`, and restore it from the archive. Compare it with the
+   original.
+2. Install rsync if needed, and mirror `/srv/site` to the peer over SSH, as
+   `alice`, into her home directory's `mirror` directory. Then change
+   `/srv/site/index.html` and run it again, watching how little it copies.
+3. Take a VM snapshot, delete something important, and roll back. Not
+   graded.
+
+## Hints
+
+- `tar -czf /root/projects.tar.gz -C /srv projects`, `tar -tzf`,
+  `rm -r`, `tar -xzf ... -C /srv`. Keep a list such as
+  `find /srv/projects -type f | xargs sha256sum` from before to compare.
+- `rsync -av /srv/site/ alice@peer:mirror/` (mind the trailing slashes);
+  `--stats` or `-v` shows what it sent. `ssh-copy-id alice@peer` saves
+  typing the password each time.
+- Snapshots belong to the hypervisor (virt-manager, VirtualBox, your cloud
+  console), not to the VM.
@@ -0,0 +1,16 @@
+M=/home/alice/mirror
+
+task 1 "/srv/projects backed up to /root/projects.tar.gz and restored"
+need "/root/projects.tar.gz is a gzip-compressed tar archive" 'tar -tzf /root/projects.tar.gz'
+need "the archive has projects/src/app.py" 'tar -tzf /root/projects.tar.gz | grep -Eq "(^|/)projects/src/app\.py$"'
+need "/srv/projects has the same files and contents as before" '(cd /srv && find projects -type f -exec sha256sum {} + | sort -k2) | cmp -s - "$LAB_STATE/projects.sha"'
+# A new directory has a new inode number or, if the number was reused, a new birth time.
+need "/srv/projects was deleted and restored" '[ "$(stat -c "%i %.9W" /srv/projects)" != "$(state_get projects-id)" ]'
+
+task 2 "/srv/site mirrored to alice@peer:mirror with rsync, after a change"
+need "rsync is installed" 'command -v rsync'
+need "/srv/site/index.html was changed" '[ "$(sha256sum </srv/site/index.html)" != "$(state_get index-sha)" ]'
+need "/home/alice/mirror has the same files as /srv/site, including the change" '[ -d $M ] && diff -r /srv/site $M'
+need "the copies belong to alice (copied over SSH as alice)" '[ -z "$(find $M ! -user alice)" ]'
+
+task 3 "A snapshot rolled back" self
@@ -0,0 +1,3 @@
+TITLE="2.5 Disaster recovery"
+CHAPTER=/lfca/2-system-administration/2-5-disaster-recovery/
+SYSTEMS="rhel debian"
@@ -0,0 +1,8 @@
+systemctl disable --now --quiet le-lab-peer-sshd.service 2>/dev/null
+rm -f /etc/systemd/system/le-lab-peer-sshd.service
+systemctl daemon-reload
+rm -rf /srv/projects /srv/site /root/projects.tar.gz
+if [ -n "$(state_get no-rsync)" ] && pkg_installed rsync; then
+	if is_rhel; then dnf -y -q remove rsync >/dev/null 2>&1; else DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge rsync >/dev/null 2>&1; fi
+fi
+true
@@ -0,0 +1,41 @@
+for p in /srv/projects /srv/site /root/projects.tar.gz; do
+	[ ! -e "$p" ] || refuse "$p already exists; move it away first"
+done
+pkg_installed rsync || state_set no-rsync 1
+user_add alice
+
+# What task 1 backs up.
+mkdir -p /srv/projects/src /srv/projects/docs
+echo 'Project plan: move the team to the new file server.' >/srv/projects/plan.txt
+printf 'print("hello")\n' >/srv/projects/src/app.py
+seq 1 500 | sed 's/^/Note /' >/srv/projects/docs/notes.md
+(cd /srv && find projects -type f -exec sha256sum {} + | sort -k2) >"$LAB_STATE/projects.sha"
+state_set projects-id "$(stat -c "%i %.9W" /srv/projects)"
+
+# What task 2 mirrors.
+mkdir -p /srv/site/images
+echo '<h1>Our shop</h1>' >/srv/site/index.html
+echo '<h1>About us</h1>' >/srv/site/about.html
+head -c 2M /dev/urandom >/srv/site/images/banner.bin
+state_set index-sha "$(sha256sum </srv/site/index.html)"
+
+# The peer and its sshd.
+peer_up
+cat >/etc/systemd/system/le-lab-peer-sshd.service <<'X'
+[Unit]
+Description=Practice lab sshd on the peer
+Requires=le-lab-peer.service
+After=le-lab-peer.service network.target
+
+[Service]
+NetworkNamespacePath=/run/netns/lepeer
+ExecStartPre=/bin/mkdir -p /run/sshd
+ExecStart=/usr/sbin/sshd -D -o ListenAddress=10.99.0.2:22 -o PidFile=none
+KillMode=process
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+X
+systemctl daemon-reload
+systemctl enable --now --quiet le-lab-peer-sshd.service
@@ -0,0 +1,40 @@
+# LFCA 3.2: Performance and availability
+
+The downtime each number of nines allows, and nginx as a load balancer in
+front of two web servers.
+
+## Before you start
+
+Work as root (`sudo -i`). Answers go in `/root/lab-answers/<task>`.
+
+The peer, at `10.99.0.2` (name `peer`), stands in for the two web server
+VMs: it runs one web server on port 8081 and another on port 8082, each
+with its own home page (`le-lab-backend1.service` and
+`le-lab-backend2.service` on this VM). This VM is the load balancer. The
+lab notes whether nginx is installed, and reset puts that back. The VM
+needs internet access to install nginx.
+
+## Tasks
+
+1. Work out the yearly downtime allowed by 99.9%, 99.95% and 99.99%, in
+   minutes, and check them against the chapter's table. Write them to
+   `/root/lab-answers/1`, one per line, in that order.
+2. Install nginx on this VM and make it a load balancer on port 80: an
+   `upstream` block with `zone backend 64k;` and the two servers
+   (`10.99.0.2:8081` and `10.99.0.2:8082`), and `proxy_pass`. Run
+   `for i in 1 2 3 4 5 6; do curl -s http://localhost/; done` and watch
+   the answers alternate. Leave nginx running and starting at boot.
+3. Stop one web server (`systemctl stop le-lab-backend2`) and watch the
+   traffic move to the other. Start it again afterwards. Not graded.
+
+## Hints
+
+- A year has 365 × 24 × 60 = 525,600 minutes; 99.9% allows 0.1% of that.
+- A file in `/etc/nginx/conf.d/` with `upstream backend { ... }` and a
+  `server { listen 80; location / { proxy_pass http://backend; } }`.
+  `nginx -t`, `systemctl enable --now nginx`, `systemctl reload nginx`.
+- Only one server block can answer for port 80 by default. On Ubuntu,
+  remove the `default` link in `/etc/nginx/sites-enabled/`.
+- A 502 Bad Gateway on the RHEL family with SELinux enforcing: nginx may
+  not connect out until you allow it with
+  `setsebool -P httpd_can_network_connect on`.
@@ -0,0 +1,28 @@
+# minutes N: the first number on line N of answer 1.
+minutes() { sed -n "${1}p" /root/lab-answers/1 2>/dev/null | tr -d , | grep -Eo '[0-9]+(\.[0-9]+)?' | head -n1; }
+# between N LOW HIGH: line N's number is in the range.
+between() { local m; m=$(minutes "$1"); [ -n "$m" ] && awk -v m="$m" -v lo="$2" -v hi="$3" 'BEGIN { exit !(m >= lo && m <= hi) }'; }
+T=$(nginx -T 2>/dev/null)
+# Six requests to the load balancer reach both web servers.
+both() {
+	local i out=''
+	for i in 1 2 3 4 5 6; do out+=$(curl -s --max-time 3 http://localhost/)$'\n'; done
+	grep -q 'Web server one' <<<"$out" && grep -q 'Web server two' <<<"$out"
+}
+
+task 1 "Yearly downtime for 99.9%, 99.95% and 99.99% in /root/lab-answers/1"
+need "line 1, 99.9%, is about 525.6 minutes" 'between 1 520 530'
+need "line 2, 99.95%, is about 262.8 minutes" 'between 2 258 268'
+need "line 3, 99.99%, is about 52.6 minutes" 'between 3 52 53.5'
+
+task 2 "nginx on port 80 balances across both web servers"
+need "nginx is installed" 'command -v nginx'
+need "nginx starts at boot" enabled nginx
+need "nginx is running" active nginx
+need "an upstream block has a zone" 'grep -Eq "^[[:space:]]*zone[[:space:]]+[A-Za-z0-9_]+[[:space:]]+[0-9]+[kKmM]?[[:space:]]*;" <<<"$T"'
+need "the upstream has both servers, 10.99.0.2:8081 and :8082" 'grep -Eq "^[[:space:]]*server[[:space:]]+(10\.99\.0\.2|peer):8081" <<<"$T" && grep -Eq "^[[:space:]]*server[[:space:]]+(10\.99\.0\.2|peer):8082" <<<"$T"'
+need "a location uses proxy_pass" 'grep -Eq "^[[:space:]]*proxy_pass[[:space:]]+http://" <<<"$T"'
+need "both web servers are running" 'active le-lab-backend1.service && active le-lab-backend2.service'
+need "six requests to http://localhost/ get answers from both web servers" both
+
+task 3 "Traffic moved when one server stopped" self
@@ -0,0 +1,3 @@
+TITLE="3.2 Performance and availability"
+CHAPTER=/lfca/3-cloud-computing/3-2-performance-availability/
+SYSTEMS="rhel debian"
@@ -0,0 +1,34 @@
+for n in 1 2; do
+	systemctl disable --now --quiet le-lab-backend$n.service 2>/dev/null
+	rm -f /etc/systemd/system/le-lab-backend$n.service
+done
+systemctl daemon-reload
+rm -rf /srv/le-backend1 /srv/le-backend2 /usr/local/lib/le-lab/httpd.py
+rmdir /usr/local/lib/le-lab 2>/dev/null
+
+# nginx as it was.
+if [ -n "$(state_get had-nginx)" ]; then
+	if [ -f "$LAB_STATE/nginx.tar" ]; then
+		rm -rf /etc/nginx
+		tar -C /etc -xf "$LAB_STATE/nginx.tar"
+		command -v restorecon >/dev/null && restorecon -R /etc/nginx
+	fi
+	if [ -n "$(state_get nginx-was-enabled)" ]; then systemctl enable --quiet nginx; else systemctl disable --quiet nginx 2>/dev/null; fi
+	if [ -n "$(state_get nginx-was-active)" ]; then systemctl restart nginx; else systemctl stop nginx 2>/dev/null; fi
+elif pkg_installed nginx; then
+	systemctl disable --now --quiet nginx 2>/dev/null
+	if is_rhel; then
+		dnf -y -q remove nginx >/dev/null 2>&1
+	else
+		DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge nginx nginx-common >/dev/null 2>&1
+	fi
+	rm -rf /etc/nginx
+fi
+
+# SELinux booleans as they were.
+if [ -s "$LAB_STATE/sebools" ] && command -v setsebool >/dev/null; then
+	while read -r b _ v; do
+		[ "$(getsebool "$b" 2>/dev/null | awk '{print $3}')" = "$v" ] || setsebool -P "$b" "$v"
+	done <"$LAB_STATE/sebools"
+fi
+true
@@ -0,0 +1,54 @@
+if pkg_installed nginx; then
+	state_set had-nginx 1
+	enabled nginx && state_set nginx-was-enabled 1
+	active nginx && state_set nginx-was-active 1
+	tar -C /etc -cf "$LAB_STATE/nginx.tar" nginx
+fi
+if command -v getsebool >/dev/null; then
+	getsebool httpd_can_network_connect httpd_can_network_relay 2>/dev/null >"$LAB_STATE/sebools" || true
+fi
+
+# Two web servers in the peer.
+peer_up
+mkdir -p /usr/local/lib/le-lab /srv/le-backend1 /srv/le-backend2
+echo 'Web server one' >/srv/le-backend1/index.html
+echo 'Web server two' >/srv/le-backend2/index.html
+cat >/usr/local/lib/le-lab/httpd.py <<'X'
+# A static web server like "python3 -m http.server PORT --bind ADDRESS
+# --directory DIR", without its reverse DNS lookup of the address, which
+# can take seconds.
+import argparse, functools, http.server, socketserver
+
+ap = argparse.ArgumentParser()
+ap.add_argument("port", type=int)
+ap.add_argument("--bind", default="")
+ap.add_argument("--directory", default=".")
+a = ap.parse_args()
+
+
+class Server(http.server.ThreadingHTTPServer):
+    def server_bind(self):
+        socketserver.TCPServer.server_bind(self)
+        self.server_name, self.server_port = self.server_address[:2]
+
+
+handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=a.directory)
+Server((a.bind, a.port), handler).serve_forever()
+X
+for n in 1 2; do
+	cat >/etc/systemd/system/le-lab-backend$n.service <<X
+[Unit]
+Description=Practice lab web server $n on the peer
+Requires=le-lab-peer.service
+After=le-lab-peer.service
+
+[Service]
+NetworkNamespacePath=/run/netns/lepeer
+ExecStart=/usr/bin/python3 /usr/local/lib/le-lab/httpd.py 808$n --bind 10.99.0.2 --directory /srv/le-backend$n
+
+[Install]
+WantedBy=multi-user.target
+X
+done
+systemctl daemon-reload
+systemctl enable --now --quiet le-lab-backend1.service le-lab-backend2.service
@@ -0,0 +1,53 @@
+# LFCA 4.1: Security
+
+SSH keys without passwords, a host firewall that allows only SSH, hashes,
+encrypting a file with a passphrase, and looking up a CVE.
+
+## Before you start
+
+Work as root (`sudo -i`), so `~` is `/root`. Answers go in
+`/root/lab-answers/<task>`. Keep a console session open while you change
+SSH or firewall settings.
+
+The lab creates user `alice` (password `Lab-Pass-1`), the files
+`/root/hash-me.txt` and `/root/secret.txt`, and a web server on port 8080.
+The peer, at `10.99.0.2` (name `peer`), stands in for another machine; this
+VM is `10.99.0.1` there. Get a shell there with
+`sudo ip netns exec lepeer bash`. The lab saves the SSH and firewall
+settings, and reset puts them back.
+
+## Tasks
+
+1. Set up SSH key login from root to `alice` on this VM (`ssh
+   alice@localhost` logs in without a password). Then set
+   `PasswordAuthentication no` in a file such as
+   `/etc/ssh/sshd_config.d/10-no-passwords.conf`, check it with
+   `sshd -T | grep -i passwordauth`, reload the SSH service, and check that
+   passwords no longer work.
+2. Turn on the host firewall (firewalld on the RHEL family, ufw on
+   Ubuntu) and allow only SSH. Confirm that the web server on port 8080 can
+   no longer be reached from the peer.
+3. Take the `sha256sum` of `/root/hash-me.txt`, change one character in
+   it, and take it again. Write the two hashes to `/root/lab-answers/3`,
+   one per line, before and after.
+4. Encrypt `/root/secret.txt` with a passphrase (`gpg -c`), delete the
+   original, then decrypt `/root/secret.txt.gpg` back to
+   `/root/secret.txt`.
+5. Look up CVE-2024-3094 in the NVD (nvd.nist.gov) and write its CVSS base
+   score to `/root/lab-answers/5`.
+
+## Hints
+
+- `ssh-keygen`, `ssh-copy-id alice@localhost`. The drop-in's name must end
+  in `.conf` and sort before any `50-cloud-init.conf`: sshd uses the first
+  value it reads. `systemctl reload sshd` (`ssh` on Ubuntu). Test with
+  `ssh -o PubkeyAuthentication=no alice@localhost`.
+- `dnf install firewalld`, `systemctl enable --now firewalld`,
+  `firewall-cmd --list-all`, `--remove-service`, `--remove-port`,
+  `--permanent`, `--reload`. `ufw allow OpenSSH`, `ufw enable`. From the
+  peer: `curl --max-time 5 http://10.99.0.1:8080/`.
+- `sha256sum FILE`; any editor, or `sed -i 's/quick/quack/'`.
+- `gpg -c /root/secret.txt`, `rm`, `gpg -o /root/secret.txt -d
+  /root/secret.txt.gpg`. If it doesn't ask for the passphrase, that's
+  gpg's cache: run `gpgconf --reload gpg-agent` first.
+- The score is on the CVE's NVD page, next to its severity.
@@ -0,0 +1,63 @@
+IF=$(state_get if)
+is_debian && mkdir -p /run/sshd
+SSHOPT=(-o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR)
+key_login() { timeout 15 ssh "${SSHOPT[@]}" -o BatchMode=yes -o PreferredAuthentications=publickey alice@localhost id -un </dev/null | grep -qx alice; }
+# A login as alice with her right password, typed by an askpass helper.
+password_login() {
+	local ap=/run/le-lab-askpass r
+	printf '#!/bin/sh\necho Lab-Pass-1\n' >$ap && chmod 700 $ap
+	SSH_ASKPASS=$ap SSH_ASKPASS_REQUIRE=force DISPLAY=none timeout 15 setsid -w ssh "${SSHOPT[@]}" \
+		-o PubkeyAuthentication=no -o PreferredAuthentications=password,keyboard-interactive \
+		-o NumberOfPasswordPrompts=1 alice@localhost id -un </dev/null 2>/dev/null | grep -qx alice
+	r=$?; rm -f $ap; return $r
+}
+
+# The firewall: firewalld if it's running, otherwise ufw.
+fwd=''; active firewalld && fwd=1
+ufw_on() { command -v ufw >/dev/null && ufw status | grep -q 'Status: active'; }
+zones() { { firewall-cmd --get-zone-of-interface="$IF" 2>/dev/null || firewall-cmd --get-default-zone; firewall-cmd --get-default-zone; } | sort -u; }
+only_ssh() {
+	local z p
+	if [ -n "$fwd" ]; then
+		for z in $(zones); do
+			for p in "" --permanent; do
+				firewall-cmd $p --zone="$z" --list-services | tr ' ' '\n' | grep -Evx 'ssh|dhcpv6-client|' && return 1
+				[ -z "$(firewall-cmd $p --zone="$z" --list-ports)" ] || return 1
+				[ "$(firewall-cmd $p --zone="$z" --get-target)" != ACCEPT ] || return 1
+			done
+		done
+		return 0
+	fi
+	! ufw status | grep ALLOW | grep -v FWD | grep -Evq '^(22(/tcp)?|OpenSSH)[[:space:]]' &&
+		ufw status verbose | grep -Eq 'Default: (deny|reject) \(incoming\)'
+}
+peer_ssh() { in_peer timeout 5 bash -c 'exec 3<>/dev/tcp/10.99.0.1/22; head -c 4 <&3' 2>/dev/null | grep -q SSH; }
+peer_web() { in_peer curl -s --max-time 5 http://10.99.0.1:8080/ | grep -q "Lab web server"; }
+hash_line() { sed -n "${1}p" /root/lab-answers/3 2>/dev/null | awk '{print tolower($1)}'; }
+
+task 1 "Key login to alice; password logins off"
+need "ssh alice@localhost logs root in with a key" key_login
+need "sshd -T shows passwordauthentication no" 'sshd -T 2>/dev/null | grep -qx "passwordauthentication no"'
+need "a file in /etc/ssh/sshd_config.d sets PasswordAuthentication no" 'grep -rhiEv "^[[:space:]]*#" /etc/ssh/sshd_config.d 2>/dev/null | grep -iEq "^[[:space:]]*PasswordAuthentication[[:space:]]+no"'
+need "the running sshd refuses alice's password" '! password_login'
+
+task 2 "The host firewall is on and allows only SSH"
+need "firewalld or ufw is on" '[ -n "$fwd" ] || ufw_on'
+need "it's on at boot" 'if [ -n "$fwd" ]; then enabled firewalld; else grep -qx "ENABLED=yes" /etc/ufw/ufw.conf; fi'
+need "it allows only SSH in, now and permanently" only_ssh
+need "the web server on 8080 is still running" active le-lab-app.service
+need "the peer can't reach port 8080" '! peer_web'
+need "the peer can still reach SSH" peer_ssh
+
+task 3 "Hashes of /root/hash-me.txt before and after a one-character change"
+need "line 1 of /root/lab-answers/3 is the hash from before" '[ "$(hash_line 1)" = "$(sha256sum <"$LAB_STATE/hash-me.txt" | cut -d" " -f1)" ]'
+need "exactly one character of /root/hash-me.txt changed" '[ "$(cmp -l "$LAB_STATE/hash-me.txt" /root/hash-me.txt 2>/dev/null | wc -l)" = 1 ] && [ "$(stat -c %s /root/hash-me.txt)" = "$(stat -c %s "$LAB_STATE/hash-me.txt")" ]'
+need "line 2 is the hash after the change" '[ "$(hash_line 2)" = "$(sha256sum </root/hash-me.txt | cut -d" " -f1)" ]'
+
+task 4 "/root/secret.txt encrypted with a passphrase, deleted and decrypted again"
+need "/root/secret.txt.gpg is encrypted with a passphrase" 'gpg --batch --list-packets /root/secret.txt.gpg </dev/null 2>/dev/null | grep -q "symkey enc packet"'
+need "/root/secret.txt is back, with its original contents" '[ "$(sha256sum </root/secret.txt)" = "$(state_get secret-sha)" ]'
+need "it was deleted and decrypted again" '[ "$(stat -c "%i %.9W" /root/secret.txt)" != "$(state_get secret-id)" ]'
+
+task 5 "CVE-2024-3094's CVSS score in /root/lab-answers/5"
+need "the answer is the score the NVD shows" '[[ "$(answer 5)" =~ ^10(\.0)?$ ]]'
@@ -0,0 +1,3 @@
+TITLE="4.1 Security"
+CHAPTER=/lfca/4-security/4-1-security/
+SYSTEMS="rhel debian"
@@ -0,0 +1,56 @@
+systemctl disable --now --quiet le-lab-app.service 2>/dev/null
+rm -f /etc/systemd/system/le-lab-app.service
+systemctl daemon-reload
+rm -rf /srv/le-app /usr/local/lib/le-lab/httpd.py
+rmdir /usr/local/lib/le-lab 2>/dev/null
+rm -f /root/hash-me.txt /root/secret.txt /root/secret.txt.gpg
+
+# sshd's configuration.
+if [ -f "$LAB_STATE/ssh.tar" ]; then
+	rm -f /etc/ssh/sshd_config.d/*
+	tar -C /etc -xf "$LAB_STATE/ssh.tar"
+	if is_rhel; then
+		systemctl restart sshd
+	else
+		systemctl restart ssh.socket 2>/dev/null
+		systemctl restart ssh.service 2>/dev/null
+	fi
+fi
+# Root's new keys and gpg's files.
+for k in /root/.ssh/id_*; do
+	[ -e "$k" ] || continue
+	grep -qxF "$k" "$LAB_STATE/root-keys" 2>/dev/null || rm -f "$k"
+done
+if [ -z "$(state_get had-gnupg)" ]; then
+	gpgconf --kill gpg-agent 2>/dev/null
+	rm -rf /root/.gnupg
+fi
+
+# The firewall as it was.
+if is_rhel; then
+	if grep -qx firewalld "$LAB_STATE/not-installed" 2>/dev/null; then
+		systemctl disable --now --quiet firewalld 2>/dev/null
+		dnf -y -q remove firewalld >/dev/null 2>&1
+		rm -rf /etc/firewalld
+	else
+		if [ -f "$LAB_STATE/firewalld.tar" ]; then
+			rm -rf /etc/firewalld
+			tar -C /etc -xf "$LAB_STATE/firewalld.tar"
+			command -v restorecon >/dev/null && restorecon -R /etc/firewalld
+		fi
+		if [ -n "$(state_get firewalld-was-active)" ]; then
+			systemctl start firewalld && firewall-cmd -q --reload
+		else
+			systemctl stop firewalld 2>/dev/null
+		fi
+		if [ -n "$(state_get firewalld-was-enabled)" ]; then systemctl enable --quiet firewalld; else systemctl disable --quiet firewalld 2>/dev/null; fi
+	fi
+else
+	[ -n "$(state_get ufw-was-active)" ] || ufw --force disable >/dev/null
+	if [ -f "$LAB_STATE/ufw.tar" ]; then
+		rm -rf /etc/ufw
+		tar -C /etc -xf "$LAB_STATE/ufw.tar"
+	fi
+	[ -n "$(state_get ufw-was-active)" ] && ufw reload >/dev/null
+fi
+true
@@ -0,0 +1,75 @@
+IF=$(ip -4 route show default | awk '{for (i = 1; i < NF; i++) if ($i == "dev") { print $(i + 1); exit }}')
+[ -n "$IF" ] || refuse "no default route; this lab needs a working network"
+state_set if "$IF"
+user_add alice
+ls -d /root/.ssh/id_* >"$LAB_STATE/root-keys" 2>/dev/null || true
+[ -e /root/.gnupg ] && state_set had-gnupg 1
+tar -C /etc -cf "$LAB_STATE/ssh.tar" ssh
+pkg_installed firewalld || { is_rhel && echo firewalld >>"$LAB_STATE/not-installed"; }
+
+# Files for tasks 3 and 4.
+echo 'The quick brown fox jumps over the lazy dog.' >/root/hash-me.txt
+cp /root/hash-me.txt "$LAB_STATE/hash-me.txt"
+printf 'Wi-Fi password: correct-horse-battery-staple\nAlarm code: 4711\n' >/root/secret.txt
+chmod 600 /root/secret.txt
+state_set secret-sha "$(sha256sum </root/secret.txt)"
+state_set secret-id "$(stat -c "%i %.9W" /root/secret.txt)"
+
+# The firewall as it is, and a starting point with port 8080 open.
+if is_rhel; then
+	if pkg_installed firewalld; then
+		tar -C /etc -cf "$LAB_STATE/firewalld.tar" firewalld
+		enabled firewalld && state_set firewalld-was-enabled 1
+	fi
+	if active firewalld; then
+		state_set firewalld-was-active 1
+		z=$(firewall-cmd --get-zone-of-interface="$IF" 2>/dev/null || firewall-cmd --get-default-zone)
+		firewall-cmd -q --permanent --zone="$z" --add-port=8080/tcp
+		firewall-cmd -q --reload
+	fi
+else
+	tar -C /etc -cf "$LAB_STATE/ufw.tar" ufw default/ufw
+	if ufw status | grep -q 'Status: active'; then
+		state_set ufw-was-active 1
+		ufw allow 8080/tcp >/dev/null
+	fi
+fi
+
+# The web server on 8080, and the peer.
+peer_up
+mkdir -p /usr/local/lib/le-lab /srv/le-app
+echo 'Lab web server on port 8080' >/srv/le-app/index.html
+cat >/usr/local/lib/le-lab/httpd.py <<'X'
+# A static web server like "python3 -m http.server PORT --directory DIR",
+# without its reverse DNS lookup of the address, which can take seconds.
+import argparse, functools, http.server, socketserver
+
+ap = argparse.ArgumentParser()
+ap.add_argument("port", type=int)
+ap.add_argument("--bind", default="")
+ap.add_argument("--directory", default=".")
+a = ap.parse_args()
+
+
+class Server(http.server.ThreadingHTTPServer):
+    def server_bind(self):
+        socketserver.TCPServer.server_bind(self)
+        self.server_name, self.server_port = self.server_address[:2]
+
+
+handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=a.directory)
+Server((a.bind, a.port), handler).serve_forever()
+X
+cat >/etc/systemd/system/le-lab-app.service <<'X'
+[Unit]
+Description=Practice lab web server on port 8080
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/python3 /usr/local/lib/le-lab/httpd.py 8080 --directory /srv/le-app
+
+[Install]
+WantedBy=multi-user.target
+X
+systemctl daemon-reload
+systemctl enable --now --quiet le-lab-app.service
@@ -0,0 +1,34 @@
+# LFCA 4.2: Sensitive data
+
+A secret committed to Git and taken out of its history again, and a file
+encrypted before its original is shredded.
+
+## Before you start
+
+Work as root (`sudo -i`). The lab installs Git if it's missing and creates
+a test repository, `/root/leaky`, in which someone committed a file
+`config.env` with a fake API key, and later commits on top. It also
+creates `/root/customers.csv`. It notes which packages were installed, and
+reset puts that back. The VM needs internet access to install the tools.
+
+## Tasks
+
+1. Find the key in `/root/leaky` with `gitleaks detect` (from Ubuntu's
+   packages; on the RHEL family, the upstream release) or with
+   `git log -p | grep`. Then remove `config.env` from the whole history
+   with `git filter-repo --force --path config.env --invert-paths`, keeping
+   the other files and commits. Note why the key would still need to be
+   rotated.
+2. Encrypt `/root/customers.csv` with `gpg -c`, and delete the original
+   with `shred -u`. Then read `man shred` and note why it isn't reliable on
+   SSDs or on copy-on-write filesystems such as Btrfs.
+
+## Hints
+
+- `cd /root/leaky`, `git log --oneline --stat`, `git log -p | grep -i key`.
+- `apt install gitleaks git-filter-repo` on Ubuntu. On the RHEL family,
+  `git-filter-repo` is in EPEL (`dnf install epel-release`, then
+  `dnf install git-filter-repo`).
+- After filter-repo, `git log --all --oneline -- config.env` shows nothing.
+- `gpg -c customers.csv` asks for the passphrase twice; `ls` after
+  `shred -u`.
@@ -0,0 +1,12 @@
+KEY=$(state_get key)
+g() { git -C /root/leaky "$@"; }
+
+task 1 "config.env and its key gone from /root/leaky's history"
+need "/root/leaky still has its other commits" 'g log --oneline | grep -q "Add app" && g log --oneline | grep -q "Document the configuration"'
+need "no commit has config.env any more" '[ -z "$(g log --all --format=%H -- config.env)" ]'
+need "the key is in no commit on any branch, tag or backup ref" '! g log --all -p | grep -qF "$KEY"'
+need "app.py and README.md are still there" '[ -f /root/leaky/app.py ] && [ -f /root/leaky/README.md ]'
+
+task 2 "/root/customers.csv encrypted, the original shredded"
+need "/root/customers.csv.gpg is encrypted with a passphrase" 'gpg --batch --list-packets /root/customers.csv.gpg </dev/null 2>/dev/null | grep -q "symkey enc packet"'
+need "/root/customers.csv is gone" '[ ! -e /root/customers.csv ]'
@@ -0,0 +1,3 @@
+TITLE="4.2 Sensitive data"
+CHAPTER=/lfca/4-security/4-2-sensitive-data/
+SYSTEMS="rhel debian"
@@ -0,0 +1,15 @@
+rm -rf /root/leaky /root/customers.csv /root/customers.csv.gpg
+if [ -z "$(state_get had-gnupg)" ]; then
+	gpgconf --kill gpg-agent 2>/dev/null
+	rm -rf /root/.gnupg
+fi
+if [ -s "$LAB_STATE/not-installed" ]; then
+	if is_rhel; then
+		dnf -y -q remove $(cat "$LAB_STATE/not-installed") >/dev/null 2>&1
+	else
+		DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge $(cat "$LAB_STATE/not-installed") >/dev/null 2>&1
+	fi
+fi
+# A gitleaks binary from upstream, if one was put here during the lab.
+[ -n "$(state_get had-gitleaks-bin)" ] || { pkg_installed gitleaks || rm -f /usr/local/bin/gitleaks; }
+true
@@ -0,0 +1,32 @@
+for p in /root/leaky /root/customers.csv /root/customers.csv.gpg; do
+	[ ! -e "$p" ] || refuse "$p already exists; move it away first"
+done
+[ -e /root/.gnupg ] && state_set had-gnupg 1
+[ -e /usr/local/bin/gitleaks ] && state_set had-gitleaks-bin 1
+# Packages the tasks install: reset removes the ones that weren't here.
+for p in git git-filter-repo $(pick epel-release gitleaks); do
+	pkg_installed "$p" || echo "$p" >>"$LAB_STATE/not-installed"
+done
+pkg_installed git || pkg_install git
+
+# The repository with a leaked key, and commits after it.
+KEY=ghp_$(head -c 200 /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 36)
+state_set key "$KEY"
+mkdir -p /root/leaky
+cd /root/leaky
+g() { git -c user.name="Sam Developer" -c user.email=[email protected] -c init.defaultBranch=main "$@" >/dev/null; }
+g init -q
+echo '# Shop backend' >README.md
+g add README.md && g commit -q -m "Add readme"
+printf 'import os\nprint("shop running")\n' >app.py
+g add app.py && g commit -q -m "Add app"
+printf 'DB_HOST=db.internal\nGITHUB_TOKEN=%s\n' "$KEY" >config.env
+g add config.env && g commit -q -m "Add configuration"
+printf 'import os\ntoken = os.environ["GITHUB_TOKEN"]\nprint("shop running")\n' >app.py
+g commit -q -am "Read the token from the environment"
+echo 'Run app.py with the variables from config.env.' >>README.md
+g commit -q -am "Document the configuration"
+cd /
+
+printf 'name,email\nAda Lovelace,[email protected]\nAlan Turing,[email protected]\n' >/root/customers.csv
+chmod 600 /root/customers.csv
@@ -0,0 +1,39 @@
+# LFCA 5.2: Git concepts
+
+A first repository, a merge conflict, a bare repository as a remote, and
+undoing a commit with `git revert`.
+
+## Before you start
+
+Work as root (`sudo -i`), so `~` is `/root`. The lab only checks that
+`~/project`, `~/app.git` and `~/app-copy` don't exist yet, and notes
+whether you already have a `~/.gitconfig` and Git (reset removes the ones
+the lab didn't find).
+
+## Tasks
+
+1. Install Git, set your name and email (and `init.defaultBranch main`, as
+   the chapter shows), create the repository `~/project`, make three
+   commits on `main`, and read the log with `git log --oneline`.
+2. Make a branch, change the same line on it and on `main`, merge the
+   branch into `main`, and resolve the conflict.
+3. Create a bare repository with `git init --bare -b main ~/app.git` to
+   act as a remote, and push `~/project`'s `main` to it. Clone it into
+   `~/app-copy`, commit a change there and push it, then `pull` the change
+   back into `~/project`.
+4. In `~/project`, commit a bad change, then undo it with `git revert`, and
+   read the log.
+
+## Hints
+
+- `git config --global user.name`, `user.email`, `init.defaultBranch`;
+  `git init ~/project`, `git add`, `git commit -m`.
+- `git switch -c NAME`, `git switch main`, `git merge NAME`; edit the file
+  to remove the `<<<<<<<`, `=======` and `>>>>>>>` lines, then `git add`
+  and `git commit`.
+- `-b main` names the bare repository's default branch. Without it or
+  `init.defaultBranch`, it's `master`, and the clone warns "remote HEAD
+  refers to nonexistent ref" and comes out empty.
+- `git remote add origin ~/app.git`, `git push -u origin main`,
+  `git clone ~/app.git ~/app-copy`, `git push`, `git pull`.
+- `git revert HEAD`, `git log --oneline`.
@@ -0,0 +1,36 @@
+g() { git -C /root/project "$@"; }
+gc() { HOME=/root git config --global --get "$1"; }
+# The newest commit made in ~/app-copy itself (not cloned or pulled).
+copy_commit() { git -C /root/app-copy log -g --format='%H %gs' 2>/dev/null | awk '$2 == "commit:" || $2 == "commit" {print $1; exit}'; }
+remote_is_app() { local u; for u in $(g remote 2>/dev/null); do g remote get-url "$u" | grep -Eq '(^|/)app\.git/?$' && return 0; done; return 1; }
+reverted() {
+	local c t
+	for c in $(g log --format=%H --grep='^Revert "' main); do
+		t=$(g log -1 --format=%B "$c" | grep -Eo 'reverts commit [0-9a-f]{7,40}' | awk '{print $3}')
+		[ -n "$t" ] && g merge-base --is-ancestor "$t" "$c" && return 0
+	done
+	return 1
+}
+
+task 1 "Git configured; ~/project with three commits on main"
+need "git is installed" 'command -v git'
+need "your name and email are set globally" '[ -n "$(gc user.name)" ] && [ -n "$(gc user.email)" ]'
+need "~/project is a repository with a branch main" 'g rev-parse -q --verify refs/heads/main'
+need "main has at least three commits" '[ "$(g rev-list --count main)" -ge 3 ]'
+
+task 2 "A branch merged into main, conflict resolved"
+need "main has a merge commit" '[ -n "$(g rev-list --merges main)" ]'
+need "no merge is left half done" '[ ! -e /root/project/.git/MERGE_HEAD ]'
+need "no conflict markers are committed" '! g grep -qE "^(<<<<<<<|>>>>>>>)( |$)" main'
+
+task 3 "~/app.git as a remote, a change pushed from ~/app-copy and pulled back"
+need "~/app.git is a bare repository" '[ "$(git --git-dir=/root/app.git rev-parse --is-bare-repository 2>/dev/null)" = true ]'
+need "~/app.git's default branch (its HEAD) is main" '[ "$(git --git-dir=/root/app.git symbolic-ref HEAD 2>/dev/null)" = refs/heads/main ]'
+need "~/project has ~/app.git as a remote" remote_is_app
+need "~/app-copy is a clone of ~/app.git" 'git -C /root/app-copy remote get-url origin | grep -Eq "(^|/)app\.git/?$"'
+need "a commit was made in ~/app-copy" '[ -n "$(copy_commit)" ]'
+need "it was pushed to ~/app.git" 'git --git-dir=/root/app.git merge-base --is-ancestor "$(copy_commit)" main'
+need "it was pulled into ~/project's main" 'g merge-base --is-ancestor "$(copy_commit)" main'
+
+task 4 "A bad commit undone with git revert"
+need "main has a Revert commit for an earlier commit" reverted
@@ -0,0 +1,3 @@
+TITLE="5.2 Git concepts"
+CHAPTER=/lfca/5-devops/5-2-git-concepts/
+SYSTEMS="rhel debian"
@@ -0,0 +1,6 @@
+rm -rf /root/project /root/app.git /root/app-copy
+[ -n "$(state_get had-gitconfig)" ] || rm -f /root/.gitconfig
+if [ -n "$(state_get no-git)" ] && pkg_installed git; then
+	if is_rhel; then dnf -y -q remove git >/dev/null 2>&1; else DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge git >/dev/null 2>&1; fi
+fi
+true
@@ -0,0 +1,6 @@
+for p in /root/project /root/app.git /root/app-copy; do
+	[ ! -e "$p" ] || refuse "$p already exists; move it away first"
+done
+[ -e /root/.gitconfig ] && state_set had-gitconfig 1
+pkg_installed git || state_set no-git 1
+true
@@ -0,0 +1,46 @@
+# LFCA 5.3: Containers
+
+Running nginx in a container, building an image from a Dockerfile, keeping
+a database in a named volume, and a container's own process list.
+
+## Before you start
+
+Work as root (`sudo -i`), so `~` is `/root`, and use Podman as root. Answers
+go in `/root/lab-answers/<task>`. The VM needs internet access to install
+Podman and pull images. Docker Hub limits anonymous pulls; if you hit the
+limit, `mirror.gcr.io/library/NAME` is the same image as
+`docker.io/library/NAME`. The lab notes whether Podman is installed; reset
+removes what the tasks create, and Podman too if the lab didn't find it.
+Containers stop at a reboot, so run `le-lab check` before you reboot.
+
+## Tasks
+
+1. Install Podman, run nginx in a container with `-p 8080:80`, and reach
+   it with `curl` from the VM. Look at its logs and open a shell in it with
+   `exec`. Leave it running.
+2. Write a Dockerfile in `~/mysite` for a static page served by nginx,
+   build it, tag it `mysite:1.0`, and run it with `-p 8081:80`. Leave it
+   running.
+3. Run PostgreSQL with a named volume:
+   `podman run -d --name db -e POSTGRES_PASSWORD=secret -v dbdata:/var/lib/postgresql docker.io/library/postgres:18`.
+   Create a table with `podman exec -it db psql -U postgres`, remove the
+   container, start a new one named `db` with the same volume, and check
+   the table is still there. Leave the new one running.
+4. Run `podman run --rm alpine ps` and see the container's process list
+   start at PID 1. Write the command running as PID 1 to
+   `/root/lab-answers/4`.
+
+## Hints
+
+- `dnf install podman` / `apt install podman`. Full image names such as
+  `docker.io/library/nginx` work everywhere; short names like `nginx`
+  depend on the registries configured.
+- `podman run -d --name web -p 8080:80 docker.io/library/nginx`,
+  `curl localhost:8080`, `podman logs web`, `podman exec -it web sh`.
+- `FROM docker.io/library/nginx` and `COPY index.html
+  /usr/share/nginx/html/`; `podman build -t mysite:1.0 ~/mysite`,
+  `podman images`.
+- In psql: `CREATE TABLE notes (id int, text text);`, `\dt`, `\q`.
+  `podman rm -f db`, `podman volume ls`. PostgreSQL takes a few seconds to
+  start.
+- `podman run --rm docker.io/library/alpine ps`.
@@ -0,0 +1,40 @@
+# The running container publishing container port 80 on host port PORT.
+on_port() {
+	local c
+	for c in $(podman ps -q 2>/dev/null); do
+		podman port "$c" 2>/dev/null | grep -Eq "^80/tcp -> .*:$1\$" && { echo "$c"; return; }
+	done
+}
+page() {
+	local i
+	for i in 1 2 3; do
+		curl -s --max-time 3 "http://localhost:$1/" | grep -qi "$2" && return 0
+		sleep 2
+	done
+	return 1
+}
+psql_tables() { podman exec db psql -U postgres -tAc "select count(*) from information_schema.tables where table_schema = 'public'" 2>/dev/null | tr -d ' '; }
+created() { podman "$1" inspect -f "{{.$2.Unix}}" "$3" 2>/dev/null; }
+
+task 1 "nginx in a container on host port 8080"
+need "podman is installed" 'command -v podman'
+need "a running container publishes port 80 on host port 8080" '[ -n "$(on_port 8080)" ]'
+need "it runs an nginx image" 'podman inspect -f "{{.ImageName}}" "$(on_port 8080)" | grep -q nginx'
+need "curl localhost:8080 gets nginx's page" 'page 8080 nginx'
+
+task 2 "Image mysite:1.0 built from ~/mysite/Dockerfile and running on 8081"
+need "/root/mysite/Dockerfile exists" '[ -f /root/mysite/Dockerfile ]'
+need "podman images lists localhost/mysite:1.0" 'podman image exists localhost/mysite:1.0'
+need "a container from mysite:1.0 publishes port 80 on host port 8081" '[ "$(podman inspect -f "{{.Image}}" "$(on_port 8081)" 2>/dev/null)" = "$(podman image inspect -f "{{.Id}}" localhost/mysite:1.0)" ]'
+need "curl localhost:8081 gets a page" 'page 8081 "<"'
+
+task 3 "PostgreSQL's table kept in volume dbdata across a new container"
+need "volume dbdata exists" 'podman volume exists dbdata'
+need "a container named db is running" '[ "$(podman inspect -f "{{.State.Running}}" db 2>/dev/null)" = true ]'
+need "db keeps /var/lib/postgresql in volume dbdata" 'podman inspect -f "{{range .Mounts}}{{println .Name .Destination}}{{end}}" db | grep -qx "dbdata /var/lib/postgresql"'
+need "the database has a table" '[ "$(psql_tables)" -ge 1 ]'
+need "db is a newer container than the volume" '[ $(( $(created container Created db) - $(created volume CreatedAt dbdata) )) -ge 5 ]'
+
+task 4 "PID 1 in an alpine container in /root/lab-answers/4"
+need "an alpine image was pulled" 'podman images --format "{{.Repository}}" | grep -q "/alpine$"'
+need "the answer is the command running as PID 1" '[ "$(answer 4)" = ps ]'
@@ -0,0 +1,3 @@
+TITLE="5.3 Containers"
+CHAPTER=/lfca/5-devops/5-3-containers/
+SYSTEMS="rhel debian"
@@ -0,0 +1,24 @@
+if command -v podman >/dev/null; then
+	if [ -z "$(state_get had-podman)" ]; then
+		podman system reset -f >/dev/null 2>&1
+	else
+		for p in 8080 8081; do
+			for c in $(podman ps -q); do
+				podman port "$c" 2>/dev/null | grep -Eq "^80/tcp -> .*:$p\$" && podman rm -f -t 0 "$c" >/dev/null 2>&1
+			done
+		done
+		for c in $(podman ps -a -q --filter ancestor=localhost/mysite:1.0); do podman rm -f -t 0 "$c" >/dev/null 2>&1; done
+		podman rm -f -t 0 db >/dev/null 2>&1
+		podman volume rm -f dbdata >/dev/null 2>&1
+		podman rmi -f localhost/mysite:1.0 >/dev/null 2>&1
+	fi
+fi
+rm -rf /root/mysite
+if [ -z "$(state_get had-podman)" ] && pkg_installed podman; then
+	if is_rhel; then
+		dnf -y -q remove podman >/dev/null 2>&1
+	else
+		DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge podman >/dev/null 2>&1
+	fi
+fi
+true
@@ -0,0 +1,7 @@
+[ ! -e /root/mysite ] || refuse "/root/mysite already exists; move it away first"
+if pkg_installed podman; then
+	state_set had-podman 1
+	podman container exists db 2>/dev/null && refuse "a container named db already exists; remove it first (podman rm -f db)"
+	podman volume exists dbdata 2>/dev/null && refuse "a volume named dbdata already exists; remove it first (podman volume rm dbdata)"
+fi
+true
@@ -0,0 +1,33 @@
+# LFCA 6.2: Software application architecture
+
+Calling a REST API and reading its JSON, HTTP status codes, and a first
+SQL database.
+
+## Before you start
+
+Work as root (`sudo -i`), so `~` is `/root`. Answers go in
+`/root/lab-answers/<task>`. The VM needs internet access. The lab notes
+whether `jq` and SQLite are installed, and reset removes the ones it
+didn't find.
+
+## Tasks
+
+1. Call a public REST API with curl and read the JSON:
+   `curl -s https://api.github.com/repos/git/git`. Install `jq` if it's
+   missing, pick out the repository's description with `jq .description`,
+   and write it to `/root/lab-answers/1`.
+2. Request a repository that doesn't exist with `curl -i` (for example
+   `https://api.github.com/repos/git/no-such-repo`), find the HTTP status
+   code, and write it to `/root/lab-answers/2`.
+3. Install SQLite (`sqlite3` on Ubuntu, `sqlite` on the RHEL family). In
+   a database `~/shop.db`, create a table, insert three rows and query
+   them with SQL.
+
+## Hints
+
+- `curl -s URL | jq .`, `jq -r .description`.
+- The status code is on the first line of `curl -i`'s output; `curl -s -o
+  /dev/null -w '%{http_code}\n' URL` prints only the code.
+- `sqlite3 ~/shop.db`, then `CREATE TABLE products (id INTEGER PRIMARY KEY,
+  name TEXT, price REAL);`, `INSERT INTO products (name, price) VALUES
+  ('mug', 9.5);`, `SELECT * FROM products;`, `.tables`, `.quit`.
@@ -0,0 +1,21 @@
+# A table in ~/shop.db with at least three rows, read without the sqlite3 tool.
+three_rows() {
+	python3 -I - <<'X'
+import sqlite3, sys
+db = sqlite3.connect("file:/root/shop.db?mode=ro", uri=True)
+tables = [r[0] for r in db.execute("select name from sqlite_master where type = 'table'")]
+sys.exit(0 if any(db.execute('select count(*) from "%s"' % t.replace('"', '""')).fetchone()[0] >= 3 for t in tables) else 1)
+X
+}
+
+task 1 "The git/git repository's description in /root/lab-answers/1"
+need "jq is installed" 'command -v jq'
+need "the answer is the description from the API" 'grep -qi "git source code mirror" /root/lab-answers/1'
+
+task 2 "The status code for a missing repository in /root/lab-answers/2"
+need "the answer is the HTTP status code" '[ "$(answer 2)" = 404 ]'
+
+task 3 "SQLite: ~/shop.db with a table of three rows"
+need "sqlite3 is installed" 'command -v sqlite3'
+need "/root/shop.db is an SQLite database" '[ -f /root/shop.db ] && head -c 15 /root/shop.db | grep -q "SQLite format"'
+need "it has a table with at least three rows" three_rows
@@ -0,0 +1,3 @@
+TITLE="6.2 Software application architecture"
+CHAPTER=/lfca/6-it-project-management/6-2-software-application-architecture/
+SYSTEMS="rhel debian"
@@ -0,0 +1,9 @@
+rm -f /root/shop.db
+if [ -s "$LAB_STATE/not-installed" ]; then
+	if is_rhel; then
+		dnf -y -q remove $(cat "$LAB_STATE/not-installed") >/dev/null 2>&1
+	else
+		DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge $(cat "$LAB_STATE/not-installed") >/dev/null 2>&1
+	fi
+fi
+true
@@ -0,0 +1,5 @@
+[ ! -e /root/shop.db ] || refuse "/root/shop.db already exists; move it away first"
+for p in jq $(pick sqlite sqlite3); do
+	pkg_installed "$p" || echo "$p" >>"$LAB_STATE/not-installed"
+done
+true
@@ -0,0 +1,34 @@
+# LFCA 6.4: Open source software and licensing
+
+Finding the licenses of installed packages, and signing off a commit.
+
+## Before you start
+
+Work as root (`sudo -i`), so `~` is `/root`. Answers go in
+`/root/lab-answers/<task>`. The lab only checks that `~/practice-repo`
+doesn't exist yet, and notes whether you already have a `~/.gitconfig` and
+Git (reset removes the ones the lab didn't find).
+
+## Tasks
+
+1. Find the license of bash, curl, OpenSSL, SQLite and zlib from their
+   package files, and say whether each is permissive, copyleft or public
+   domain. Write one line per program to `/root/lab-answers/1`: the name in
+   lowercase, a space, and the kind, such as `bash copyleft`.
+2. Install Git if it's missing, set your name and email, create a practice
+   repository `~/practice-repo`, make a commit with `git commit -s`, and
+   look at the `Signed-off-by` line.
+
+## Hints
+
+- Ubuntu: the `Files: *` section of `/usr/share/doc/PACKAGE/copyright`.
+  RHEL family: `rpm -q --qf '%{LICENSE}\n' PACKAGE`.
+- Package names differ: `dpkg -S $(command -v curl)` or
+  `rpm -qf $(command -v curl)` finds the one for a program. For a library
+  with no program, `dpkg -S libz.so.1` or `rpm -qf /usr/lib64/libz.so.1`.
+  SQLite's command-line tool may not be installed; its library is
+  (`libsqlite3-0` on Ubuntu, `sqlite-libs` on the RHEL family).
+- `rpm` calls SQLite's license `blessing`: SPDX's name for its
+  public-domain dedication.
+- `git config --global user.name`, `user.email`; `git init`,
+  `git commit -s -m "..."`, `git log -1`.
@@ -0,0 +1,14 @@
+# kind NAME KIND: answer 1 has the line "NAME KIND".
+kind() { sed 's/^[[:space:]]*//; s/[[:space:]]*$//' /root/lab-answers/1 2>/dev/null | tr 'A-Z' 'a-z' | grep -Eq "^$1[[:space:]:,-]+$2\$"; }
+
+task 1 "The kinds of license in /root/lab-answers/1"
+need "bash is copyleft (GPLv3 or later)" 'kind bash copyleft'
+need "curl is permissive (the curl license, MIT-style)" 'kind curl permissive'
+need "openssl is permissive (Apache 2.0)" 'kind openssl permissive'
+need "sqlite is public domain" 'kind sqlite "public[ -]domain"'
+need "zlib is permissive (the zlib license)" 'kind zlib permissive'
+
+task 2 "A signed-off commit in ~/practice-repo"
+need "git is installed" 'command -v git'
+need "/root/practice-repo is a repository with a commit" 'git -C /root/practice-repo rev-parse -q --verify HEAD'
+need "a commit has a Signed-off-by line with a name and email" 'git -C /root/practice-repo log --format=%B | grep -Eq "^Signed-off-by: .+ <[^>]+@[^>]+>$"'
@@ -0,0 +1,3 @@
+TITLE="6.4 Open source software and licensing"
+CHAPTER=/lfca/6-it-project-management/6-4-open-source-licensing/
+SYSTEMS="rhel debian"
@@ -0,0 +1,6 @@
+rm -rf /root/practice-repo
+[ -n "$(state_get had-gitconfig)" ] || rm -f /root/.gitconfig
+if [ -n "$(state_get no-git)" ] && pkg_installed git; then
+	if is_rhel; then dnf -y -q remove git >/dev/null 2>&1; else DEBIAN_FRONTEND=noninteractive apt-get -y -qq --autoremove purge git >/dev/null 2>&1; fi
+fi
+true
@@ -0,0 +1,4 @@
+[ ! -e /root/practice-repo ] || refuse "/root/practice-repo already exists; move it away first"
+[ -e /root/.gitconfig ] && state_set had-gitconfig 1
+pkg_installed git || state_set no-git 1
+true
@@ -0,0 +1,56 @@
+# LPIC-3 331.1: X.509 certificates and public key infrastructures
+
+A two-level CA with OpenSSL: a root, an intermediate run with `openssl ca`,
+a server certificate, a revocation and a CRL, other file formats, and
+CFSSL.
+
+## Before you start
+
+The lab makes nothing but checks that `/root/pki` doesn't exist yet. Do
+everything as root, in `/root/pki`, with the file names the tasks give.
+Elliptic-curve P-256 keys without a passphrase are fine throughout.
+
+## Tasks
+
+1. Make a root CA, `root.key` (mode 600) and the self-signed `root.crt`
+   for `/O=Lab/CN=Lab Root CA`, valid for ten years, with
+   `basicConstraints` critical `CA:TRUE` and `keyUsage` critical
+   `keyCertSign` and `cRLSign`.
+2. Make an intermediate CA, `/O=Lab/CN=Lab Issuing CA`, in `int/`: its key
+   in `int/private/int.key` and its certificate `int/int.crt`, signed by the
+   root and allowing no CAs below it (`pathlen:0`). Set up `int/` for
+   `openssl ca` with a configuration `int/ca.cnf`, a database
+   `int/index.txt` and serial files starting at 1000.
+3. Request a server certificate for `/O=Lab/CN=www.lab.test` with the
+   names `www.lab.test` and `lab.test` (`www.key`, `www.csr`), and issue it
+   from the intermediate with `openssl ca` as `www.crt`, for server
+   authentication. Check that it verifies against the root only when the
+   intermediate is supplied.
+4. Revoke `www.crt` with reason `keyCompromise`, publish a CRL `int.crl`
+   from the intermediate, and check that verification with the CRL now
+   fails.
+5. Save `www.crt` in DER form as `www.der`, and bundle the key, the
+   certificate and the intermediate into `www.p12` with the password
+   `lab-only`.
+6. Ubuntu: with CFSSL (package `golang-cfssl`), make a root CA
+   `Lab CFSSL Root` (organization `Lab`) from a JSON request, written as
+   `cfssl-ca.pem` and `cfssl-ca-key.pem`.
+
+## Hints
+
+- `openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -noenc
+  -days ... -subj ... -addext ...`; `openssl x509 -noout -subject -issuer
+  -ext basicConstraints,keyUsage`.
+- `openssl x509 -req -CA root.crt -CAkey root.key -CAcreateserial
+  -extfile`; a `[ ca ]` section with `dir`, `database`, `new_certs_dir`,
+  `serial`, `crlnumber`, `certificate`, `private_key`, `policy` and
+  `copy_extensions = copy`.
+- `openssl req -new ... -addext subjectAltName=...`;
+  `openssl ca -config int/ca.cnf -extensions ...`; `openssl verify -CAfile
+  root.crt -untrusted int/int.crt www.crt`.
+- `openssl ca -revoke ... -crl_reason`, `openssl ca -gencrl`;
+  `openssl verify -crl_check -CAfile chain.pem -CRLfile int.crl`.
+- `openssl x509 -outform DER`; `openssl pkcs12 -export -inkey -in
+  -certfile -passout`.
+- `cfssl gencert -initca ca.json | cfssljson -bare cfssl-ca`;
+  `cfssl certinfo -cert`.
@@ -0,0 +1,53 @@
+P=/root/pki
+cd $P 2>/dev/null || cd /
+# x FILE OPTIONS: openssl x509 -noout on a certificate.
+x() { local f=$1; shift; openssl x509 -in "$f" -noout "$@" 2>/dev/null; }
+# key_matches CERT KEY: the key is the certificate's.
+key_matches() { [ -n "$(x "$1" -pubkey)" ] && [ "$(x "$1" -pubkey)" = "$(openssl pkey -in "$2" -pubout 2>/dev/null)" ]; }
+ext() { x "$1" -ext "$2" | tail -n +2 | tr -d ' '; }
+
+task 1 "Root CA root.crt, self-signed, ten years, CA with keyCertSign and cRLSign"
+need "$P/root.crt is a certificate" 'x root.crt'
+need "root.key is its key, mode 600" 'key_matches root.crt root.key && [ "$(stat -c %a root.key)" = 600 ]'
+need "the subject is O = Lab, CN = Lab Root CA" 'x root.crt -subject -nameopt oneline | grep -Eq "O = Lab, CN = Lab Root CA$"'
+need "it's self-signed" '[ "$(x root.crt -subject)" = "subject=$(x root.crt -issuer | cut -d= -f2-)" ] && openssl verify -CAfile root.crt root.crt'
+need "basicConstraints is critical, CA:TRUE" 'x root.crt -ext basicConstraints | grep -q critical && ext root.crt basicConstraints | grep -q "^CA:TRUE"'
+need "keyUsage is critical, Certificate Sign and CRL Sign" 'x root.crt -ext keyUsage | grep -q critical && ext root.crt keyUsage | grep -q CertificateSign,CRLSign'
+need "it's valid for about ten years" 'x root.crt -checkend $((3640 * 86400)) >/dev/null'
+
+task 2 "Intermediate CA int/int.crt, pathlen:0, set up for openssl ca"
+need "int/int.crt is signed by the root" 'openssl verify -CAfile root.crt int/int.crt'
+need "int/private/int.key is its key" 'key_matches int/int.crt int/private/int.key'
+need "the subject is O = Lab, CN = Lab Issuing CA" 'x int/int.crt -subject -nameopt oneline | grep -Eq "O = Lab, CN = Lab Issuing CA$"'
+need "basicConstraints is CA:TRUE, pathlen:0" 'ext int/int.crt basicConstraints | grep -qx "CA:TRUE,pathlen:0"'
+need "int/ca.cnf, int/index.txt and int/serial exist" '[ -f int/ca.cnf ] && [ -f int/index.txt ] && [ -f int/serial ]'
+
+task 3 "www.crt for www.lab.test and lab.test, issued with openssl ca"
+need "www.crt is issued by the intermediate" 'x www.crt && [ "$(x www.crt -issuer | cut -d= -f2-)" = "$(x int/int.crt -subject | cut -d= -f2-)" ]'
+need "www.key is its key" 'key_matches www.crt www.key'
+need "its subject is O = Lab, CN = www.lab.test" 'x www.crt -subject -nameopt oneline | grep -Eq "O = Lab, CN = www.lab.test$"'
+need "its names are www.lab.test and lab.test" 's=$(ext www.crt subjectAltName); grep -q "DNS:www.lab.test" <<<"$s" && grep -Eq "DNS:lab.test(,|$)" <<<"$s"'
+need "it's for server authentication" 'ext www.crt extendedKeyUsage | grep -q "TLSWebServerAuthentication"'
+need "it's not a CA" '! ext www.crt basicConstraints | grep -q CA:TRUE'
+need "it verifies against the root with the intermediate supplied" 'openssl verify -CAfile root.crt -untrusted int/int.crt www.crt'
+need "int/index.txt has its serial (it came from openssl ca)" 's=$(x www.crt -serial | cut -d= -f2); awk -F"\t" -v s="$s" "\$4 == s { f = 1 } END { exit !f }" int/index.txt'
+
+task 4 "www.crt revoked for key compromise, CRL int.crl"
+need "int/index.txt marks www.crt revoked" 's=$(x www.crt -serial | cut -d= -f2); awk -F"\t" -v s="$s" "\$4 == s && \$1 == \"R\" { f = 1 } END { exit !f }" int/index.txt'
+need "int.crl is signed by the intermediate" 'openssl crl -in int.crl -CAfile int/int.crt -noout'
+need "int.crl lists www.crt with reason Key Compromise" 's=$(x www.crt -serial | cut -d= -f2); openssl crl -in int.crl -noout -text | grep -A5 "Serial Number: $s" | grep -q "Key Compromise"'
+need "verification with the CRL reports it revoked" 'cat int/int.crt root.crt >/tmp/le-chain.pem; openssl verify -crl_check -CAfile /tmp/le-chain.pem -CRLfile int.crl www.crt 2>&1 | grep -q "certificate revoked"'
+rm -f /tmp/le-chain.pem
+
+task 5 "www.der and www.p12"
+need "www.der is www.crt in DER form" '[ -n "$(x www.crt -fingerprint)" ] && [ "$(openssl x509 -inform DER -in www.der -noout -fingerprint 2>/dev/null)" = "$(x www.crt -fingerprint)" ] && ! grep -q BEGIN www.der'
+need "www.p12 opens with the password lab-only" 'openssl pkcs12 -in www.p12 -passin pass:lab-only -noout'
+need "www.p12 holds www.crt and the intermediate" 'openssl pkcs12 -in www.p12 -passin pass:lab-only -nokeys 2>/dev/null >/tmp/le-p12.pem; [ "$(grep -c "BEGIN CERTIFICATE" /tmp/le-p12.pem)" = 2 ] && grep -q "Lab Issuing CA" <(openssl crl2pkcs7 -nocrl -certfile /tmp/le-p12.pem | openssl pkcs7 -print_certs -noout)'
+need "www.p12 holds www.key" '[ "$(openssl pkcs12 -in www.p12 -passin pass:lab-only -nocerts -nodes 2>/dev/null | openssl pkey -pubout 2>/dev/null)" = "$(openssl pkey -in www.key -pubout 2>/dev/null)" ]'
+rm -f /tmp/le-p12.pem
+
+task 6 "CFSSL root CA cfssl-ca.pem" debian
+need "cfssl-ca.pem is a CA certificate" 'ext cfssl-ca.pem basicConstraints | grep -q "^CA:TRUE"'
+need "its subject has O = Lab, CN = Lab CFSSL Root" 's=$(x cfssl-ca.pem -subject -nameopt oneline); grep -q "O = Lab" <<<"$s" && grep -q "CN = Lab CFSSL Root" <<<"$s"'
+need "it's self-signed" 'openssl verify -CAfile cfssl-ca.pem cfssl-ca.pem'
+need "cfssl-ca-key.pem is its key" 'key_matches cfssl-ca.pem cfssl-ca-key.pem'
@@ -0,0 +1,3 @@
+TITLE="331.1 X.509 certificates and public key infrastructures"
+CHAPTER=/lpic-3-303/331-cryptography/331-1-x509-and-pki/
+SYSTEMS="rhel debian"
@@ -0,0 +1,17 @@
+rm -rf /root/pki
+rm -f /tmp/le-chain.pem /tmp/le-p12.pem
+# The packages installed since setup started, and only those: not the
+# dependencies they leave unneeded that were there before.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+if [ -s "$LAB_STATE/pkgs-before" ]; then
+	new=$(pkgs_now | comm -13 "$LAB_STATE/pkgs-before" - | grep -Ev '^(kernel|linux-)')
+	if [ -n "$new" ]; then
+		say "Removing the packages installed since the lab started..."
+		if is_rhel; then
+			dnf -y -q remove --setopt=clean_requirements_on_remove=False $new >/dev/null 2>&1
+		else
+			DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge $new >/dev/null 2>&1
+		fi
+	fi
+fi
+true
@@ -0,0 +1,7 @@
+[ -e /root/pki ] && refuse "/root/pki already exists; move it away and start again"
+# Reset removes every package installed from here on, by setup or by the
+# tasks, and only those.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+pkgs_now >"$LAB_STATE/pkgs-before"
+command -v openssl >/dev/null || pkg_install openssl
+true
@@ -0,0 +1,54 @@
+# LPIC-3 331.2: Certificates for encryption, signing and authentication
+
+Apache with two HTTPS sites on one address, HSTS, a client-certificate
+area and OCSP stapling, all from a small lab CA.
+
+## Before you start
+
+The lab points the made-up names `alpha.test` and `beta.test` at
+`127.0.0.1` in `/etc/hosts`, and checks that nothing listens on port 443
+yet. Keep your CA, keys and certificates in `/root/tls`, with the file names
+the tasks give, and copy what Apache needs wherever you like. Ubuntu's
+package is `apache2`; on the RHEL family it's `httpd` with `mod_ssl`.
+
+## Tasks
+
+1. In `/root/tls`, make a lab CA `ca.crt` (`/O=Lab/CN=Lab TLS CA`, key
+   `ca.key`), and sign with it: `alpha.crt` and `beta.crt` (keys
+   `alpha.key` and `beta.key`) for server authentication, each with its
+   name (`alpha.test`, `beta.test`) as subject alternative name and the
+   OCSP responder `http://127.0.0.1:8888` in its authority information
+   access; and a client certificate `user.crt` (key `user.key`) for
+   `/O=Lab/CN=lab-user`, for client authentication.
+2. Install Apache and serve two HTTPS sites on port 443: `alpha.test`,
+   whose page says `alpha site`, and `beta.test`, whose page says
+   `beta site`, each with its own certificate. Allow only TLS 1.2 and 1.3,
+   and send HSTS (`Strict-Transport-Security`) on both. Apache must start
+   at boot.
+3. On `alpha.test`, make `/admin/` (a page that says `admin area`) require
+   a client certificate issued by the lab CA with organization `Lab`.
+4. Run an OCSP responder for the lab CA on port 8888 with
+   `openssl ocsp` and an index file that lists `alpha.crt` as valid, and
+   turn on OCSP stapling, so that a client asking for the status of
+   `alpha.test` gets a stapled `good` answer.
+
+## Hints
+
+- `openssl req -x509 ... -addext basicConstraints=critical,CA:TRUE`;
+  `openssl req -new`, then `openssl x509 -req -CA ca.crt -CAkey ca.key
+  -CAcreateserial -extfile` with `subjectAltName`, `extendedKeyUsage` and
+  `authorityInfoAccess=OCSP;URI:...` lines.
+- Two `<VirtualHost *:443>` blocks with `ServerName`, `SSLEngine on`,
+  `SSLCertificateFile`, `SSLCertificateKeyFile`, `SSLProtocol` and
+  `Header always set Strict-Transport-Security ...`. Ubuntu: `a2enmod ssl
+  headers`, `a2ensite`. Check with `apachectl configtest` and
+  `openssl s_client -connect 127.0.0.1:443 -servername beta.test`.
+- `SSLCACertificateFile`, and in a `<Location /admin>`:
+  `SSLVerifyClient require`, `Require expr "%{SSL_CLIENT_S_DN_O} == 'Lab'"`.
+  Test with `curl --cacert ca.crt --cert user.crt --key user.key`.
+- An index line is `V`, the expiry as `YYMMDDHHMMSSZ`, an empty field, the
+  serial, `unknown` and the subject, separated by tabs.
+  `openssl ocsp -port 8888 -index ... -CA ca.crt -rsigner ca.crt -rkey
+  ca.key`, in the background. `SSLStaplingCache` (outside the virtual
+  hosts) and `SSLUseStapling on`; `openssl s_client ... -status`. On the
+  RHEL family, see what SELinux says when Apache asks the responder.
@@ -0,0 +1,45 @@
+T=/root/tls
+cd $T 2>/dev/null || cd /
+x() { local f=$1; shift; openssl x509 -in "$f" -noout "$@" 2>/dev/null; }
+ext() { x "$1" -ext "$2" | tail -n +2 | tr -d ' '; }
+key_matches() { [ -n "$(x "$1" -pubkey)" ] && [ "$(x "$1" -pubkey)" = "$(openssl pkey -in "$2" -pubout 2>/dev/null)" ]; }
+# sclient NAME [options]: a handshake with the server for NAME on 127.0.0.1:443.
+sclient() { local n=$1; shift; timeout 10 openssl s_client -connect 127.0.0.1:443 -servername "$n" -CAfile $T/ca.crt "$@" </dev/null 2>&1; }
+CURL=(curl -s --max-time 5 --cacert $T/ca.crt)
+SVC=$(pick httpd apache2)
+
+task 1 "Lab CA, alpha.crt and beta.crt with OCSP URL, client certificate user.crt"
+need "ca.crt is a CA for O = Lab, CN = Lab TLS CA" 'x ca.crt -subject -nameopt oneline | grep -Eq "O = Lab, CN = Lab TLS CA$" && ext ca.crt basicConstraints | grep -q "^CA:TRUE"'
+need "ca.key is its key" 'key_matches ca.crt ca.key'
+for s in alpha beta; do
+	need "$s.crt is signed by the CA, with key $s.key" "openssl verify -CAfile ca.crt $s.crt && key_matches $s.crt $s.key"
+	need "$s.crt names $s.test" "ext $s.crt subjectAltName | grep -q 'DNS:$s.test'"
+	need "$s.crt is for server authentication" "ext $s.crt extendedKeyUsage | grep -q TLSWebServerAuthentication"
+	need "$s.crt names the OCSP responder http://127.0.0.1:8888" "x $s.crt -ocsp_uri | grep -Eq '^http://127\.0\.0\.1:8888/?$'"
+done
+need "user.crt is signed by the CA, with key user.key" 'openssl verify -CAfile ca.crt user.crt && key_matches user.crt user.key'
+need "user.crt is for O = Lab, CN = lab-user" 'x user.crt -subject -nameopt oneline | grep -Eq "O = Lab, CN = lab-user$"'
+need "user.crt is for client authentication" 'ext user.crt extendedKeyUsage | grep -q TLSWebClientAuthentication'
+
+task 2 "Two HTTPS sites on 443, by name, TLS 1.2+, HSTS"
+need "Apache is running and enabled" "active $SVC && enabled $SVC"
+need "something listens on port 443" 'for i in $(seq 15); do listening 443 tcp && exit 0; sleep 1; done; exit 1'
+for s in alpha beta; do
+	need "$s.test gets $s.crt, which verifies" "sclient $s.test >/tmp/le-s.out; grep -q 'Verify return code: 0' /tmp/le-s.out && grep -Eq '^subject=.*CN ?= ?$s.test' /tmp/le-s.out"
+	need "https://$s.test/ says $s site" "\"\${CURL[@]}\" https://$s.test/ | grep -q '$s site'"
+	need "https://$s.test/ sends Strict-Transport-Security" "\"\${CURL[@]}\" -I https://$s.test/ | grep -qi '^strict-transport-security:'"
+done
+need "TLS 1.3 works" 'sclient alpha.test -tls1_3 | grep -q "TLSv1.3"'
+need "TLS 1.1 is refused" '! sclient alpha.test -tls1_1 -cipher DEFAULT@SECLEVEL=0 | grep -q "^New, TLSv1"'
+rm -f /tmp/le-s.out
+
+task 3 "alpha.test/admin/ needs a lab client certificate"
+need "without a client certificate, /admin/ is refused" '! "${CURL[@]}" https://alpha.test/admin/ | grep -q "admin area"'
+need "with user.crt, /admin/ says admin area" '"${CURL[@]}" --cert user.crt --key user.key https://alpha.test/admin/ | grep -q "admin area"'
+need "the rest of alpha.test needs no certificate" '"${CURL[@]}" https://alpha.test/ | grep -q "alpha site"'
+
+task 4 "OCSP responder on port 8888, stapled good answer"
+need "an OCSP responder listens on port 8888" 'listening 8888 tcp'
+need "it says alpha.crt is good" 'openssl ocsp -issuer ca.crt -cert alpha.crt -url http://127.0.0.1:8888 -CAfile ca.crt -timeout 5 2>/dev/null | grep -q "alpha.crt: good"'
+need "alpha.test staples a good OCSP answer" 'for i in 1 2 3; do sclient alpha.test -status >/tmp/le-s.out; grep -q "OCSP Response Status: successful" /tmp/le-s.out && grep -q "Cert Status: good" /tmp/le-s.out && exit 0; sleep 2; done; exit 1'
+rm -f /tmp/le-s.out
@@ -0,0 +1,3 @@
+TITLE="331.2 Certificates for encryption, signing and authentication"
+CHAPTER=/lpic-3-303/331-cryptography/331-2-x509-in-apache/
+SYSTEMS="rhel debian"
@@ -0,0 +1,29 @@
+pkill -f '^openssl ocsp' 2>/dev/null
+SVC=$(pick httpd apache2) D=$(pick /etc/httpd /etc/apache2)
+systemctl disable --now --quiet $SVC 2>/dev/null
+# The packages installed since setup started, and only those: not the
+# dependencies they leave unneeded that were there before.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+if [ -s "$LAB_STATE/pkgs-before" ]; then
+	new=$(pkgs_now | comm -13 "$LAB_STATE/pkgs-before" - | grep -Ev '^(kernel|linux-)')
+	if [ -n "$new" ]; then
+		say "Removing the packages installed since the lab started..."
+		if is_rhel; then
+			dnf -y -q remove --setopt=clean_requirements_on_remove=False $new >/dev/null 2>&1
+		else
+			DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge $new >/dev/null 2>&1
+		fi
+	fi
+fi
+# Apache's configuration as it was, or none.
+rm -rf $D /var/www/alpha /var/www/beta
+if [ -f "$LAB_STATE/apache.tar" ]; then
+	tar -C / -xf "$LAB_STATE/apache.tar"
+	[ -n "$(state_get was-enabled)" ] && systemctl enable --now --quiet $SVC
+fi
+if is_rhel; then
+	[ -n "$(state_get had-nc)" ] || setsebool -P httpd_can_network_connect off 2>/dev/null
+	[ -s "$LAB_STATE/port-8888" ] || semanage port -d -p tcp 8888 2>/dev/null
+fi
+rm -rf /root/tls /tmp/le-s.out
+true
@@ -0,0 +1,18 @@
+listening 443 tcp && refuse "something already listens on port 443; stop it and start again"
+pgrep -f '^openssl ocsp' >/dev/null && refuse "an openssl ocsp responder is already running; stop it and start again"
+[ -e /root/tls ] && refuse "/root/tls already exists; move it away and start again"
+# Reset removes every package installed from here on, by setup or by the
+# tasks, and only those.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+pkgs_now >"$LAB_STATE/pkgs-before"
+command -v openssl >/dev/null || pkg_install openssl
+hosts_add "127.0.0.1 alpha.test beta.test"
+# Apache's configuration, if it's there, for reset to put back.
+D=$(pick /etc/httpd /etc/apache2)
+[ -d "$D" ] && tar -C / -cf "$LAB_STATE/apache.tar" "${D#/}"
+if is_rhel; then
+	getsebool httpd_can_network_connect 2>/dev/null | grep -q 'on$' && state_set had-nc 1
+	semanage port -l 2>/dev/null | grep -w 8888 >"$LAB_STATE/port-8888" || true
+fi
+pkg_installed "$(pick httpd apache2)" && enabled "$(pick httpd apache2)" && state_set was-enabled 1
+true
@@ -0,0 +1,59 @@
+# LPIC-3 331.3: Encrypted file systems
+
+LUKS on a spare disk: key slots, a key file, the LUKS1 to LUKS2
+conversion, unlocking at boot from crypttab, network-bound unlocking with
+Clevis and Tang, and an eCryptfs private directory.
+
+## Before you start
+
+The lab adds a spare disk, `/dev/loop40` (512 MiB, empty). It's an image
+file under `/var/lib/le-lab/disks/`, attached again at every boot, before
+crypttab is read.
+
+The peer, at `10.99.0.2` (name `peer`), stands in for the other VM and
+already runs a Tang server on its port 80 (`le-lab-peer-tang.socket`):
+`curl http://10.99.0.2/adv` shows its advertisement.
+
+On Ubuntu, the lab also creates the user `crypto` (password `Lab-Pass-1`)
+for task 5.
+
+## Tasks
+
+1. Install `cryptsetup` if it's missing. Format `/dev/loop40` as LUKS1
+   with the passphrase `lab-pass-one`, open it as `secret`, make an ext4
+   file system on it, mount it on `/mnt/secret`, and write `top secret` to
+   `/mnt/secret/note.txt`.
+2. Add a 64-byte random key file, `/root/secret.key` (mode 400), to a
+   second key slot. Then close the volume and convert it to LUKS2.
+3. Unlock and mount the volume at boot the way the system does it: a
+   crypttab entry `secret` that names the volume by UUID and uses the key
+   file, and an fstab entry that mounts `/dev/mapper/secret` on
+   `/mnt/secret`, both with `nofail`. Start it through systemd and check
+   that `note.txt` is there.
+4. Install Clevis with its LUKS support, and bind the volume to the Tang
+   server at `http://10.99.0.2`, so that `clevis luks unlock` opens it with
+   no passphrase while Tang is reachable. The passphrase must still work.
+5. Ubuntu: set up an eCryptfs private directory for `crypto`, wrapped with
+   their login password `Lab-Pass-1`, that doesn't mount automatically at
+   login. Mount it as `crypto`, write `private data` to
+   `~/Private/diary.txt`, and unmount it again.
+
+## Hints
+
+- `cryptsetup luksFormat --type luks1`, `cryptsetup open`,
+  `cryptsetup status`; `echo -n 'lab-pass-one' | cryptsetup ... -` reads
+  the passphrase from standard input.
+- `dd if=/dev/urandom bs=64 count=1`, `cryptsetup luksAddKey`,
+  `cryptsetup luksDump`, `cryptsetup convert --type luks2` (the volume
+  must be closed).
+- `cryptsetup luksUUID`; a crypttab line is name, device, key file,
+  options. `systemctl daemon-reload`,
+  `systemctl start [email protected]`, `mount /mnt/secret`.
+- `clevis luks bind -d DEVICE tang '{"url":"http://10.99.0.2"}'`,
+  `clevis luks list`; `clevis luks unlock -d DEVICE -n NAME` needs the
+  volume closed first.
+- `ecryptfs-utils`, `modprobe ecryptfs`;
+  `su - crypto -c "ecryptfs-setup-private --nopwcheck --noautomount ..."`.
+  To mount without a terminal, `ecryptfs-insert-wrapped-passphrase-into-keyring
+  ~/.ecryptfs/wrapped-passphrase -` with the login password on standard
+  input, then `/sbin/mount.ecryptfs_private`; `/sbin/umount.ecryptfs_private`.
@@ -0,0 +1,37 @@
+D=/dev/loop40
+UUID=$(blkid -s UUID -o value $D 2>/dev/null)
+# crypttab_secret FIELD: a field of the crypttab entry "secret".
+crypttab_secret() { awk -v f="$1" '$1 == "secret" { print $f; exit }' /etc/crypttab 2>/dev/null; }
+
+task 1 "LUKS on /dev/loop40, open as secret, ext4 on /mnt/secret with note.txt"
+need "/dev/loop40 is a LUKS volume" '[ "$(blkid -s TYPE -o value $D)" = crypto_LUKS ]'
+need "the passphrase lab-pass-one opens it" 'printf %s lab-pass-one | cryptsetup open --test-passphrase $D -'
+need "it's open as /dev/mapper/secret" 'cryptsetup status secret | grep -Eq "device:[[:space:]]+$D$"'
+need "/mnt/secret is mounted from /dev/mapper/secret, ext4" '[ "$(findmnt -n -o SOURCE,FSTYPE /mnt/secret | tr -s " ")" = "/dev/mapper/secret ext4" ]'
+need "/mnt/secret/note.txt says top secret" 'grep -qx "top secret" /mnt/secret/note.txt'
+
+task 2 "Key file /root/secret.key in a second slot, header converted to LUKS2"
+need "/root/secret.key is 64 bytes, mode 400" '[ "$(stat -c %s:%a /root/secret.key)" = 64:400 ]'
+need "the key file opens the volume" 'cryptsetup open --test-passphrase --key-file /root/secret.key $D'
+need "the header is LUKS2" 'cryptsetup luksDump $D | grep -Eq "^Version:[[:space:]]+2$"'
+need "a key slot still uses PBKDF2 (it was converted from LUKS1)" 'cryptsetup luksDump $D | grep -Eq "^[[:space:]]+PBKDF:[[:space:]]+pbkdf2"'
+
+task 3 "crypttab and fstab unlock and mount it at boot"
+need "/etc/crypttab has an entry secret for the volume's UUID" '[ -n "$UUID" ] && [ "$(crypttab_secret 2)" = "UUID=$UUID" ]'
+need "the entry uses /root/secret.key" '[ "$(crypttab_secret 3)" = /root/secret.key ]'
+need "the entry has nofail" 'crypttab_secret 4 | tr , "\n" | grep -qx nofail'
+need "/etc/fstab mounts /dev/mapper/secret on /mnt/secret, with nofail" 'awk "\$1 == \"/dev/mapper/secret\" && \$2 == \"/mnt/secret\" && \$4 ~ /(^|,)nofail(,|$)/ { f = 1 } END { exit !f }" /etc/fstab'
+need "[email protected] is active" active [email protected]
+need "/mnt/secret is mounted with note.txt" 'grep -qx "top secret" /mnt/secret/note.txt'
+
+task 4 "Bound to Tang at http://10.99.0.2 with Clevis"
+need "clevis luks list shows a tang binding to 10.99.0.2" 'clevis luks list -d $D 2>/dev/null | grep -q "tang.*10\.99\.0\.2"'
+need "Clevis gets a working key from Tang" 'for s in $(clevis luks list -d $D 2>/dev/null | grep tang | cut -d: -f1); do timeout 15 clevis luks pass -d $D -s $s 2>/dev/null | cryptsetup open --test-passphrase --key-file - $D && exit 0; done; exit 1'
+need "the passphrase lab-pass-one still works" 'printf %s lab-pass-one | cryptsetup open --test-passphrase $D -'
+
+task 5 "eCryptfs private directory for crypto" debian
+need "crypto has a wrapped passphrase" '[ -s /home/crypto/.ecryptfs/wrapped-passphrase ]'
+need "it doesn't mount automatically at login" '[ ! -e /home/crypto/.ecryptfs/auto-mount ]'
+need "~/.Private holds an encrypted file name" 'ls /home/crypto/.Private | grep -q "^ECRYPTFS_FNEK_ENCRYPTED\."'
+need "nothing in ~/.Private is stored in the clear" '! grep -rqs "private data" /home/crypto/.Private'
+need "mounted with Lab-Pass-1, ~/Private/diary.txt says private data" 'modprobe ecryptfs; if grep -q " /home/crypto/Private ecryptfs " /proc/mounts; then grep -qx "private data" /home/crypto/Private/diary.txt; else su - crypto -c "printf %s Lab-Pass-1 | ecryptfs-insert-wrapped-passphrase-into-keyring ~/.ecryptfs/wrapped-passphrase - && /sbin/mount.ecryptfs_private && grep -qx \"private data\" ~/Private/diary.txt; r=\$?; /sbin/umount.ecryptfs_private; exit \$r"; fi'
@@ -0,0 +1,3 @@
+TITLE="331.3 Encrypted file systems"
+CHAPTER=/lpic-3-303/331-cryptography/331-3-encrypted-file-systems/
+SYSTEMS="rhel debian"
@@ -0,0 +1,29 @@
+umount /mnt/secret 2>/dev/null
+systemctl stop [email protected] 2>/dev/null
+cryptsetup close secret 2>/dev/null
+cp -a "$LAB_STATE/fstab" /etc/fstab 2>/dev/null
+if [ -f "$LAB_STATE/crypttab" ]; then cp -a "$LAB_STATE/crypttab" /etc/crypttab; else rm -f /etc/crypttab; fi
+rm -f /root/secret.key
+rmdir /mnt/secret 2>/dev/null
+for u in le-lab-peer-tang.socket; do systemctl disable --now --quiet $u 2>/dev/null; done
+systemctl stop 'le-lab-peer-tang@*' 2>/dev/null
+rm -f /etc/systemd/system/le-lab-peer-tang.socket /etc/systemd/system/[email protected]
+systemctl daemon-reload
+# crypto's private directory goes with the user; its mount first.
+umount /home/crypto/Private 2>/dev/null
+# The packages installed since setup started, and only those: not the
+# dependencies they leave unneeded that were there before.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+if [ -s "$LAB_STATE/pkgs-before" ]; then
+	new=$(pkgs_now | comm -13 "$LAB_STATE/pkgs-before" - | grep -Ev '^(kernel|linux-)')
+	if [ -n "$new" ]; then
+		say "Removing the packages installed since the lab started..."
+		if is_rhel; then
+			dnf -y -q remove --setopt=clean_requirements_on_remove=False $new >/dev/null 2>&1
+		else
+			DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge $new >/dev/null 2>&1
+		fi
+	fi
+fi
+grep -qx tang <<<"${new:-}" && rm -rf $(pick /var/db/tang /var/lib/tang)
+true
@@ -0,0 +1,48 @@
+[ -e /dev/mapper/secret ] && refuse "a device-mapper device called secret already exists; close it and start again"
+findmnt /mnt/secret >/dev/null && refuse "something is mounted on /mnt/secret; unmount it and start again"
+grep -Eqs '^[[:space:]]*secret[[:space:]]' /etc/crypttab && refuse "/etc/crypttab already has an entry called secret; remove it and start again"
+[ -e /root/secret.key ] && refuse "/root/secret.key already exists; move it away and start again"
+# Reset removes every package installed from here on, by setup or by the
+# tasks, and only those.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+pkgs_now >"$LAB_STATE/pkgs-before"
+cp -a /etc/fstab "$LAB_STATE/fstab"
+[ -f /etc/crypttab ] && cp -a /etc/crypttab "$LAB_STATE/crypttab"
+
+disk_add 0 512M >/dev/null
+is_debian && user_add crypto
+
+# The peer's Tang server: its own keys, on the peer's port 80.
+pkg_install tang
+systemctl disable --now --quiet tangd.socket 2>/dev/null || true
+K=$(pick /var/db/tang /var/lib/tang)
+mkdir -p $K
+ls $K/*.jwk >/dev/null 2>&1 || /usr/libexec/tangd-keygen $K
+peer_up
+cat >/etc/systemd/system/le-lab-peer-tang.socket <<'X'
+[Unit]
+Description=Practice lab Tang server on the peer
+DefaultDependencies=no
+Requires=le-lab-peer.service
+After=le-lab-peer.service
+
+[Socket]
+NetworkNamespacePath=/run/netns/lepeer
+ListenStream=10.99.0.2:80
+Accept=yes
+
+[Install]
+WantedBy=multi-user.target
+X
+cat >/etc/systemd/system/[email protected] <<X
+[Unit]
+Description=Practice lab Tang server on the peer (connection)
+
+[Service]
+StandardInput=socket
+StandardOutput=socket
+StandardError=journal
+ExecStart=/usr/libexec/tangd $K
+X
+systemctl daemon-reload
+systemctl enable --now --quiet le-lab-peer-tang.socket
@@ -0,0 +1,84 @@
+# LPIC-3 331.4: DNS and cryptography
+
+BIND as an authoritative-only server for two made-up zones, one signed by
+a `dnssec-policy` and one signed by hand, with trust anchors to validate
+them, TSIG for transfers and updates, CAA and DANE records, and DNS over
+TLS.
+
+## Before you start
+
+This VM is the authoritative server, at `10.99.0.1` (interface
+`le-host`). The peer, at `10.99.0.2` (name `peer`), stands in for the
+other VM and plays the client and validator: the lab checks your server
+from there. Get a shell in the peer with `sudo ip netns exec lepeer bash`;
+it shares this VM's files and programs.
+
+The package is `bind` (with `bind-utils`) on the RHEL family and `bind9`
+(with `bind9-dnsutils`) on Ubuntu. Keep your work files (sources of the
+hand-signed zone, keys, certificates) in `/root/dns`. Where these tasks
+say "the key file", they mean `/etc/named/lab-key.conf` on the RHEL family
+and `/etc/bind/lab-key.conf` on Ubuntu. The firewall, if it's running, is
+yours to open.
+
+## Tasks
+
+1. Install BIND and make it an authoritative-only server (no recursion
+   for anyone), listening on `127.0.0.1` and `10.99.0.1`, and starting at
+   boot. Serve the zone `lab.test` with an SOA, `NS ns1.lab.test.`, the
+   CAA records `0 issue "letsencrypt.org"`, `0 issuewild ";"` and
+   `0 iodef "mailto:[email protected]"`, and A records for `ns1` and `www`
+   pointing at `10.99.0.1`. Sign it with a `dnssec-policy` that has a
+   separate KSK and ZSK (ECDSAP256SHA256) and uses NSEC3. Open the firewall
+   for DNS if it's running.
+2. Sign a second zone, `manual.test`, by hand with a KSK and a ZSK
+   (`dnssec-keygen`, `dnssec-signzone`), with the same `NS` and A records
+   (`ns1.manual.test.` and `www`, at `10.99.0.1`), and serve the signed
+   file. Its denials use NSEC.
+3. Make a TSIG key `lab-key` (HMAC-SHA256) in the key file. Allow zone
+   transfers of `lab.test` only with it, and dynamic updates of `lab.test`
+   only with it. Then add `mail.lab.test`, A `10.99.0.3`, with a signed
+   dynamic update.
+4. Write `/root/dns/lab-anchors.conf`, a `trust-anchors` block with a
+   `static-ds` entry for each zone built from its KSK, so that, from the
+   peer, `delv @10.99.0.1 -a /root/dns/lab-anchors.conf +root=lab.test
+   www.lab.test` (and the same for `manual.test`) reports
+   `fully validated`. Break `manual.test` once by editing an address in its
+   signed file without re-signing, see validation fail, and fix it by
+   signing again.
+5. Publish DANE for a TLS service on port 4433: make a self-signed
+   certificate for `www.lab.test` (`/root/dns/www.crt`, key
+   `/root/dns/www.key`), add its `3 1 1` TLSA record at
+   `_4433._tcp.www.lab.test` with a signed update, and serve it with
+   `openssl s_server` on port 4433. Open the firewall for it if it's
+   running.
+6. Answer DNS over TLS on `10.99.0.1` port 853, so that
+   `dig +tls @10.99.0.1 www.lab.test` works from the peer. Open the
+   firewall for it if it's running.
+
+## Hints
+
+- RHEL family: `/etc/named.conf`, zones under `/var/named` (SELinux lets
+  named write in `/var/named/dynamic`, which inline signing needs).
+  Ubuntu: `/etc/bind/named.conf.options` and `named.conf.local`, zones
+  under `/var/lib/bind`. `recursion no;`, `listen-on`, `named-checkconf
+  -z`, `named-checkzone`, `rndc reload`.
+- `dnssec-policy NAME { keys { ksk ...; zsk ...; }; nsec3param iterations
+  0 optout no salt-length 0; };` and in the zone `dnssec-policy NAME;
+  inline-signing yes;`; `rndc dnssec -status lab.test`;
+  `dig @10.99.0.1 lab.test DNSKEY +dnssec +multi`.
+- `dnssec-keygen -a ECDSAP256SHA256 [-f KSK] manual.test`,
+  `dnssec-signzone -S -N INCREMENT -o manual.test manual.test.db`.
+- `tsig-keygen -a hmac-sha256 lab-key`, `include` it, `allow-transfer {
+  key lab-key; };`, `update-policy { grant lab-key zonesub ANY; };`;
+  `dig -k KEYFILE ... AXFR`, `nsupdate -k KEYFILE`.
+- `dig ... DNSKEY | dnssec-dsfromkey -f - ZONE`; an entry looks like
+  `lab.test. static-ds 12345 13 2 "ABCD...";`. `rndc flush` doesn't matter
+  here: `delv` validates for itself. `delv +cd`, `dig +cd` and
+  `+dnssec` show more.
+- `openssl req -x509 -newkey ec ...`; the TLSA data is the SHA-256 of the
+  public key in DER: `openssl x509 -noout -pubkey | openssl pkey -pubin
+  -outform DER | sha256sum`. `openssl s_server -accept 4433 -cert ... -key
+  ... -www`, in the background; `openssl s_client -dane_tlsa_domain
+  -dane_tlsa_rrdata`.
+- `listen-on port 853 tls ephemeral { 10.99.0.1; };`; `firewall-cmd
+  --add-service=dns --add-service=dns-over-tls`, or `ufw allow`.
@@ -0,0 +1,49 @@
+KEYF=$(pick /etc/named/lab-key.conf /etc/bind/lab-key.conf)
+A=10.99.0.1
+# pdig ARGS: dig from the peer, at this VM.
+pdig() { in_peer dig @$A +time=3 +tries=1 "$@" 2>/dev/null; }
+# signed ZONE NAME TYPE: the answer comes with an RRSIG.
+signed() { pdig "$2" "$3" +dnssec +norec | grep -Eq "^$2\.?[[:space:]].*IN[[:space:]]+RRSIG[[:space:]]+$3 "; }
+dnskeys() { pdig "$1" DNSKEY +norec +short | awk '{print $1}' | sort -u | tr '\n' ' '; }
+validated() { in_peer timeout 20 delv @$A -a /root/dns/lab-anchors.conf +root="$1" "www.$1" 2>&1 | grep -q "; fully validated"; }
+
+task 1 "Authoritative-only BIND serving lab.test, signed with NSEC3"
+need "BIND is running and enabled" 'active named && enabled named'
+need "the peer gets an authoritative answer for lab.test SOA from 10.99.0.1" 'for i in $(seq 15); do pdig lab.test SOA +norec | grep -q "flags:.* aa" && exit 0; sleep 1; done; exit 1'
+need "it answers on 127.0.0.1 too" 'dig @127.0.0.1 +time=3 +tries=1 lab.test SOA +norec | grep -q "flags:.* aa"'
+need "recursion is off (no ra flag)" '! pdig www.lab.test A | grep -q "flags:[^;]* ra[ ;]"'
+need "www.lab.test and ns1.lab.test are 10.99.0.1" '[ "$(pdig www.lab.test A +short)" = 10.99.0.1 ] && [ "$(pdig ns1.lab.test A +short)" = 10.99.0.1 ]'
+need "the NS record is ns1.lab.test." '[ "$(pdig lab.test NS +short)" = ns1.lab.test. ]'
+need "the three CAA records are there" 'c=$(pdig lab.test CAA +short); grep -q "0 issue \"letsencrypt.org\"" <<<"$c" && grep -q "0 issuewild \";\"" <<<"$c" && grep -q "0 iodef \"mailto:[email protected]\"" <<<"$c"'
+need "lab.test has a KSK (257) and a ZSK (256)" 'for i in $(seq 10); do [ "$(dnskeys lab.test)" = "256 257 " ] && exit 0; sleep 1; done; exit 1'
+need "www.lab.test comes with an RRSIG" 'signed lab.test www.lab.test A'
+need "a missing name is denied with NSEC3" 'pdig nothere.lab.test A +dnssec +norec | grep -q "NXDOMAIN" && pdig nothere.lab.test A +dnssec +norec | grep -Eq "IN[[:space:]]+NSEC3[[:space:]]"'
+
+task 2 "manual.test signed by hand, with NSEC"
+need "the server answers for manual.test" 'pdig manual.test SOA +norec | grep -q "flags:.* aa"'
+need "www.manual.test is 10.99.0.1" '[ "$(pdig www.manual.test A +short)" = 10.99.0.1 ]'
+need "manual.test has a KSK (257) and a ZSK (256)" '[ "$(dnskeys manual.test)" = "256 257 " ]'
+need "www.manual.test comes with an RRSIG" 'signed manual.test www.manual.test A'
+need "a missing name is denied with NSEC" 'pdig nothere.manual.test A +dnssec +norec | grep -Eq "IN[[:space:]]+NSEC[[:space:]]"'
+
+task 3 "TSIG key lab-key for transfers and updates; mail.lab.test added"
+need "the key file defines lab-key" 'grep -Eq "key \"?lab-key\"?" $KEYF && grep -q hmac-sha256 $KEYF'
+need "a transfer of lab.test without the key is refused" '! pdig lab.test AXFR | grep -Eq "IN[[:space:]]+SOA"'
+need "a transfer with the key works" 'pdig -k $KEYF lab.test AXFR | grep -Eq "IN[[:space:]]+RRSIG"'
+need "mail.lab.test is 10.99.0.3, signed" '[ "$(pdig mail.lab.test A +short)" = 10.99.0.3 ] && signed lab.test mail.lab.test A'
+need "updates without the key are refused" 'conf=$(named-checkconf -p 2>/dev/null); ! grep -Eq "allow-update[[:space:]]*\{[^}]*(any|10\.|127\.|localhost|localnets)" <<<"$conf"'
+
+task 4 "Trust anchors validate both zones from the peer"
+need "/root/dns/lab-anchors.conf has static-ds entries for both zones" 'grep -Eq "lab\.test\.?[[:space:]]+static-ds" /root/dns/lab-anchors.conf && grep -Eq "manual\.test\.?[[:space:]]+static-ds" /root/dns/lab-anchors.conf'
+need "delv fully validates www.lab.test" 'validated lab.test'
+need "delv fully validates www.manual.test" 'validated manual.test'
+
+task 5 "DANE: TLSA 3 1 1 for the service on port 4433"
+need "/root/dns/www.crt and www.key are a pair" '[ -n "$(openssl x509 -in /root/dns/www.crt -noout -pubkey 2>/dev/null)" ] && [ "$(openssl x509 -in /root/dns/www.crt -noout -pubkey)" = "$(openssl pkey -in /root/dns/www.key -pubout 2>/dev/null)" ]'
+need "_4433._tcp.www.lab.test has a signed 3 1 1 TLSA record" 'pdig _4433._tcp.www.lab.test TLSA +short | grep -q "^3 1 1 " && signed lab.test _4433._tcp.www.lab.test TLSA'
+need "the TLSA record matches www.crt" 'h=$(openssl x509 -in /root/dns/www.crt -noout -pubkey | openssl pkey -pubin -outform DER | sha256sum | cut -d" " -f1); pdig _4433._tcp.www.lab.test TLSA +short | tr -d " " | tr A-F a-f | grep -qx "311$h"'
+need "the peer reaches a TLS service on 10.99.0.1:4433" 'for i in $(seq 10); do in_peer timeout 5 openssl s_client -connect $A:4433 </dev/null 2>/dev/null | grep -q "BEGIN CERTIFICATE" && exit 0; sleep 1; done; exit 1'
+need "s_client verifies it with DANE" 't=$(pdig _4433._tcp.www.lab.test TLSA +short | awk "{ d = \"\"; for (i = 4; i <= NF; i++) d = d \$i; print \$1, \$2, \$3, d }"); in_peer timeout 5 openssl s_client -connect $A:4433 -dane_tlsa_domain www.lab.test -dane_tlsa_rrdata "$t" </dev/null 2>&1 | grep -q "Verification: OK"'
+
+task 6 "DNS over TLS on 10.99.0.1:853"
+need "dig +tls from the peer gets www.lab.test" 'for i in $(seq 10); do [ "$(pdig +tls www.lab.test A +short)" = 10.99.0.1 ] && exit 0; sleep 1; done; exit 1'
@@ -0,0 +1,3 @@
+TITLE="331.4 DNS and cryptography"
+CHAPTER=/lpic-3-303/331-cryptography/331-4-dns-and-cryptography/
+SYSTEMS="rhel debian"
@@ -0,0 +1,29 @@
+pkill -f '^openssl s_server .*4433' 2>/dev/null
+systemctl disable --now --quiet named 2>/dev/null
+# The packages installed since setup started, and only those: not the
+# dependencies they leave unneeded that were there before.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+if [ -s "$LAB_STATE/pkgs-before" ]; then
+	new=$(pkgs_now | comm -13 "$LAB_STATE/pkgs-before" - | grep -Ev '^(kernel|linux-)')
+	if [ -n "$new" ]; then
+		say "Removing the packages installed since the lab started..."
+		if is_rhel; then
+			dnf -y -q remove --setopt=clean_requirements_on_remove=False $new >/dev/null 2>&1
+		else
+			DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge $new >/dev/null 2>&1
+		fi
+	fi
+fi
+rm -rf $(pick "/etc/named.conf /etc/named /var/named" "/etc/bind /var/lib/bind /var/cache/bind")
+[ -f "$LAB_STATE/bind.tar" ] && tar -C / -xf "$LAB_STATE/bind.tar"
+is_rhel && [ -e /var/named ] && restorecon -R /etc/named.conf /etc/named /var/named 2>/dev/null
+[ -n "$(state_get named-enabled)" ] && systemctl enable --now --quiet named 2>/dev/null
+if [ -f "$LAB_STATE/firewalld.tar" ]; then
+	rm -rf /etc/firewalld && tar -C /etc -xf "$LAB_STATE/firewalld.tar"
+	firewall-cmd -q --reload
+elif [ -f "$LAB_STATE/ufw.tar" ]; then
+	rm -rf /etc/ufw && tar -C /etc -xf "$LAB_STATE/ufw.tar"
+	ufw reload >/dev/null
+fi
+rm -rf /root/dns
+true
@@ -0,0 +1,20 @@
+listening 4433 tcp && refuse "something already listens on port 4433; stop it and start again"
+[ -e /root/dns ] && refuse "/root/dns already exists; move it away and start again"
+command -v openssl >/dev/null || pkg_install openssl
+# Reset removes every package installed from here on, by setup or by the
+# tasks, and only those.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+pkgs_now >"$LAB_STATE/pkgs-before"
+# BIND's and the firewall's configuration, for reset to put back.
+for d in $(pick "etc/named.conf etc/named var/named" "etc/bind var/lib/bind var/cache/bind"); do
+	[ -e /$d ] && printf '%s\n' "$d" >>"$LAB_STATE/saved"
+done
+[ -s "$LAB_STATE/saved" ] && tar -C / -cf "$LAB_STATE/bind.tar" $(cat "$LAB_STATE/saved")
+enabled "$(pick named named)" && state_set named-enabled 1
+if systemctl is-active --quiet firewalld; then
+	tar -C /etc -cf "$LAB_STATE/firewalld.tar" firewalld
+elif command -v ufw >/dev/null && ufw status | grep -q 'Status: active'; then
+	tar -C /etc -cf "$LAB_STATE/ufw.tar" ufw
+fi
+peer_up
+true
@@ -0,0 +1,59 @@
+# LPIC-3 332.1: Host hardening
+
+File capabilities, USBGuard, an SSH certificate authority for host and
+user certificates with a revocation list, and a small chroot jail.
+
+## Before you start
+
+The lab creates the user `certuser` (password `Lab-Pass-1`), with no
+`authorized_keys`, for tasks 4 and 5. Keep the SSH CA and the user key in
+`/root/sshca`, with the file names the tasks give. You change sshd's
+configuration in tasks 3 to 5: keep a second session open while you do,
+and check with `sshd -t` before you reload. Your own logins don't change.
+
+The chapter's GRUB password, polkit, ASLR, systemd sandboxing and CPU
+vulnerability steps are for looking around; try them, but they aren't
+graded here.
+
+## Tasks
+
+1. Copy the Python interpreter (the real file, not a link) to
+   `/usr/local/bin/lab-python`, and give the copy the file capability that
+   lets an ordinary user bind ports below 1024 with it. Check that
+   `python3` still can't.
+2. Install USBGuard, generate a policy that allows the devices attached
+   now as `/etc/usbguard/rules.conf` (mode 600), and run the daemon,
+   enabled at boot. Try blocking and allowing a device again.
+3. Make an SSH CA, `/root/sshca/ca` (with `ca.pub`), and give sshd a host
+   certificate for its ED25519 host key, signed by the CA for the name
+   `localhost`, so that a client that trusts only the CA
+   (`@cert-authority localhost ...` in its known-hosts file) connects
+   without a prompt.
+4. Make sshd trust the CA for user certificates. Make a key pair
+   `/root/sshca/user` for `certuser`, sign a one-hour certificate for it
+   (`user-cert.pub`, principal `certuser`), and log in as `certuser` with
+   it.
+5. Revoke `/root/sshca/user.pub` with a key revocation list,
+   `/etc/ssh/revoked.krl`, that sshd uses, so that login is refused. Other
+   certificates from the CA must still work.
+6. Build a chroot jail in `/srv/jail` with `/bin/bash` and `/bin/ls` and
+   the libraries they need, so that `chroot /srv/jail /bin/bash -c 'ls /bin'`
+   works.
+
+## Hints
+
+- `readlink -f "$(command -v python3)"`; `setcap cap_net_bind_service=ep`,
+  `getcap`. Test as an ordinary user: `sudo -u nobody /usr/local/bin/lab-python
+  -c 'import socket; socket.socket().bind(("127.0.0.1", 80))'`. `capsh
+  --decode`, `/proc/self/status`.
+- `usbguard generate-policy`, `systemctl enable --now usbguard`,
+  `usbguard list-devices`, `usbguard block-device`, `usbguard
+  allow-device`; each device's `authorized` file under `/sys/bus/usb/devices`.
+- `ssh-keygen -t ed25519 -f ca`; `ssh-keygen -s ca -I ID -h -n localhost
+  -V +52w /etc/ssh/ssh_host_ed25519_key.pub`; `HostCertificate` in a file in
+  `/etc/ssh/sshd_config.d/`; `ssh-keyscan -c localhost`.
+- `TrustedUserCAKeys`; `ssh-keygen -s ca -I ID -n certuser -V +1h user.pub`,
+  `ssh-keygen -L -f user-cert.pub`; `ssh -i /root/sshca/user certuser@localhost`.
+- `ssh-keygen -k -f revoked.krl user.pub`, `ssh-keygen -Q -f revoked.krl
+  user.pub`; `RevokedKeys`.
+- `cp --parents`, `ldd`, `cp -L`.
@@ -0,0 +1,46 @@
+C=/root/sshca
+SSHOPT=(-o BatchMode=yes -o ConnectTimeout=5 -o LogLevel=ERROR -o IdentitiesOnly=yes)
+# login KEY [options]: certuser logs in with KEY (and its certificate).
+login() { local k=$1; shift; timeout 15 ssh "${SSHOPT[@]}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$k" "$@" certuser@localhost 'echo le-shell-ok' </dev/null 2>/dev/null | grep -q le-shell-ok; }
+T=$(mktemp -d)
+# A fresh key and certificate from the CA, as anyone trusting it could get.
+fresh_cert() { [ -f $C/ca ] && ssh-keygen -q -t ed25519 -N '' -f $T/fresh && ssh-keygen -q -s $C/ca -I le-check -n certuser -V +5m $T/fresh.pub; }
+is_debian && mkdir -p /run/sshd
+
+task 1 "/usr/local/bin/lab-python can bind low ports for anyone"
+need "/usr/local/bin/lab-python is a copy of Python, not a link" '[ -f /usr/local/bin/lab-python ] && [ ! -L /usr/local/bin/lab-python ] && /usr/local/bin/lab-python -c "import sys"'
+need "it has cap_net_bind_service in its permitted and effective sets" 'getcap /usr/local/bin/lab-python | grep -Eq "cap_net_bind_service[=+]ep"'
+need "nobody can bind port 999 with it" 'runuser -u nobody -- /usr/local/bin/lab-python -c "import socket; socket.socket().bind((\"127.0.0.1\", 999))"'
+need "nobody still can't bind port 999 with python3" '! runuser -u nobody -- python3 -c "import socket; socket.socket().bind((\"127.0.0.1\", 999))"'
+
+task 2 "USBGuard running with a policy for the attached devices"
+need "usbguard is running and enabled" 'active usbguard && enabled usbguard'
+need "/etc/usbguard/rules.conf exists, mode 600" '[ "$(stat -c %a /etc/usbguard/rules.conf)" = 600 ]'
+need "every attached device is allowed" 'usbguard list-devices >$T/devs && ! grep -Eq ": (block|reject) " $T/devs'
+need "the policy has a rule for each attached device" '[ "$(grep -c "^allow" /etc/usbguard/rules.conf)" -ge "$(grep -c . $T/devs)" ]'
+
+task 3 "sshd has a host certificate from the CA for localhost"
+need "$C/ca and ca.pub are an ED25519 key pair" '[ "$(ssh-keygen -y -f $C/ca 2>/dev/null | cut -d" " -f2)" = "$(cut -d" " -f2 $C/ca.pub)" ] && grep -q "^ssh-ed25519 " $C/ca.pub'
+need "sshd offers a host certificate" 'timeout 10 ssh-keyscan -c -t ed25519 localhost 2>/dev/null | grep -o "[email protected] .*" >$T/hostcert.pub && [ -s $T/hostcert.pub ]'
+need "it's signed by the CA" 'ssh-keygen -L -f $T/hostcert.pub | grep -q "Signing CA: .*$(ssh-keygen -lf $C/ca.pub | cut -d" " -f2)"'
+need "it's a host certificate for localhost" 'ssh-keygen -L -f $T/hostcert.pub >$T/l && grep -q "Type: .*host certificate" $T/l && sed -n "/Principals:/,/Critical/p" $T/l | grep -qx "[[:space:]]*localhost"'
+need "it's valid now" 'v=$(ssh-keygen -L -f $T/hostcert.pub | grep "Valid:"); grep -q forever <<<"$v" || { e=$(sed -E "s/.* to //" <<<"$v"); [ "$(date -d "$e" +%s)" -gt "$(date +%s)" ]; }'
+
+task 4 "sshd trusts the CA for user certificates; certuser logs in with one"
+need "/root/sshca/user and user-cert.pub exist" '[ -f $C/user ] && [ -f $C/user-cert.pub ]'
+need "user-cert.pub is signed by the CA for principal certuser" 'ssh-keygen -L -f $C/user-cert.pub >$T/u && grep -q "Signing CA: .*$(ssh-keygen -lf $C/ca.pub | cut -d" " -f2)" $T/u && sed -n "/Principals:/,/Critical/p" $T/u | grep -qx "[[:space:]]*certuser"'
+need "certuser has no authorized_keys" '[ ! -s /home/certuser/.ssh/authorized_keys ]'
+need "sshd -T shows trustedusercakeys" 'sshd -T 2>/dev/null | grep -q "^trustedusercakeys /"'
+need "certuser logs in with a new certificate from the CA, checking the host by its certificate" 'fresh_cert && echo "@cert-authority localhost $(cat $C/ca.pub)" >$T/kh && timeout 15 ssh "${SSHOPT[@]}" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$T/kh -i $T/fresh certuser@localhost "echo le-shell-ok" </dev/null 2>/dev/null | grep -q le-shell-ok'
+
+task 5 "user.pub revoked in /etc/ssh/revoked.krl"
+need "sshd -T shows revokedkeys /etc/ssh/revoked.krl" 'sshd -T 2>/dev/null | grep -qx "revokedkeys /etc/ssh/revoked.krl"'
+need "the KRL lists user.pub as revoked" 'ssh-keygen -Q -f /etc/ssh/revoked.krl $C/user.pub | grep -q REVOKED'
+need "certuser can't log in with /root/sshca/user" '[ -f $C/user ] && ! login $C/user'
+need "other certificates from the CA still work" '[ -f $T/fresh-cert.pub ] || fresh_cert; login $T/fresh'
+
+task 6 "A chroot jail in /srv/jail with bash and ls"
+need "/srv/jail/bin/bash and /srv/jail/bin/ls exist" '[ -x /srv/jail/bin/bash ] && [ -x /srv/jail/bin/ls ]'
+need "chroot /srv/jail /bin/bash -c \"ls /bin\" lists them" 'chroot /srv/jail /bin/bash -c "ls /bin" >$T/ls && grep -qx bash $T/ls && grep -qx ls $T/ls'
+need "inside, cd /.. stays at /" '[ "$(chroot /srv/jail /bin/bash -c "cd /../..; pwd")" = / ]'
+rm -rf "$T"
@@ -0,0 +1,3 @@
+TITLE="332.1 Host hardening"
+CHAPTER=/lpic-3-303/332-host-security/332-1-host-hardening/
+SYSTEMS="rhel debian"
@@ -0,0 +1,33 @@
+if [ -f "$LAB_STATE/ssh.tar" ]; then
+	rm -f /etc/ssh/sshd_config.d/* /etc/ssh/revoked.krl /etc/ssh/*-cert.pub
+	tar -C /etc -xf "$LAB_STATE/ssh.tar"
+	if is_rhel; then
+		systemctl restart sshd
+	else
+		systemctl restart ssh.socket 2>/dev/null
+		systemctl restart ssh.service 2>/dev/null
+	fi
+fi
+rm -rf /root/sshca /srv/jail /usr/local/bin/lab-python
+pkg_installed usbguard && [ ! -f "$LAB_STATE/usbguard.tar" ] && systemctl disable --now --quiet usbguard 2>/dev/null
+# The packages installed since setup started, and only those: not the
+# dependencies they leave unneeded that were there before.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+if [ -s "$LAB_STATE/pkgs-before" ]; then
+	new=$(pkgs_now | comm -13 "$LAB_STATE/pkgs-before" - | grep -Ev '^(kernel|linux-)')
+	if [ -n "$new" ]; then
+		say "Removing the packages installed since the lab started..."
+		if is_rhel; then
+			dnf -y -q remove --setopt=clean_requirements_on_remove=False $new >/dev/null 2>&1
+		else
+			DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge $new >/dev/null 2>&1
+		fi
+	fi
+fi
+if ! pkg_installed usbguard; then
+	rm -rf /etc/usbguard /var/log/usbguard
+elif [ -f "$LAB_STATE/usbguard.tar" ]; then
+	rm -rf /etc/usbguard && tar -C /etc -xf "$LAB_STATE/usbguard.tar"
+	if [ -n "$(state_get usbguard-enabled)" ]; then systemctl restart usbguard; else systemctl disable --now --quiet usbguard; fi
+fi
+true
@@ -0,0 +1,15 @@
+for p in /srv/jail /root/sshca /usr/local/bin/lab-python /etc/ssh/revoked.krl; do
+	[ -e $p ] && refuse "$p already exists; move it away and start again"
+done
+# Reset removes every package installed from here on, by setup or by the
+# tasks, and only those.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+pkgs_now >"$LAB_STATE/pkgs-before"
+user_add certuser
+# Saved for reset: sshd's configuration, USBGuard's.
+tar -C /etc -cf "$LAB_STATE/ssh.tar" ssh
+if pkg_installed usbguard; then
+	[ -d /etc/usbguard ] && tar -C /etc -cf "$LAB_STATE/usbguard.tar" usbguard
+	enabled usbguard && state_set usbguard-enabled 1
+fi
+true
@@ -0,0 +1,58 @@
+# LPIC-3 332.2: Host intrusion detection
+
+Audit rules and searches, package verification, AIDE with a scheduled
+check, OpenSCAP, and rootkit scanners.
+
+## Before you start
+
+The lab has changed one configuration file that belongs to one of the
+packages `logrotate`, `sudo`, or the OpenSSH client (`openssh-clients` on
+the RHEL family, `openssh-client` on Ubuntu), for you to find in task 2.
+Write answers to the files in `/root/lab-answers/`.
+
+Do task 1's test as your own login user (not root), from your SSH or
+console session, so that the audit system knows who you are.
+
+## Tasks
+
+1. Install the audit tools if they're missing (`audit-rules` on the RHEL
+   family, `auditd` on Ubuntu). Add two audit rules, permanently, with a
+   file in `/etc/audit/rules.d/` and `augenrules`: a watch on `/etc/hosts`
+   for writes and attribute changes with the key `hosts-change`, and a
+   rule, key `access-denied`, for `openat` calls that fail with `EACCES` by
+   users with a login UID of 1000 or more. Then change `/etc/hosts` (and
+   put it back), try to read `/etc/shadow` as your login user, and find
+   both events with `ausearch`.
+2. Find the changed file with your package manager's verify function, and
+   write its full path to `/root/lab-answers/2`.
+3. Install AIDE and write your own configuration, `/root/aide/lab.conf`,
+   that reads the database from `/root/aide/aide.db`, writes new ones to
+   `/root/aide/aide.db.new`, uncompressed, and covers `/etc/ssh` and
+   `/usr/sbin` with permissions, owner, group, size, mtime and SHA-256.
+   Build the database as `/root/aide/aide.db`. Plant a file in
+   `/usr/sbin`, change a file in `/etc/ssh`, see AIDE report both, and put
+   things back.
+4. Schedule that check nightly at 3:30 with a file in `/etc/cron.d`,
+   mailing its output to root.
+5. RHEL family: evaluate the CIS Level 1 server profile's rules
+   `package_aide_installed` and `sshd_disable_root_login` with `oscap`,
+   saving the results to `/root/oscap-results.xml`.
+6. Ubuntu: install `chkrootkit`, `rkhunter` and `debsums`, record
+   rkhunter's baseline of file properties, and run both scanners.
+
+## Hints
+
+- `auditctl -w PATH -p wa -k KEY`; `-a always,exit -F arch=b64 -S openat
+  -F exit=-EACCES -F 'auid>=1000' -F auid!=unset -k KEY`; `augenrules
+  --load`, `auditctl -l`; `ausearch -k KEY -i`, `-f FILE`, `aureport -k
+  --summary`.
+- `rpm -Va` or `rpm -V PACKAGE`; `dpkg --verify`, `debsums -as`.
+- `database_in=file:...`, `database_out=file:...`, `gzip_dbout=no`, a
+  rule line such as `LabRule = p+u+g+s+m+sha256`, then `/etc/ssh LabRule`;
+  `aide --init -c FILE`, `aide --check -c FILE`.
+- `MAILTO=root`, then `minute hour day month weekday user command`.
+- `/usr/share/xml/scap/ssg/content/` (pick the data stream for your
+  distribution); `oscap info --profiles`; `oscap xccdf eval --profile ...
+  --rule xccdf_org.ssgproject.content_rule_NAME --results FILE`.
+- `apt-get install --no-install-recommends`; `rkhunter --propupd`,
+  `rkhunter --check --sk`, `chkrootkit -q`; `/var/log/rkhunter.log`.
@@ -0,0 +1,48 @@
+AR=$(auditctl -l 2>/dev/null)
+# ev KEY [ausearch options]: a syscall event with this key is logged.
+ev() { local k=$1; shift; ausearch --input-logs -m SYSCALL -k "$k" "$@" </dev/null 2>/dev/null | grep -q "type=SYSCALL"; }
+# rule_result RULE: the result oscap saved for a rule.
+rule_result() {
+	python3 -c '
+import sys, xml.etree.ElementTree as ET
+for e in ET.parse("/root/oscap-results.xml").iter():
+    if e.tag.endswith("}rule-result") and e.get("idref", "").endswith("_rule_" + sys.argv[1]):
+        for r in e:
+            if r.tag.endswith("}result"): print(r.text)' "$1" 2>/dev/null
+}
+
+task 1 "Permanent audit rules hosts-change and access-denied; events found"
+need "auditd is running and enabled" 'active auditd && enabled auditd'
+need "the watch on /etc/hosts (wa, key hosts-change) is loaded" 'grep -Eq "^-w /etc/hosts -p (wa|aw) -k hosts-change$" <<<"$AR"'
+need "the access-denied rule is loaded" 'grep -- "-S openat" <<<"$AR" | grep -- "exit=-EACCES" | grep -- "auid>=1000" | grep -q "key=access-denied"'
+need "both rules are in /etc/audit/rules.d" 'cat /etc/audit/rules.d/*.rules 2>/dev/null >/tmp/le-rules; grep -Eq "^-w /etc/hosts .*hosts-change" /tmp/le-rules && grep -q "access-denied" /tmp/le-rules'
+need "a change to /etc/hosts was logged" 'ev hosts-change -f /etc/hosts'
+need "a login user's denied read of /etc/shadow was logged" 'ev access-denied -f /etc/shadow'
+rm -f /tmp/le-rules
+
+task 2 "The changed packaged file, in /root/lab-answers/2"
+need "/root/lab-answers/2 names the file the lab changed" '[ "$(answer 2)" = "$(state_get changed)" ]'
+
+task 3 "AIDE database /root/aide/aide.db for /etc/ssh and /usr/sbin"
+need "/root/aide/lab.conf passes aide --config-check" 'aide --config-check -c /root/aide/lab.conf'
+need "it reads /root/aide/aide.db and writes /root/aide/aide.db.new" 'grep -Eq "^[[:space:]]*database_in[[:space:]]*=[[:space:]]*file:/root/aide/aide\.db[[:space:]]*$" /root/aide/lab.conf && grep -Eq "^[[:space:]]*database_out[[:space:]]*=[[:space:]]*file:/root/aide/aide\.db\.new[[:space:]]*$" /root/aide/lab.conf'
+need "/root/aide/aide.db is an uncompressed AIDE database" 'head -c 200 /root/aide/aide.db | grep -q "^@@begin_db"'
+need "it has SHA-256 sums" 'grep -m1 "^@@db_spec" /root/aide/aide.db | grep -q sha256'
+need "it covers /etc/ssh and /usr/sbin" 'grep -q "^/etc/ssh/ssh_config " /root/aide/aide.db && grep -q "^/usr/sbin/" /root/aide/aide.db'
+need "it doesn't cover the rest of the system" '! grep -Eq "^/(bin|etc/passwd|var|home)[/ ]" /root/aide/aide.db'
+need "aide --check -c /root/aide/lab.conf runs" 'aide --check -c /root/aide/lab.conf >/dev/null 2>&1; [ $? -lt 14 ]'
+need "/usr/sbin has no file the lab check would call planted" '[ -z "$(find /usr/sbin -newer /root/aide/aide.db -type f 2>/dev/null)" ]'
+
+task 4 "Nightly AIDE check at 3:30 from /etc/cron.d, mailed to root"
+need "a file in /etc/cron.d runs aide --check -c /root/aide/lab.conf at 3:30 as root" 'grep -hEv "^[[:space:]]*#" /etc/cron.d/* 2>/dev/null | grep -E "aide.*--check.*-c[[:space:]]*/root/aide/lab\.conf|aide.*-c[[:space:]]*/root/aide/lab\.conf.*--check" | grep -Eq "^30[[:space:]]+0?3[[:space:]]+\*[[:space:]]+\*[[:space:]]+\*[[:space:]]+root[[:space:]]"'
+need "it mails its output to root" 'grep -l "aide" /etc/cron.d/* 2>/dev/null | xargs grep -Eqs "^[[:space:]]*MAILTO=\"?root\"?"'
+
+task 5 "OpenSCAP results for two CIS rules in /root/oscap-results.xml" rhel
+need "/root/oscap-results.xml holds XCCDF results" 'grep -q "TestResult" /root/oscap-results.xml'
+need "it has a result for package_aide_installed" 'rule_result package_aide_installed | grep -Eqx "pass|fail"'
+need "it has a result for sshd_disable_root_login" 'rule_result sshd_disable_root_login | grep -Eqx "pass|fail"'
+
+task 6 "chkrootkit, rkhunter (with a baseline) and debsums" debian
+need "chkrootkit, rkhunter and debsums are installed" 'pkg_installed chkrootkit && pkg_installed rkhunter && pkg_installed debsums'
+need "rkhunter has a baseline of file properties" 'grep -q "^File:" /var/lib/rkhunter/db/rkhunter.dat'
+need "rkhunter has run its checks" 'grep -qs "Rootkit checks" /var/log/rkhunter.log'
@@ -0,0 +1,3 @@
+TITLE="332.2 Host intrusion detection"
+CHAPTER=/lpic-3-303/332-host-security/332-2-host-intrusion-detection/
+SYSTEMS="rhel debian"
@@ -0,0 +1,27 @@
+# Task 2's file, as it was.
+f=$(state_get changed)
+[ -n "$f" ] && [ -f "$LAB_STATE/changed.orig" ] && cp -a "$LAB_STATE/changed.orig" "$f"
+# Audit rules files the lab didn't find.
+for r in /etc/audit/rules.d/*; do
+	[ -e "$r" ] || continue
+	grep -qxF "${r##*/}" "$LAB_STATE/rules-d" 2>/dev/null || rm -f "$r"
+done
+command -v augenrules >/dev/null && augenrules --load >/dev/null 2>&1
+for c in /etc/cron.d/*; do grep -qs /root/aide/lab.conf "$c" && rm -f "$c"; done
+rm -rf /root/aide /root/oscap-results.xml /tmp/le-rules
+# The packages installed since setup started, and only those: not the
+# dependencies they leave unneeded that were there before.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+if [ -s "$LAB_STATE/pkgs-before" ]; then
+	new=$(pkgs_now | comm -13 "$LAB_STATE/pkgs-before" - | grep -Ev '^(kernel|linux-)')
+	if [ -n "$new" ]; then
+		say "Removing the packages installed since the lab started..."
+		if is_rhel; then
+			dnf -y -q remove --setopt=clean_requirements_on_remove=False $new >/dev/null 2>&1
+		else
+			DEBIAN_FRONTEND=noninteractive apt-get -y -qq purge $new >/dev/null 2>&1
+		fi
+	fi
+fi
+grep -qx rkhunter <<<"${new:-}" && rm -rf /var/lib/rkhunter /var/log/rkhunter.log*
+true
@@ -0,0 +1,18 @@
+for p in /root/aide /root/oscap-results.xml; do
+	[ -e $p ] && refuse "$p already exists; move it away and start again"
+done
+ls /etc/cron.d/ 2>/dev/null | grep -q aide && refuse "/etc/cron.d already has an AIDE job; remove it and start again"
+# Reset removes every package installed from here on, by setup or by the
+# tasks, and only those.
+pkgs_now() { if is_rhel; then rpm -qa --qf '%{NAME}\n'; else dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 == "ii" { print $2 }'; fi | sort -u; }
+pkgs_now >"$LAB_STATE/pkgs-before"
+ls /etc/audit/rules.d >"$LAB_STATE/rules-d" 2>/dev/null || true
+
+# The changed file for task 2.
+files=(/etc/logrotate.conf /etc/sudo.conf /etc/ssh/ssh_config)
+f=${files[RANDOM % 3]}
+[ -f "$f" ] || f=/etc/logrotate.conf
+cp -a "$f" "$LAB_STATE/changed.orig"
+state_set changed "$f"
+printf '# changed for the practice lab\n' >>"$f"
+true