Add AppArmor Support

docs/architecture.md

@@ -140,6 +140,7 @@ In-place patch (`cuos patch*`) modifies `/system.json` directly and keeps previo
 | Network | Minimal exposed ports, host network only if required |
 | Privileges | App container without `--privileged`; granular capabilities |
 | CA | Inject custom CA via `custom_ca_certs` & rotate regularly |
+| Confinement | AppArmor is on, and dockerd confines the app container with `docker-default` — `docs/common/apparmor.md` |
 
 ---
 
@@ -150,3 +151,4 @@ In-place patch (`cuos patch*`) modifies `/system.json` directly and keeps previo
 * `image-factory/create_image.sh` – Raw image creation
 * `installer-factory/create_iso.sh` – Installer ISO
 * `docs/common/btrfs-usage.md` – Subvolume layout
+* `docs/common/apparmor.md` – What AppArmor confines, and what it does not

docs/common/apparmor.md

@@ -0,0 +1,84 @@
+# AppArmor
+
+CuOS boots with AppArmor enabled (`apparmor=1` on the kernel command line) on
+every platform except LXC. `cuos selftest` reports the state under the
+`apparmor` check.
+
+## What is confined
+
+| | Profile | Where it comes from |
+|---|---|---|
+| The application container | `docker-default` | dockerd's built-in template, loaded when the daemon starts |
+| `dhclient` | `/usr/sbin/dhclient` | Debian's `isc-dhcp-client` package |
+| PAM's password check | `unix-chkpwd` | Debian's `apparmor` package |
+
+That is the whole list, and `docker-default` is the point of the exercise: it is
+what stands between the application container and the host. Debian's
+`apparmor-profiles` and `apparmor-profiles-extra` packages are not installed —
+they cover samba, syslog, avahi and desktop software, none of which is here.
+
+## What is not confined
+
+- **dockerd, and the CuOS scripts.** A daemon that must be able to do anything
+  cannot usefully be confined, and the CuOS scripts run as root and start
+  privileged containers.
+- **The updater.** `do-update.sh` runs it with `--privileged`, and Docker runs
+  privileged containers unconfined. It repartitions the root disk, so it needs
+  to be. It is pulled by digest, and the digest is checked before it runs.
+- **sshd.** Debian ships no profile for it, and it is masked unless
+  `os_ssh_server` is set.
+- **The LXC platform.** Profiles are loaded into the host's kernel and a
+  container cannot load its own, so the LXC image carries no AppArmor at all.
+  Confining a CuOS guest is the LXC host's business.
+- **The installer ISO.** A separate image without the AppArmor package.
+
+## Giving an application more than `docker-default`
+
+The flags of the application container come from the `dev.cuos.app_command`
+label on the app image (see `cuos-app-init.md`), so the profile is selected
+there:
+
+```
+LABEL dev.cuos.app_command="--security-opt apparmor=my-profile ..."
+```
+
+Note that the label **replaces** the default flag set rather than adding to it.
+
+The profile itself has to be loaded into the kernel before the container
+starts, and CuOS ships no mechanism for that yet. While an application is being
+brought up, `--security-opt apparmor=unconfined` is the escape hatch.
+
+## Reading a denial
+
+```
+journalctl -b | grep DENIED
+```
+
+Each line names the profile, the operation and the path. To take one profile
+out of the way without touching the rest:
+
+```
+aa-complain /etc/apparmor.d/usr.sbin.dhclient   # log, do not block
+aa-enforce  /etc/apparmor.d/usr.sbin.dhclient   # back to blocking
+```
+
+Both are lost on the next boot, which is what you want while diagnosing. A
+permanent addition goes into the profile's `local/` include —
+`/etc/apparmor.d/local/<profile>` — which survives package upgrades.
+
+## The `/data` trap
+
+CuOS keeps mutable state in `/data` and symlinks into it: `/var/log`,
+`/var/lib/dhcp`, `/var/lib/docker`, `/var/lib/containerd`, `/root`.
+
+**AppArmor mediates the resolved path, not the symlink.** A stock profile that
+allows `/var/lib/dhcp/dhclient*` therefore denies the write that actually lands
+on `/data/dhcp/dhclient.leases`. That is why the image ships
+
+```
+/data/dhcp/ r,
+/data/dhcp/dhclient* lrw,
+```
+
+in `/etc/apparmor.d/local/usr.sbin.dhclient`. Any profile adopted from a
+distribution package has to be read against these symlinks first.

installer-factory/Dockerfile

@@ -27,7 +27,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     linux-image-amd64 \
     mtools \
     pv \
-    selinux-policy-default \
     syslinux \
     xorriso \
     && apt-get clean && rm -rf /var/lib/apt/lists/*

installer-factory/prepare_iso.sh

@@ -20,6 +20,7 @@ mv "/boot/vmlinuz"-* "${ISO_DIR}/boot/vmlinuz"
 mv "${INITRD_IMAGE}" "${ISO_DIR}/boot/initrd.img"
 
 # Create grub config
+# apparmor=0, unlike the system image: this one has no apparmor package.
 cat <<EOF >"${ISO_DIR}/boot/grub/grub.cfg"
 set timeout=0
 set default=0

system/Dockerfile

@@ -48,6 +48,7 @@ RUN sed -i 's/^Components: main$/Components: main non-free-firmware/' /etc/apt/s
   ## cuos os extra for wake on lan \
   ethtool \
   ## cuos os dependencies \
+  apparmor \
   btrfs-progs \
   console-setup \
   dbus \
@@ -57,7 +58,6 @@ RUN sed -i 's/^Components: main$/Components: main non-free-firmware/' /etc/apt/s
   kbd \
   kmod \
   lsb-release \
-  selinux-policy-default \
   systemd \
   systemd-sysv \
   systemd-timesyncd \
@@ -89,6 +89,9 @@ RUN sed -i 's/^Components: main$/Components: main non-free-firmware/' /etc/apt/s
   && systemctl disable hv-kvp-daemon.service \
   && systemctl disable hv-vss-daemon.service \
   && systemctl enable systemd-timesyncd.service \
+  && systemctl enable apparmor.service \
+  ## /var/lib/dhcp is a symlink into /data, and AppArmor sees the resolved path \
+  && printf "/data/dhcp/ r,\n/data/dhcp/dhclient* lrw,\n" >>/etc/apparmor.d/local/usr.sbin.dhclient \
   && systemctl mask ssh.socket \
   && systemctl mask sshd-keygen.service \
   && printf "Port 4222\nPermitRootLogin yes\n" >/etc/ssh/sshd_config.d/cuos.conf \

system/Dockerfile.lxc

@@ -16,6 +16,8 @@ RUN echo "Cache buster: v1" \
   ## cuos system dependencies (lxc) \
   busybox-static \
   ## cuos os dependencies \
+  # A container cannot load profiles - that is the LXC host's business \
+  #-apparmor \
   #-btrfs-progs \
   dbus \
   dbus-broker \
@@ -24,7 +26,6 @@ RUN echo "Cache buster: v1" \
   #-kbd \
   kmod \
   lsb-release \
-  #-selinux-policy-default \
   systemd \
   systemd-sysv \
   #-systemd-timesyncd \

system/Dockerfile.orangepi-zero3

@@ -75,6 +75,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   # rpi-swap \
   # iw \
   ## cuos os dependencies \
+  apparmor \
   btrfs-progs \
   console-setup \
   dbus \
@@ -84,7 +85,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   kbd \
   kmod \
   lsb-release \
-  selinux-policy-default \
   systemd \
   systemd-sysv \
   systemd-timesyncd \
@@ -112,6 +112,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   vim-tiny \
   && apt-get clean && rm -rf /var/lib/apt/lists/* \
   && systemctl enable systemd-timesyncd.service \
+  && systemctl enable apparmor.service \
+  ## /var/lib/dhcp is a symlink into /data, and AppArmor sees the resolved path \
+  && printf "/data/dhcp/ r,\n/data/dhcp/dhclient* lrw,\n" >>/etc/apparmor.d/local/usr.sbin.dhclient \
   && systemctl mask ssh.socket \
   && systemctl mask sshd-keygen.service \
   && printf "Port 4222\nPermitRootLogin yes\n" >/etc/ssh/sshd_config.d/cuos.conf \

system/Dockerfile.rpi-arm32

@@ -114,6 +114,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   # rpi-swap \
   # iw \
   ## cuos os dependencies \
+  apparmor \
   btrfs-progs \
   console-setup \
   dbus \
@@ -123,7 +124,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   kbd \
   kmod \
   lsb-release \
-  selinux-policy-default \
   systemd \
   systemd-sysv \
   systemd-timesyncd \
@@ -151,6 +151,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   vim-tiny \
   && apt-get clean && rm -rf /var/lib/apt/lists/* \
   && systemctl enable systemd-timesyncd.service \
+  && systemctl enable apparmor.service \
+  ## /var/lib/dhcp is a symlink into /data, and AppArmor sees the resolved path \
+  && printf "/data/dhcp/ r,\n/data/dhcp/dhclient* lrw,\n" >>/etc/apparmor.d/local/usr.sbin.dhclient \
   && systemctl mask ssh.socket \
   && systemctl mask sshd-keygen.service \
   && printf "Port 4222\nPermitRootLogin yes\n" >/etc/ssh/sshd_config.d/cuos.conf \

system/Dockerfile.rpi-arm64

@@ -95,6 +95,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   # rpi-swap \
   # iw \
   ## cuos os dependencies \
+  apparmor \
   btrfs-progs \
   console-setup \
   dbus \
@@ -104,7 +105,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   kbd \
   kmod \
   lsb-release \
-  selinux-policy-default \
   systemd \
   systemd-sysv \
   systemd-timesyncd \
@@ -132,6 +132,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   vim-tiny \
   && apt-get clean && rm -rf /var/lib/apt/lists/* \
   && systemctl enable systemd-timesyncd.service \
+  && systemctl enable apparmor.service \
+  ## /var/lib/dhcp is a symlink into /data, and AppArmor sees the resolved path \
+  && printf "/data/dhcp/ r,\n/data/dhcp/dhclient* lrw,\n" >>/etc/apparmor.d/local/usr.sbin.dhclient \
   && systemctl mask ssh.socket \
   && systemctl mask sshd-keygen.service \
   && printf "Port 4222\nPermitRootLogin yes\n" >/etc/ssh/sshd_config.d/cuos.conf \

system/armbianEnv.txt

@@ -7,5 +7,5 @@ disp_mode=1920x1080p60
 overlay_prefix=sun50i-h616
 rootdev=LABEL=system
 rootfstype=btrfs
-extraargs=rootflags=subvol=@os/system-{{SLOT}} loglevel=3 noresume apparmor=0
+extraargs=rootflags=subvol=@os/system-{{SLOT}} loglevel=3 noresume apparmor=1
 usbstoragequirks=0x2537:0x1066:u,0x2537:0x1068:u

system/cuos/install-kernel-grub.sh

@@ -82,7 +82,7 @@ menuentry '${SLOT_NAME}' --unrestricted {
 
     search --no-floppy --label boot --set=root
 
-    linux /${filename_kernel} root=LABEL=system rootfstype=btrfs rootflags=subvol=${CONTAINER_ROOTFS} ro loglevel=3 noresume apparmor=0
+    linux /${filename_kernel} root=LABEL=system rootfstype=btrfs rootflags=subvol=${CONTAINER_ROOTFS} ro loglevel=3 noresume apparmor=1
     initrd /${filename_initrd}
 }
 

system/cuos/install-kernel-rpi.sh

@@ -38,7 +38,7 @@ fi
 mv "${TARGET_BOOT}"/cmdline.txt "${TARGET_BOOT}"/cmdline-previous.txt 2>/dev/null || true
 
 cat <<EOF > "${TARGET_BOOT}/cmdline.txt"
-console=serial0,115200 console=tty1 rootwait root=LABEL=system rootfstype=btrfs rootflags=subvol=${CONTAINER_ROOTFS} fsck.repair=yes ro loglevel=3 noresume apparmor=0
+console=serial0,115200 console=tty1 rootwait root=LABEL=system rootfstype=btrfs rootflags=subvol=${CONTAINER_ROOTFS} fsck.repair=yes ro loglevel=3 noresume apparmor=1
 EOF
 
 # Attach options from system.json

system/cuos/selftest.sh

@@ -80,6 +80,15 @@ listening_on() {
   ss -ltn 2>/dev/null | grep -qE "[:.]$1[[:space:]]"
 }
 
+apparmor_enabled() {
+  [[ "$(cat /sys/module/apparmor/parameters/enabled 2>/dev/null)" == "Y" ]]
+}
+
+# "<profile> (<mode>)" per line, for every profile loaded into the kernel.
+apparmor_profiles() {
+  cat /sys/kernel/security/apparmor/profiles 2>/dev/null
+}
+
 # How many cuos journal entries of priority error or worse this boot produced.
 journal_errors() {
   journalctl -t cuos -b -p 3 --no-pager -o cat 2>/dev/null | grep -c . || true
@@ -177,6 +186,35 @@ check_docker() {
   failed "docker" "docker is not responding"
 }
 
+# docker-default is the only thing confining the application container, and
+# both ways of losing it - the LSM off, or dockerd never loading the profile -
+# are silent. So it is asked for by name.
+check_apparmor() {
+  if is_container; then
+    skipped "apparmor" "profiles belong to the host's kernel; a container loads none of its own"
+    return
+  fi
+  if ! apparmor_enabled; then
+    failed "apparmor" "the kernel has AppArmor off - check apparmor=1 on the command line"
+    return
+  fi
+
+  local profiles count mode
+  profiles="$(apparmor_profiles)"
+  count="$(grep -c . <<<"${profiles}")"
+  mode="$(sed -n 's/^docker-default (\(.*\))$/\1/p' <<<"${profiles}")"
+
+  if [[ -z "${mode}" ]]; then
+    failed "apparmor" "dockerd loaded no docker-default; profiles in the kernel: ${count}"
+    return
+  fi
+  if [[ "${mode}" != "enforce" ]]; then
+    failed "apparmor" "docker-default is in ${mode} mode, so it confines nothing"
+    return
+  fi
+  ok "apparmor" "${count} profiles loaded, docker-default enforcing"
+}
+
 check_app() {
   local status
   status="$(container_state "${APP_CONTAINER}")"
@@ -278,6 +316,7 @@ selftest() {
   check_subvolumes
   check_state
   check_docker
+  check_apparmor
   check_app
   check_network
   check_hostname

system/cuos/selftest.test.sh

@@ -33,6 +33,10 @@ MOCK_DOCKER_OK=0
 MOCK_APP_STATE="running"
 MOCK_LISTENING=0
 MOCK_JOURNAL_ERRORS=0
+MOCK_APPARMOR_ENABLED=0
+MOCK_APPARMOR_PROFILES="docker-default (enforce)
+/usr/sbin/dhclient (enforce)
+unix-chkpwd (enforce)"
 
 virt_type() { echo "${MOCK_VIRT}"; }
 subvolumes() { echo "${MOCK_SUBVOLUMES}"; }
@@ -42,6 +46,8 @@ docker_ok() { return "${MOCK_DOCKER_OK}"; }
 container_state() { echo "${MOCK_APP_STATE}"; }
 listening_on() { return "${MOCK_LISTENING}"; }
 journal_errors() { echo "${MOCK_JOURNAL_ERRORS}"; }
+apparmor_enabled() { return "${MOCK_APPARMOR_ENABLED}"; }
+apparmor_profiles() { echo "${MOCK_APPARMOR_PROFILES}"; }
 
 # A healthy system, which each test then breaks in exactly one way.
 reset_system() {
@@ -56,6 +62,10 @@ reset_system() {
   MOCK_APP_STATE="running"
   MOCK_LISTENING=0
   MOCK_JOURNAL_ERRORS=0
+  MOCK_APPARMOR_ENABLED=0
+  MOCK_APPARMOR_PROFILES="docker-default (enforce)
+/usr/sbin/dhclient (enforce)
+unix-chkpwd (enforce)"
 
   cat >"${CONFIG_PATH}" <<'EOF'
 {
@@ -151,6 +161,36 @@ expect "state: anything but running fails" "failed" status_of state is_updating
 break_docker() { MOCK_DOCKER_OK=1; }
 expect "docker: an unresponsive daemon fails" "failed" status_of docker break_docker
 
+lsm_off() { MOCK_APPARMOR_ENABLED=1; }
+expect "apparmor: a kernel with the LSM off fails" "failed" status_of apparmor lsm_off
+expect "apparmor: the failure names the command line" \
+  "the kernel has AppArmor off - check apparmor=1 on the command line" \
+  detail_of apparmor lsm_off
+
+no_docker_default() { MOCK_APPARMOR_PROFILES="/usr/sbin/dhclient (enforce)"; }
+expect "apparmor: enabled but no docker-default fails" "failed" status_of \
+  apparmor no_docker_default
+expect "apparmor: the failure counts what did load" \
+  "dockerd loaded no docker-default; profiles in the kernel: 1" detail_of \
+  apparmor no_docker_default
+
+complaining() { MOCK_APPARMOR_PROFILES="docker-default (complain)"; }
+expect "apparmor: a complaining docker-default fails" "failed" status_of \
+  apparmor complaining
+expect "apparmor: and says it confines nothing" \
+  "docker-default is in complain mode, so it confines nothing" detail_of \
+  apparmor complaining
+
+# "docker-default-something" is a different profile, not this one.
+similar_name() { MOCK_APPARMOR_PROFILES="docker-default-old (enforce)"; }
+expect "apparmor: a profile merely starting with the name does not count" \
+  "failed" status_of apparmor similar_name
+
+expect "apparmor: a healthy system reports the count" \
+  "3 profiles loaded, docker-default enforcing" detail_of apparmor nothing
+expect "apparmor: skipped in a container" "skipped" status_of apparmor \
+  be_a_container
+
 app_exited() { MOCK_APP_STATE="exited"; }
 expect "app: a stopped application fails" "failed" status_of app app_exited
 app_missing() { MOCK_APP_STATE=""; }
@@ -193,7 +233,7 @@ expect "a skipped check does not" "true" overall be_a_container
 check_count() {
   report nothing | jq -r '.checks | length'
 }
-expect "every check reports" "11" check_count
+expect "every check reports" "12" check_count
 
 summary_adds_up() {
   report nothing | jq -r '