diff --git a/docs/architecture.md b/docs/architecture.md index 4dbd807..b9428ae 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -140,6 +140,7 @@ In-place patch (`cuos patch*`) modifies `/system.json` directly and keeps previo | Network | Minimal exposed ports, host network only if required | | Privileges | App container without `--privileged`; granular capabilities | | CA | Inject custom CA via `custom_ca_certs` & rotate regularly | +| Confinement | AppArmor is on, and dockerd confines the app container with `docker-default` — `docs/common/apparmor.md` | --- @@ -150,3 +151,4 @@ In-place patch (`cuos patch*`) modifies `/system.json` directly and keeps previo * `image-factory/create_image.sh` – Raw image creation * `installer-factory/create_iso.sh` – Installer ISO * `docs/common/btrfs-usage.md` – Subvolume layout +* `docs/common/apparmor.md` – What AppArmor confines, and what it does not diff --git a/docs/common/apparmor.md b/docs/common/apparmor.md new file mode 100644 index 0000000..c09a72a --- /dev/null +++ b/docs/common/apparmor.md @@ -0,0 +1,84 @@ +# AppArmor + +CuOS boots with AppArmor enabled (`apparmor=1` on the kernel command line) on +every platform except LXC. `cuos selftest` reports the state under the +`apparmor` check. + +## What is confined + +| | Profile | Where it comes from | +|---|---|---| +| The application container | `docker-default` | dockerd's built-in template, loaded when the daemon starts | +| `dhclient` | `/usr/sbin/dhclient` | Debian's `isc-dhcp-client` package | +| PAM's password check | `unix-chkpwd` | Debian's `apparmor` package | + +That is the whole list, and `docker-default` is the point of the exercise: it is +what stands between the application container and the host. Debian's +`apparmor-profiles` and `apparmor-profiles-extra` packages are not installed — +they cover samba, syslog, avahi and desktop software, none of which is here. + +## What is not confined + +- **dockerd, and the CuOS scripts.** A daemon that must be able to do anything + cannot usefully be confined, and the CuOS scripts run as root and start + privileged containers. +- **The updater.** `do-update.sh` runs it with `--privileged`, and Docker runs + privileged containers unconfined. It repartitions the root disk, so it needs + to be. It is pulled by digest, and the digest is checked before it runs. +- **sshd.** Debian ships no profile for it, and it is masked unless + `os_ssh_server` is set. +- **The LXC platform.** Profiles are loaded into the host's kernel and a + container cannot load its own, so the LXC image carries no AppArmor at all. + Confining a CuOS guest is the LXC host's business. +- **The installer ISO.** A separate image without the AppArmor package. + +## Giving an application more than `docker-default` + +The flags of the application container come from the `dev.cuos.app_command` +label on the app image (see `cuos-app-init.md`), so the profile is selected +there: + +``` +LABEL dev.cuos.app_command="--security-opt apparmor=my-profile ..." +``` + +Note that the label **replaces** the default flag set rather than adding to it. + +The profile itself has to be loaded into the kernel before the container +starts, and CuOS ships no mechanism for that yet. While an application is being +brought up, `--security-opt apparmor=unconfined` is the escape hatch. + +## Reading a denial + +``` +journalctl -b | grep DENIED +``` + +Each line names the profile, the operation and the path. To take one profile +out of the way without touching the rest: + +``` +aa-complain /etc/apparmor.d/usr.sbin.dhclient # log, do not block +aa-enforce /etc/apparmor.d/usr.sbin.dhclient # back to blocking +``` + +Both are lost on the next boot, which is what you want while diagnosing. A +permanent addition goes into the profile's `local/` include — +`/etc/apparmor.d/local/` — which survives package upgrades. + +## The `/data` trap + +CuOS keeps mutable state in `/data` and symlinks into it: `/var/log`, +`/var/lib/dhcp`, `/var/lib/docker`, `/var/lib/containerd`, `/root`. + +**AppArmor mediates the resolved path, not the symlink.** A stock profile that +allows `/var/lib/dhcp/dhclient*` therefore denies the write that actually lands +on `/data/dhcp/dhclient.leases`. That is why the image ships + +``` +/data/dhcp/ r, +/data/dhcp/dhclient* lrw, +``` + +in `/etc/apparmor.d/local/usr.sbin.dhclient`. Any profile adopted from a +distribution package has to be read against these symlinks first. diff --git a/installer-factory/Dockerfile b/installer-factory/Dockerfile index 4813f5d..8aadbac 100644 --- a/installer-factory/Dockerfile +++ b/installer-factory/Dockerfile @@ -27,7 +27,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ linux-image-amd64 \ mtools \ pv \ - selinux-policy-default \ syslinux \ xorriso \ && apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/installer-factory/prepare_iso.sh b/installer-factory/prepare_iso.sh index f922dd9..76bbe71 100755 --- a/installer-factory/prepare_iso.sh +++ b/installer-factory/prepare_iso.sh @@ -20,6 +20,7 @@ mv "/boot/vmlinuz"-* "${ISO_DIR}/boot/vmlinuz" mv "${INITRD_IMAGE}" "${ISO_DIR}/boot/initrd.img" # Create grub config +# apparmor=0, unlike the system image: this one has no apparmor package. cat <"${ISO_DIR}/boot/grub/grub.cfg" set timeout=0 set default=0 diff --git a/system/Dockerfile b/system/Dockerfile index af1b6e8..4d90676 100644 --- a/system/Dockerfile +++ b/system/Dockerfile @@ -48,6 +48,7 @@ RUN sed -i 's/^Components: main$/Components: main non-free-firmware/' /etc/apt/s ## cuos os extra for wake on lan \ ethtool \ ## cuos os dependencies \ + apparmor \ btrfs-progs \ console-setup \ dbus \ @@ -57,7 +58,6 @@ RUN sed -i 's/^Components: main$/Components: main non-free-firmware/' /etc/apt/s kbd \ kmod \ lsb-release \ - selinux-policy-default \ systemd \ systemd-sysv \ systemd-timesyncd \ @@ -89,6 +89,9 @@ RUN sed -i 's/^Components: main$/Components: main non-free-firmware/' /etc/apt/s && systemctl disable hv-kvp-daemon.service \ && systemctl disable hv-vss-daemon.service \ && systemctl enable systemd-timesyncd.service \ + && systemctl enable apparmor.service \ + ## /var/lib/dhcp is a symlink into /data, and AppArmor sees the resolved path \ + && printf "/data/dhcp/ r,\n/data/dhcp/dhclient* lrw,\n" >>/etc/apparmor.d/local/usr.sbin.dhclient \ && systemctl mask ssh.socket \ && systemctl mask sshd-keygen.service \ && printf "Port 4222\nPermitRootLogin yes\n" >/etc/ssh/sshd_config.d/cuos.conf \ diff --git a/system/Dockerfile.lxc b/system/Dockerfile.lxc index fe3d8c6..8d12ef0 100644 --- a/system/Dockerfile.lxc +++ b/system/Dockerfile.lxc @@ -16,6 +16,8 @@ RUN echo "Cache buster: v1" \ ## cuos system dependencies (lxc) \ busybox-static \ ## cuos os dependencies \ + # A container cannot load profiles - that is the LXC host's business \ + #-apparmor \ #-btrfs-progs \ dbus \ dbus-broker \ @@ -24,7 +26,6 @@ RUN echo "Cache buster: v1" \ #-kbd \ kmod \ lsb-release \ - #-selinux-policy-default \ systemd \ systemd-sysv \ #-systemd-timesyncd \ diff --git a/system/Dockerfile.orangepi-zero3 b/system/Dockerfile.orangepi-zero3 index 9f5e4ed..1d08bd8 100644 --- a/system/Dockerfile.orangepi-zero3 +++ b/system/Dockerfile.orangepi-zero3 @@ -75,6 +75,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ # rpi-swap \ # iw \ ## cuos os dependencies \ + apparmor \ btrfs-progs \ console-setup \ dbus \ @@ -84,7 +85,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ kbd \ kmod \ lsb-release \ - selinux-policy-default \ systemd \ systemd-sysv \ systemd-timesyncd \ @@ -112,6 +112,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ vim-tiny \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ && systemctl enable systemd-timesyncd.service \ + && systemctl enable apparmor.service \ + ## /var/lib/dhcp is a symlink into /data, and AppArmor sees the resolved path \ + && printf "/data/dhcp/ r,\n/data/dhcp/dhclient* lrw,\n" >>/etc/apparmor.d/local/usr.sbin.dhclient \ && systemctl mask ssh.socket \ && systemctl mask sshd-keygen.service \ && printf "Port 4222\nPermitRootLogin yes\n" >/etc/ssh/sshd_config.d/cuos.conf \ diff --git a/system/Dockerfile.rpi-arm32 b/system/Dockerfile.rpi-arm32 index d61b01c..9115c02 100644 --- a/system/Dockerfile.rpi-arm32 +++ b/system/Dockerfile.rpi-arm32 @@ -114,6 +114,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ # rpi-swap \ # iw \ ## cuos os dependencies \ + apparmor \ btrfs-progs \ console-setup \ dbus \ @@ -123,7 +124,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ kbd \ kmod \ lsb-release \ - selinux-policy-default \ systemd \ systemd-sysv \ systemd-timesyncd \ @@ -151,6 +151,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ vim-tiny \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ && systemctl enable systemd-timesyncd.service \ + && systemctl enable apparmor.service \ + ## /var/lib/dhcp is a symlink into /data, and AppArmor sees the resolved path \ + && printf "/data/dhcp/ r,\n/data/dhcp/dhclient* lrw,\n" >>/etc/apparmor.d/local/usr.sbin.dhclient \ && systemctl mask ssh.socket \ && systemctl mask sshd-keygen.service \ && printf "Port 4222\nPermitRootLogin yes\n" >/etc/ssh/sshd_config.d/cuos.conf \ diff --git a/system/Dockerfile.rpi-arm64 b/system/Dockerfile.rpi-arm64 index f6c627d..82baaa3 100644 --- a/system/Dockerfile.rpi-arm64 +++ b/system/Dockerfile.rpi-arm64 @@ -95,6 +95,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ # rpi-swap \ # iw \ ## cuos os dependencies \ + apparmor \ btrfs-progs \ console-setup \ dbus \ @@ -104,7 +105,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ kbd \ kmod \ lsb-release \ - selinux-policy-default \ systemd \ systemd-sysv \ systemd-timesyncd \ @@ -132,6 +132,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ vim-tiny \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ && systemctl enable systemd-timesyncd.service \ + && systemctl enable apparmor.service \ + ## /var/lib/dhcp is a symlink into /data, and AppArmor sees the resolved path \ + && printf "/data/dhcp/ r,\n/data/dhcp/dhclient* lrw,\n" >>/etc/apparmor.d/local/usr.sbin.dhclient \ && systemctl mask ssh.socket \ && systemctl mask sshd-keygen.service \ && printf "Port 4222\nPermitRootLogin yes\n" >/etc/ssh/sshd_config.d/cuos.conf \ diff --git a/system/armbianEnv.txt b/system/armbianEnv.txt index 2c304f4..b74244f 100644 --- a/system/armbianEnv.txt +++ b/system/armbianEnv.txt @@ -7,5 +7,5 @@ disp_mode=1920x1080p60 overlay_prefix=sun50i-h616 rootdev=LABEL=system rootfstype=btrfs -extraargs=rootflags=subvol=@os/system-{{SLOT}} loglevel=3 noresume apparmor=0 +extraargs=rootflags=subvol=@os/system-{{SLOT}} loglevel=3 noresume apparmor=1 usbstoragequirks=0x2537:0x1066:u,0x2537:0x1068:u diff --git a/system/cuos/install-kernel-grub.sh b/system/cuos/install-kernel-grub.sh index 8ab3b51..47660a2 100755 --- a/system/cuos/install-kernel-grub.sh +++ b/system/cuos/install-kernel-grub.sh @@ -82,7 +82,7 @@ menuentry '${SLOT_NAME}' --unrestricted { search --no-floppy --label boot --set=root - linux /${filename_kernel} root=LABEL=system rootfstype=btrfs rootflags=subvol=${CONTAINER_ROOTFS} ro loglevel=3 noresume apparmor=0 + linux /${filename_kernel} root=LABEL=system rootfstype=btrfs rootflags=subvol=${CONTAINER_ROOTFS} ro loglevel=3 noresume apparmor=1 initrd /${filename_initrd} } diff --git a/system/cuos/install-kernel-rpi.sh b/system/cuos/install-kernel-rpi.sh index 9633144..1c3a91d 100755 --- a/system/cuos/install-kernel-rpi.sh +++ b/system/cuos/install-kernel-rpi.sh @@ -38,7 +38,7 @@ fi mv "${TARGET_BOOT}"/cmdline.txt "${TARGET_BOOT}"/cmdline-previous.txt 2>/dev/null || true cat < "${TARGET_BOOT}/cmdline.txt" -console=serial0,115200 console=tty1 rootwait root=LABEL=system rootfstype=btrfs rootflags=subvol=${CONTAINER_ROOTFS} fsck.repair=yes ro loglevel=3 noresume apparmor=0 +console=serial0,115200 console=tty1 rootwait root=LABEL=system rootfstype=btrfs rootflags=subvol=${CONTAINER_ROOTFS} fsck.repair=yes ro loglevel=3 noresume apparmor=1 EOF # Attach options from system.json diff --git a/system/cuos/selftest.sh b/system/cuos/selftest.sh index 277a018..70e36e9 100755 --- a/system/cuos/selftest.sh +++ b/system/cuos/selftest.sh @@ -80,6 +80,15 @@ listening_on() { ss -ltn 2>/dev/null | grep -qE "[:.]$1[[:space:]]" } +apparmor_enabled() { + [[ "$(cat /sys/module/apparmor/parameters/enabled 2>/dev/null)" == "Y" ]] +} + +# " ()" per line, for every profile loaded into the kernel. +apparmor_profiles() { + cat /sys/kernel/security/apparmor/profiles 2>/dev/null +} + # How many cuos journal entries of priority error or worse this boot produced. journal_errors() { journalctl -t cuos -b -p 3 --no-pager -o cat 2>/dev/null | grep -c . || true @@ -177,6 +186,35 @@ check_docker() { failed "docker" "docker is not responding" } +# docker-default is the only thing confining the application container, and +# both ways of losing it - the LSM off, or dockerd never loading the profile - +# are silent. So it is asked for by name. +check_apparmor() { + if is_container; then + skipped "apparmor" "profiles belong to the host's kernel; a container loads none of its own" + return + fi + if ! apparmor_enabled; then + failed "apparmor" "the kernel has AppArmor off - check apparmor=1 on the command line" + return + fi + + local profiles count mode + profiles="$(apparmor_profiles)" + count="$(grep -c . <<<"${profiles}")" + mode="$(sed -n 's/^docker-default (\(.*\))$/\1/p' <<<"${profiles}")" + + if [[ -z "${mode}" ]]; then + failed "apparmor" "dockerd loaded no docker-default; profiles in the kernel: ${count}" + return + fi + if [[ "${mode}" != "enforce" ]]; then + failed "apparmor" "docker-default is in ${mode} mode, so it confines nothing" + return + fi + ok "apparmor" "${count} profiles loaded, docker-default enforcing" +} + check_app() { local status status="$(container_state "${APP_CONTAINER}")" @@ -278,6 +316,7 @@ selftest() { check_subvolumes check_state check_docker + check_apparmor check_app check_network check_hostname diff --git a/system/cuos/selftest.test.sh b/system/cuos/selftest.test.sh index 7f9b227..a068a25 100644 --- a/system/cuos/selftest.test.sh +++ b/system/cuos/selftest.test.sh @@ -33,6 +33,10 @@ MOCK_DOCKER_OK=0 MOCK_APP_STATE="running" MOCK_LISTENING=0 MOCK_JOURNAL_ERRORS=0 +MOCK_APPARMOR_ENABLED=0 +MOCK_APPARMOR_PROFILES="docker-default (enforce) +/usr/sbin/dhclient (enforce) +unix-chkpwd (enforce)" virt_type() { echo "${MOCK_VIRT}"; } subvolumes() { echo "${MOCK_SUBVOLUMES}"; } @@ -42,6 +46,8 @@ docker_ok() { return "${MOCK_DOCKER_OK}"; } container_state() { echo "${MOCK_APP_STATE}"; } listening_on() { return "${MOCK_LISTENING}"; } journal_errors() { echo "${MOCK_JOURNAL_ERRORS}"; } +apparmor_enabled() { return "${MOCK_APPARMOR_ENABLED}"; } +apparmor_profiles() { echo "${MOCK_APPARMOR_PROFILES}"; } # A healthy system, which each test then breaks in exactly one way. reset_system() { @@ -56,6 +62,10 @@ reset_system() { MOCK_APP_STATE="running" MOCK_LISTENING=0 MOCK_JOURNAL_ERRORS=0 + MOCK_APPARMOR_ENABLED=0 + MOCK_APPARMOR_PROFILES="docker-default (enforce) +/usr/sbin/dhclient (enforce) +unix-chkpwd (enforce)" cat >"${CONFIG_PATH}" <<'EOF' { @@ -151,6 +161,36 @@ expect "state: anything but running fails" "failed" status_of state is_updating break_docker() { MOCK_DOCKER_OK=1; } expect "docker: an unresponsive daemon fails" "failed" status_of docker break_docker +lsm_off() { MOCK_APPARMOR_ENABLED=1; } +expect "apparmor: a kernel with the LSM off fails" "failed" status_of apparmor lsm_off +expect "apparmor: the failure names the command line" \ + "the kernel has AppArmor off - check apparmor=1 on the command line" \ + detail_of apparmor lsm_off + +no_docker_default() { MOCK_APPARMOR_PROFILES="/usr/sbin/dhclient (enforce)"; } +expect "apparmor: enabled but no docker-default fails" "failed" status_of \ + apparmor no_docker_default +expect "apparmor: the failure counts what did load" \ + "dockerd loaded no docker-default; profiles in the kernel: 1" detail_of \ + apparmor no_docker_default + +complaining() { MOCK_APPARMOR_PROFILES="docker-default (complain)"; } +expect "apparmor: a complaining docker-default fails" "failed" status_of \ + apparmor complaining +expect "apparmor: and says it confines nothing" \ + "docker-default is in complain mode, so it confines nothing" detail_of \ + apparmor complaining + +# "docker-default-something" is a different profile, not this one. +similar_name() { MOCK_APPARMOR_PROFILES="docker-default-old (enforce)"; } +expect "apparmor: a profile merely starting with the name does not count" \ + "failed" status_of apparmor similar_name + +expect "apparmor: a healthy system reports the count" \ + "3 profiles loaded, docker-default enforcing" detail_of apparmor nothing +expect "apparmor: skipped in a container" "skipped" status_of apparmor \ + be_a_container + app_exited() { MOCK_APP_STATE="exited"; } expect "app: a stopped application fails" "failed" status_of app app_exited app_missing() { MOCK_APP_STATE=""; } @@ -193,7 +233,7 @@ expect "a skipped check does not" "true" overall be_a_container check_count() { report nothing | jq -r '.checks | length' } -expect "every check reports" "11" check_count +expect "every check reports" "12" check_count summary_adds_up() { report nothing | jq -r '