From d8560788ecc7a522c52cb808c13ed7478bf869cf Mon Sep 17 00:00:00 2001 From: vpetersson-bot Date: Sat, 24 Oct 2026 18:15:37 +0000 Subject: [PATCH] fix(compose): stop a crash filling the card with core dumps A process that crashes inside a container writes `core.` into its working directory. That is `WORKDIR /usr/src/app`: inside the container, on no mount, and reachable by no cleanup we have. The host's `core_pattern` is the kernel default `core`, a relative path, and `core_uses_pid=1` means every crash adds a file rather than replacing one. `ulimit -c` is `unlimited` in a container by default, so nothing bounds it. The result is invisible: `docker images` does not list it and `docker image prune` cannot touch a live container's writable layer, so the space goes missing with nothing to account for it. A device in the field reached 24.2 GB of dumps in anthias-celery before its card filled. - Set `ulimits: core: 0` on every service in all three compose templates, as one anchor aliased everywhere so the rule cannot drift between services or between deployments. - Add tests asserting every service in every template resolves to `core: 0`, since the real risk is a sixth service added later by someone who does not know this rule exists. Changing the service definition also recreates the containers on the next upgrade, which discards the layer holding any dumps already there. Measured on a Pi 4: before, three crashes left three files of 4,861,952 bytes each and grew the layer from 1.21 MB to 11.4 MB. After, six crashes across celery and server left none, and the worker stayed healthy. Co-Authored-By: Claude Opus 5 Co-authored-by: multica-agent --- docker-compose.balena.dev.yml.tmpl | 19 +++++++ docker-compose.balena.yml.tmpl | 19 +++++++ docker-compose.yml.tmpl | 19 +++++++ tests/test_core_dump_limits.py | 91 ++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+) create mode 100644 tests/test_core_dump_limits.py diff --git a/docker-compose.balena.dev.yml.tmpl b/docker-compose.balena.dev.yml.tmpl index aa7279dac..98ccf827b 100644 --- a/docker-compose.balena.dev.yml.tmpl +++ b/docker-compose.balena.dev.yml.tmpl @@ -4,6 +4,21 @@ version: "2" services: anthias-server: image: ghcr.io/screenly/anthias-server:${GIT_SHORT_HASH}-${BOARD} + # A crash must not cost the device its storage. The host's + # core_pattern is the kernel default `core`, a relative path, so a + # dump lands in the crashing process's working directory, which is + # WORKDIR /usr/src/app: inside the container, on no mount. With + # core_uses_pid=1 every crash adds a fresh `core.` rather than + # replacing one, and nothing sweeps them. They are invisible to + # `docker images`, and `docker image prune` cannot touch a live + # container's writable layer, so the usual disk investigation comes + # back clean while the card fills. A device in the field reached + # 24.2 GB of them in anthias-celery (issue 3417). Sentry already + # reports crashes, so on a device a dump buys nothing and costs the + # card. Anchored here and aliased by every service below: the rule + # is "any of our containers", not "the one that happened to crash". + ulimits: &no-core-dumps + core: 0 build: context: . dockerfile: ./docker/Dockerfile.server @@ -35,6 +50,7 @@ services: anthias-viewer: image: ghcr.io/screenly/anthias-viewer:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps build: context: . dockerfile: ./docker/Dockerfile.viewer @@ -67,6 +83,7 @@ services: # Runs on the same image as anthias-server with a CMD override. # See docker-compose.yml.tmpl for context on the merge. image: ghcr.io/screenly/anthias-server:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps command: > nice -n 19 ionice -c 3 celery -A anthias_server.celery_tasks.celery worker -B -n worker@anthias @@ -140,6 +157,7 @@ services: # like. anthias-wifi-connect: image: ghcr.io/screenly-labs/wifi-connect:2026.10.13 + ulimits: *no-core-dumps # Serving DHCP, which does not survive Docker's NAT, and binding the # access point's own address on the wireless device. network_mode: host @@ -219,6 +237,7 @@ services: redis: image: ghcr.io/screenly/anthias-redis:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps build: context: . dockerfile: ./docker/Dockerfile.redis diff --git a/docker-compose.balena.yml.tmpl b/docker-compose.balena.yml.tmpl index 2676aca3c..005d3a413 100644 --- a/docker-compose.balena.yml.tmpl +++ b/docker-compose.balena.yml.tmpl @@ -4,6 +4,21 @@ version: "2" services: anthias-server: image: ghcr.io/screenly/anthias-server:${GIT_SHORT_HASH}-${BOARD} + # A crash must not cost the device its storage. The host's + # core_pattern is the kernel default `core`, a relative path, so a + # dump lands in the crashing process's working directory, which is + # WORKDIR /usr/src/app: inside the container, on no mount. With + # core_uses_pid=1 every crash adds a fresh `core.` rather than + # replacing one, and nothing sweeps them. They are invisible to + # `docker images`, and `docker image prune` cannot touch a live + # container's writable layer, so the usual disk investigation comes + # back clean while the card fills. A device in the field reached + # 24.2 GB of them in anthias-celery (issue 3417). Sentry already + # reports crashes, so on a device a dump buys nothing and costs the + # card. Anchored here and aliased by every service below: the rule + # is "any of our containers", not "the one that happened to crash". + ulimits: &no-core-dumps + core: 0 ports: - 80:8080 environment: @@ -32,6 +47,7 @@ services: anthias-viewer: image: ghcr.io/screenly/anthias-viewer:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps depends_on: - anthias-server # Issue #480 — to localise URL-asset rendering, set LANG (and @@ -65,6 +81,7 @@ services: # Runs on the same image as anthias-server with a CMD override. # See docker-compose.yml.tmpl for context on the merge. image: ghcr.io/screenly/anthias-server:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps command: > nice -n 19 ionice -c 3 celery -A anthias_server.celery_tasks.celery worker -B -n worker@anthias @@ -138,6 +155,7 @@ services: # like. anthias-wifi-connect: image: ghcr.io/screenly-labs/wifi-connect:2026.10.13 + ulimits: *no-core-dumps # Serving DHCP, which does not survive Docker's NAT, and binding the # access point's own address on the wireless device. network_mode: host @@ -217,6 +235,7 @@ services: redis: image: ghcr.io/screenly/anthias-redis:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps ports: - 127.0.0.1:6379:6379 restart: always diff --git a/docker-compose.yml.tmpl b/docker-compose.yml.tmpl index 8d38348a4..fd6d2b254 100644 --- a/docker-compose.yml.tmpl +++ b/docker-compose.yml.tmpl @@ -3,6 +3,21 @@ services: anthias-server: image: ghcr.io/screenly/anthias-server:${DOCKER_TAG}-${DEVICE_TYPE} + # A crash must not cost the device its storage. The host's + # core_pattern is the kernel default `core`, a relative path, so a + # dump lands in the crashing process's working directory, which is + # WORKDIR /usr/src/app: inside the container, on no mount. With + # core_uses_pid=1 every crash adds a fresh `core.` rather than + # replacing one, and nothing sweeps them. They are invisible to + # `docker images`, and `docker image prune` cannot touch a live + # container's writable layer, so the usual disk investigation comes + # back clean while the card fills. A device in the field reached + # 24.2 GB of them in anthias-celery (issue 3417). Sentry already + # reports crashes, so on a device a dump buys nothing and costs the + # card. Anchored here and aliased by every service below: the rule + # is "any of our containers", not "the one that happened to crash". + ulimits: &no-core-dumps + core: 0 build: context: . dockerfile: docker/Dockerfile.server @@ -68,6 +83,7 @@ services: anthias-viewer: image: ghcr.io/screenly/anthias-viewer:${DOCKER_TAG}-${DEVICE_TYPE} + ulimits: *no-core-dumps build: context: . dockerfile: docker/Dockerfile.viewer @@ -146,6 +162,7 @@ services: # and a separate celery image was duplicating ~825 MB extracted of # identical content per device. See refactor: drop celery image. image: ghcr.io/screenly/anthias-server:${DOCKER_TAG}-${DEVICE_TYPE} + ulimits: *no-core-dumps # ``mem_limit`` keeps a runaway celery task (e.g. a HEIC → # WebP convert on a decompression-bomb fixture) from eating # the box's RAM and starving the viewer. 60% of host (computed @@ -211,6 +228,7 @@ services: # can never land on a fleet without a deliberate bump here. anthias-wifi-connect: image: ghcr.io/screenly-labs/wifi-connect:2026.10.13 + ulimits: *no-core-dumps # Pinned, because three other things address this container BY # NAME and compose's generated name is `anthias-anthias-wifi- # connect-1`: the systemd unit's `docker start`, the @@ -354,6 +372,7 @@ services: redis: image: ghcr.io/screenly/anthias-redis:${DOCKER_TAG}-${DEVICE_TYPE} + ulimits: *no-core-dumps build: context: . dockerfile: docker/Dockerfile.redis diff --git a/tests/test_core_dump_limits.py b/tests/test_core_dump_limits.py new file mode 100644 index 000000000..d42e66bb2 --- /dev/null +++ b/tests/test_core_dump_limits.py @@ -0,0 +1,91 @@ +""" +Every container Anthias ships must refuse to write a core dump. + +The device's card is the thing at stake, not the dump. A crashing +process writes `core.` into its working directory, which for our +images is `WORKDIR /usr/src/app`: inside the container, on no mount, and +reachable by no cleanup we have. `docker images` does not list it and +`docker image prune` cannot touch a live container's writable layer, so +the space goes missing with nothing to show for it. A device in the +field accumulated 24.2 GB that way (issue 3417). + +These assertions are over the compose templates rather than a running +stack because that is where the limit is declared, and because the +failure they guard against is someone adding a sixth service and not +knowing this rule exists. +""" + +import os + +import pytest +import yaml + +REPO_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir), +) + +TEMPLATES = ( + 'docker-compose.yml.tmpl', + 'docker-compose.balena.yml.tmpl', + 'docker-compose.balena.dev.yml.tmpl', +) + + +Service = dict[str, object] + + +def _services(template: str) -> dict[str, Service]: + with open(os.path.join(REPO_ROOT, template)) as f: + document = yaml.safe_load(f) + services: dict[str, Service] = document['services'] + return services + + +@pytest.mark.parametrize('template', TEMPLATES) +def test_every_service_disables_core_dumps(template: str) -> None: + offenders = { + name: service.get('ulimits') + for name, service in _services(template).items() + if not isinstance(limits := service.get('ulimits'), dict) + or limits.get('core') != 0 + } + + assert not offenders, ( + f'{template}: these services would write core dumps into their ' + f'own writable layer: {sorted(offenders)}. Add ' + f'`ulimits: *no-core-dumps` to each.' + ) + + +@pytest.mark.parametrize('template', TEMPLATES) +def test_core_limit_is_shared_rather_than_retyped(template: str) -> None: + """One anchor, aliased everywhere else. + + Five copies of `core: 0` would drift the moment someone edits one of + them, and the YAML alias is what makes "every service" checkable by + reading a single line. + """ + with open(os.path.join(REPO_ROOT, template)) as f: + body = f.read() + + assert body.count('ulimits: &no-core-dumps') == 1, ( + f'{template}: expected exactly one anchor definition' + ) + assert body.count('ulimits: *no-core-dumps') == ( + len(_services(template)) - 1 + ), f'{template}: every service but the anchor should alias it' + + +def test_templates_agree_on_which_services_exist() -> None: + """The balena templates are easy to update in isolation and forget. + + If they ever diverge, the parametrised checks above still pass on + each file separately while one deployment quietly loses a service's + protection, so compare them directly. + """ + by_template = {t: sorted(_services(t)) for t in TEMPLATES} + distinct = {tuple(v) for v in by_template.values()} + + assert len(distinct) == 1, ( + f'compose templates list different services: {by_template}' + )