diff --git a/docker-compose.balena.dev.yml.tmpl b/docker-compose.balena.dev.yml.tmpl index aa7279dac..98ccf827b 100644 --- a/docker-compose.balena.dev.yml.tmpl +++ b/docker-compose.balena.dev.yml.tmpl @@ -4,6 +4,21 @@ version: "2" services: anthias-server: image: ghcr.io/screenly/anthias-server:${GIT_SHORT_HASH}-${BOARD} + # A crash must not cost the device its storage. The host's + # core_pattern is the kernel default `core`, a relative path, so a + # dump lands in the crashing process's working directory, which is + # WORKDIR /usr/src/app: inside the container, on no mount. With + # core_uses_pid=1 every crash adds a fresh `core.` rather than + # replacing one, and nothing sweeps them. They are invisible to + # `docker images`, and `docker image prune` cannot touch a live + # container's writable layer, so the usual disk investigation comes + # back clean while the card fills. A device in the field reached + # 24.2 GB of them in anthias-celery (issue 3417). Sentry already + # reports crashes, so on a device a dump buys nothing and costs the + # card. Anchored here and aliased by every service below: the rule + # is "any of our containers", not "the one that happened to crash". + ulimits: &no-core-dumps + core: 0 build: context: . dockerfile: ./docker/Dockerfile.server @@ -35,6 +50,7 @@ services: anthias-viewer: image: ghcr.io/screenly/anthias-viewer:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps build: context: . dockerfile: ./docker/Dockerfile.viewer @@ -67,6 +83,7 @@ services: # Runs on the same image as anthias-server with a CMD override. # See docker-compose.yml.tmpl for context on the merge. image: ghcr.io/screenly/anthias-server:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps command: > nice -n 19 ionice -c 3 celery -A anthias_server.celery_tasks.celery worker -B -n worker@anthias @@ -140,6 +157,7 @@ services: # like. anthias-wifi-connect: image: ghcr.io/screenly-labs/wifi-connect:2026.10.13 + ulimits: *no-core-dumps # Serving DHCP, which does not survive Docker's NAT, and binding the # access point's own address on the wireless device. network_mode: host @@ -219,6 +237,7 @@ services: redis: image: ghcr.io/screenly/anthias-redis:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps build: context: . dockerfile: ./docker/Dockerfile.redis diff --git a/docker-compose.balena.yml.tmpl b/docker-compose.balena.yml.tmpl index 2676aca3c..005d3a413 100644 --- a/docker-compose.balena.yml.tmpl +++ b/docker-compose.balena.yml.tmpl @@ -4,6 +4,21 @@ version: "2" services: anthias-server: image: ghcr.io/screenly/anthias-server:${GIT_SHORT_HASH}-${BOARD} + # A crash must not cost the device its storage. The host's + # core_pattern is the kernel default `core`, a relative path, so a + # dump lands in the crashing process's working directory, which is + # WORKDIR /usr/src/app: inside the container, on no mount. With + # core_uses_pid=1 every crash adds a fresh `core.` rather than + # replacing one, and nothing sweeps them. They are invisible to + # `docker images`, and `docker image prune` cannot touch a live + # container's writable layer, so the usual disk investigation comes + # back clean while the card fills. A device in the field reached + # 24.2 GB of them in anthias-celery (issue 3417). Sentry already + # reports crashes, so on a device a dump buys nothing and costs the + # card. Anchored here and aliased by every service below: the rule + # is "any of our containers", not "the one that happened to crash". + ulimits: &no-core-dumps + core: 0 ports: - 80:8080 environment: @@ -32,6 +47,7 @@ services: anthias-viewer: image: ghcr.io/screenly/anthias-viewer:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps depends_on: - anthias-server # Issue #480 — to localise URL-asset rendering, set LANG (and @@ -65,6 +81,7 @@ services: # Runs on the same image as anthias-server with a CMD override. # See docker-compose.yml.tmpl for context on the merge. image: ghcr.io/screenly/anthias-server:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps command: > nice -n 19 ionice -c 3 celery -A anthias_server.celery_tasks.celery worker -B -n worker@anthias @@ -138,6 +155,7 @@ services: # like. anthias-wifi-connect: image: ghcr.io/screenly-labs/wifi-connect:2026.10.13 + ulimits: *no-core-dumps # Serving DHCP, which does not survive Docker's NAT, and binding the # access point's own address on the wireless device. network_mode: host @@ -217,6 +235,7 @@ services: redis: image: ghcr.io/screenly/anthias-redis:${GIT_SHORT_HASH}-${BOARD} + ulimits: *no-core-dumps ports: - 127.0.0.1:6379:6379 restart: always diff --git a/docker-compose.yml.tmpl b/docker-compose.yml.tmpl index 8d38348a4..fd6d2b254 100644 --- a/docker-compose.yml.tmpl +++ b/docker-compose.yml.tmpl @@ -3,6 +3,21 @@ services: anthias-server: image: ghcr.io/screenly/anthias-server:${DOCKER_TAG}-${DEVICE_TYPE} + # A crash must not cost the device its storage. The host's + # core_pattern is the kernel default `core`, a relative path, so a + # dump lands in the crashing process's working directory, which is + # WORKDIR /usr/src/app: inside the container, on no mount. With + # core_uses_pid=1 every crash adds a fresh `core.` rather than + # replacing one, and nothing sweeps them. They are invisible to + # `docker images`, and `docker image prune` cannot touch a live + # container's writable layer, so the usual disk investigation comes + # back clean while the card fills. A device in the field reached + # 24.2 GB of them in anthias-celery (issue 3417). Sentry already + # reports crashes, so on a device a dump buys nothing and costs the + # card. Anchored here and aliased by every service below: the rule + # is "any of our containers", not "the one that happened to crash". + ulimits: &no-core-dumps + core: 0 build: context: . dockerfile: docker/Dockerfile.server @@ -68,6 +83,7 @@ services: anthias-viewer: image: ghcr.io/screenly/anthias-viewer:${DOCKER_TAG}-${DEVICE_TYPE} + ulimits: *no-core-dumps build: context: . dockerfile: docker/Dockerfile.viewer @@ -146,6 +162,7 @@ services: # and a separate celery image was duplicating ~825 MB extracted of # identical content per device. See refactor: drop celery image. image: ghcr.io/screenly/anthias-server:${DOCKER_TAG}-${DEVICE_TYPE} + ulimits: *no-core-dumps # ``mem_limit`` keeps a runaway celery task (e.g. a HEIC → # WebP convert on a decompression-bomb fixture) from eating # the box's RAM and starving the viewer. 60% of host (computed @@ -211,6 +228,7 @@ services: # can never land on a fleet without a deliberate bump here. anthias-wifi-connect: image: ghcr.io/screenly-labs/wifi-connect:2026.10.13 + ulimits: *no-core-dumps # Pinned, because three other things address this container BY # NAME and compose's generated name is `anthias-anthias-wifi- # connect-1`: the systemd unit's `docker start`, the @@ -354,6 +372,7 @@ services: redis: image: ghcr.io/screenly/anthias-redis:${DOCKER_TAG}-${DEVICE_TYPE} + ulimits: *no-core-dumps build: context: . dockerfile: docker/Dockerfile.redis diff --git a/tests/test_core_dump_limits.py b/tests/test_core_dump_limits.py new file mode 100644 index 000000000..d42e66bb2 --- /dev/null +++ b/tests/test_core_dump_limits.py @@ -0,0 +1,91 @@ +""" +Every container Anthias ships must refuse to write a core dump. + +The device's card is the thing at stake, not the dump. A crashing +process writes `core.` into its working directory, which for our +images is `WORKDIR /usr/src/app`: inside the container, on no mount, and +reachable by no cleanup we have. `docker images` does not list it and +`docker image prune` cannot touch a live container's writable layer, so +the space goes missing with nothing to show for it. A device in the +field accumulated 24.2 GB that way (issue 3417). + +These assertions are over the compose templates rather than a running +stack because that is where the limit is declared, and because the +failure they guard against is someone adding a sixth service and not +knowing this rule exists. +""" + +import os + +import pytest +import yaml + +REPO_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir), +) + +TEMPLATES = ( + 'docker-compose.yml.tmpl', + 'docker-compose.balena.yml.tmpl', + 'docker-compose.balena.dev.yml.tmpl', +) + + +Service = dict[str, object] + + +def _services(template: str) -> dict[str, Service]: + with open(os.path.join(REPO_ROOT, template)) as f: + document = yaml.safe_load(f) + services: dict[str, Service] = document['services'] + return services + + +@pytest.mark.parametrize('template', TEMPLATES) +def test_every_service_disables_core_dumps(template: str) -> None: + offenders = { + name: service.get('ulimits') + for name, service in _services(template).items() + if not isinstance(limits := service.get('ulimits'), dict) + or limits.get('core') != 0 + } + + assert not offenders, ( + f'{template}: these services would write core dumps into their ' + f'own writable layer: {sorted(offenders)}. Add ' + f'`ulimits: *no-core-dumps` to each.' + ) + + +@pytest.mark.parametrize('template', TEMPLATES) +def test_core_limit_is_shared_rather_than_retyped(template: str) -> None: + """One anchor, aliased everywhere else. + + Five copies of `core: 0` would drift the moment someone edits one of + them, and the YAML alias is what makes "every service" checkable by + reading a single line. + """ + with open(os.path.join(REPO_ROOT, template)) as f: + body = f.read() + + assert body.count('ulimits: &no-core-dumps') == 1, ( + f'{template}: expected exactly one anchor definition' + ) + assert body.count('ulimits: *no-core-dumps') == ( + len(_services(template)) - 1 + ), f'{template}: every service but the anchor should alias it' + + +def test_templates_agree_on_which_services_exist() -> None: + """The balena templates are easy to update in isolation and forget. + + If they ever diverge, the parametrised checks above still pass on + each file separately while one deployment quietly loses a service's + protection, so compare them directly. + """ + by_template = {t: sorted(_services(t)) for t in TEMPLATES} + distinct = {tuple(v) for v in by_template.values()} + + assert len(distinct) == 1, ( + f'compose templates list different services: {by_template}' + )