diff --git a/chart/templates/s3proxy/configmap.yaml b/chart/templates/s3proxy/configmap.yaml index 4ed8a93..ed3cd19 100644 --- a/chart/templates/s3proxy/configmap.yaml +++ b/chart/templates/s3proxy/configmap.yaml @@ -22,3 +22,7 @@ data: S3PROXY_DASHBOARD_UI: "true" S3PROXY_DASHBOARD_PATH: {{ .Values.dashboard.path | quote }} {{- end }} + {{- /* Arbitrary extra env (e.g. diagnostics like S3PROXY_TRACEMALLOC=1). */ -}} + {{- range $k, $v := .Values.extraConfig }} + {{ $k }}: {{ $v | quote }} + {{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index fabc1d9..7bcf2b6 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -255,4 +255,7 @@ topologySpreadConstraints: [] podDisruptionBudget: enabled: true minAvailable: 1 - # maxUnavailable: 1 # Alternative to minAvailable \ No newline at end of file + # maxUnavailable: 1 # Alternative to minAvailable +# Arbitrary extra S3PROXY_* env, injected via the config ConfigMap (envFrom). +# Used for time-boxed diagnostics, e.g. extraConfig: { S3PROXY_TRACEMALLOC: "1" }. +extraConfig: {} diff --git a/s3proxy/app.py b/s3proxy/app.py index aaa5c24..06cdbea 100644 --- a/s3proxy/app.py +++ b/s3proxy/app.py @@ -2,10 +2,14 @@ from __future__ import annotations +import asyncio +import contextlib import logging import os +import signal import sys import time +import tracemalloc import uuid from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -65,6 +69,52 @@ def _silence_health_probe_access_logs() -> None: access_logger.addFilter(_health_probe_filter) +def _dump_tracemalloc(limit: int = 20) -> None: + """Log the top live Python allocations by size, with location. + + Diagnostic only (gated by S3PROXY_TRACEMALLOC). Lets us see, under real + backup load, exactly which call sites hold the resident memory that drives + the OOM -- instead of guessing. + """ + if not tracemalloc.is_tracing(): + return + snap = tracemalloc.take_snapshot() + stats = snap.statistics("lineno") + total_mb = sum(s.size for s in stats) / 1024 / 1024 + logger.warning("TRACEMALLOC_SNAPSHOT", total_tracked_mb=round(total_mb, 1), shown=limit) + for i, st in enumerate(stats[:limit], 1): + fr = st.traceback[0] + logger.warning( + "TRACEMALLOC_TOP", + rank=i, + size_mb=round(st.size / 1024 / 1024, 2), + count=st.count, + loc=f"{fr.filename}:{fr.lineno}", + ) + + +async def _periodic_tracemalloc(interval: int) -> None: + while True: + await asyncio.sleep(interval) + _dump_tracemalloc() + + +def _maybe_start_tracemalloc() -> asyncio.Task | None: + """Enable tracemalloc + periodic/SIGUSR1 heap dumps when S3PROXY_TRACEMALLOC is set. + + No-op (zero overhead) when unset. Used for one-pod, time-boxed profiling. + """ + if not os.environ.get("S3PROXY_TRACEMALLOC"): + return None + frames = int(os.environ.get("S3PROXY_TRACEMALLOC_FRAMES", "4")) + interval = int(os.environ.get("S3PROXY_TRACEMALLOC_INTERVAL", "15")) + tracemalloc.start(frames) + logger.warning("TRACEMALLOC_ENABLED", frames=frames, interval_sec=interval) + with contextlib.suppress(NotImplementedError, RuntimeError): + asyncio.get_running_loop().add_signal_handler(signal.SIGUSR1, _dump_tracemalloc) + return asyncio.create_task(_periodic_tracemalloc(interval)) + + def create_lifespan(settings: Settings, credentials_store: dict[str, str]) -> AsyncIterator[None]: """Create lifespan context manager for FastAPI app. @@ -114,8 +164,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: app.state.stats_store = stats_store app.state.start_time = time.monotonic() + tracemalloc_task = _maybe_start_tracemalloc() + yield + if tracemalloc_task is not None: + tracemalloc_task.cancel() await stats_store.aclose() # flush buffered samples before Redis closes await close_redis() await close_http_client() diff --git a/tests/unit/test_tracemalloc_profiling.py b/tests/unit/test_tracemalloc_profiling.py new file mode 100644 index 0000000..90ff6af --- /dev/null +++ b/tests/unit/test_tracemalloc_profiling.py @@ -0,0 +1,41 @@ +"""Self-check for the gated tracemalloc heap-dump diagnostic. + +Off by default (no env) => zero overhead, no tracing started. When enabled it +must take a snapshot and not raise. Used for one-pod, time-boxed prod profiling +to identify the live allocations driving the OOM. +""" + +import tracemalloc + +from s3proxy import app + + +def test_disabled_by_default(monkeypatch): + monkeypatch.delenv("S3PROXY_TRACEMALLOC", raising=False) + assert app._maybe_start_tracemalloc() is None + + +def test_dump_is_noop_when_not_tracing(): + # Should not raise even if tracemalloc isn't running. + if tracemalloc.is_tracing(): + tracemalloc.stop() + app._dump_tracemalloc() # no exception = pass + + +def test_dump_reports_allocations_when_tracing(): + tracemalloc.start(2) + try: + blob = bytearray(4 * 1024 * 1024) # 4MB, should show up + # Capture warning logs to confirm it emits a snapshot + top lines. + events = [] + import structlog + + app.logger = structlog.wrap_logger( + app.logger, processors=[lambda _l, _m, ev: events.append(ev) or ev] + ) + app._dump_tracemalloc(limit=5) + assert blob is not None + assert any(e.get("event") == "TRACEMALLOC_SNAPSHOT" for e in events) + assert any(e.get("event") == "TRACEMALLOC_TOP" for e in events) + finally: + tracemalloc.stop()