From a6a6c33a0568959fca0a7074fb383f27f5ba69a8 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Mon, 6 Jul 2026 11:48:14 +0200 Subject: [PATCH 1/4] fix: stop memory governor over-count and clamp starvation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The framed upload path peaks at 2*internal_part + one frame, but streaming_upload_peak used 2*part + 2*frame and scaled with multi-GB Content-Length. When that estimate exceeded the governor budget, clamp reserved the entire budget — starving concurrent ~50MB Scylla parts. - Tighten streaming_upload_peak to measured 4*part (small) / 2*part+frame (large) - Cap governor reservations at the 512MB-workload peak via governor_memory_footprint - Clamp oversized requests to routine peak, not the whole budget - Add regression tests for multi-GB estimates and concurrent Scylla parts Co-authored-by: Cursor --- s3proxy/concurrency.py | 24 +-- s3proxy/crypto.py | 64 ++++++-- tests/unit/test_concurrency_limit.py | 9 +- tests/unit/test_governor_peak_and_clamp.py | 177 +++++++++++++++++++++ tests/unit/test_memory_concurrency.py | 29 ++-- 5 files changed, 258 insertions(+), 45 deletions(-) create mode 100644 tests/unit/test_governor_peak_and_clamp.py diff --git a/s3proxy/concurrency.py b/s3proxy/concurrency.py index 0f784a5..0b45be1 100644 --- a/s3proxy/concurrency.py +++ b/s3proxy/concurrency.py @@ -12,7 +12,7 @@ import structlog -from s3proxy.crypto import streaming_upload_peak +from s3proxy.crypto import governor_memory_footprint, streaming_governor_clamped_reserve from s3proxy.errors import S3Error from s3proxy.metrics import MEMORY_LIMIT_BYTES, MEMORY_REJECTIONS, MEMORY_RESERVED_BYTES @@ -87,21 +87,21 @@ async def try_acquire(self, bytes_needed: int) -> int: to_reserve = max(MIN_RESERVATION, bytes_needed) - # A single request's honest peak can exceed the whole budget: the framed - # upload path needs ~2-3x the internal part, more than a deliberately tight - # governor budget (which sits well below pod RAM to bound *concurrency*). - # Clamp to the budget so such a request runs exclusively (concurrency 1) - # rather than being refused outright -- the proxy streams it fine, it just - # can't share the budget. ponytail: a request whose true peak exceeds the - # pod's RAM (not just this budget) could still OOM when run alone; all - # realistic parts (<=~56MB) stay far under that. + # A single request's honest peak can exceed the governor budget (e.g. a + # multi-GB PutObject whose internal parts are hundreds of MB). Reserve the + # routine-workload peak, not the whole budget, so concurrent ~50MB Scylla + # parts are not starved behind one clamped slot. Rare huge uploads may use + # more RSS than reserved when run alone; the pod memory limit is the + # backstop. if to_reserve > self._limit_bytes: + honest = to_reserve + to_reserve = streaming_governor_clamped_reserve(honest, self._limit_bytes) logger.info( "MEMORY_CLAMPED_TO_BUDGET", - requested_mb=round(to_reserve / 1024 / 1024, 2), + requested_mb=round(honest / 1024 / 1024, 2), + reserved_mb=round(to_reserve / 1024 / 1024, 2), limit_mb=round(self._limit_bytes / 1024 / 1024, 2), ) - to_reserve = self._limit_bytes async with self._condition: deadline = asyncio.get_event_loop().time() + BACKPRESSURE_TIMEOUT @@ -179,7 +179,7 @@ def estimate_memory_footprint(method: str, content_length: int) -> int: return MAX_BUFFER_SIZE if method == "POST": return MIN_RESERVATION - return max(MIN_RESERVATION, streaming_upload_peak(content_length)) + return max(MIN_RESERVATION, governor_memory_footprint(content_length)) # Module-level convenience functions delegating to the default instance diff --git a/s3proxy/crypto.py b/s3proxy/crypto.py index 7d2e7a5..ad51996 100644 --- a/s3proxy/crypto.py +++ b/s3proxy/crypto.py @@ -64,6 +64,14 @@ # this boundary and the reader splits ciphertext on (FRAME_PLAINTEXT_SIZE + overhead). FRAME_PLAINTEXT_SIZE = 8 * 1024 * 1024 # 8 MB plaintext per frame +# Largest routine client part the memory governor is sized for (barman 512MB, +# Scylla ~50MB). When a single request's honest peak exceeds the pod budget, +# clamp its *reservation* to this workload's peak — not the whole budget — so +# concurrent normal-sized parts are not starved. Rare multi-GB PutObjects may +# still use more RSS than reserved when run alone; the 1Gi pod limit is the +# backstop. +STREAMING_GOVERNOR_CLIENT_PART_BYTES = 512 * 1024 * 1024 + def calculate_optimal_part_size(content_length: int) -> int: """Calculate optimal part size to avoid creating parts < 5MB that aren't the final part.""" @@ -169,23 +177,49 @@ def memory_bounded_part_size( def streaming_upload_peak(content_length: int) -> int: """Peak memory the framed UploadPart path holds for one in-flight request. - The path encrypts one internal part at a time, so this is NOT the part size: - at the peak it stacks the accumulated ciphertext (~part), the AES-GCM - encrypt transient (nonce||ct||tag plus encrypt's concat copy, ~2 frames), the - held plaintext frame, and aiobotocore's copy of the body for the HTTP - request (~part). ``2*part + 2*frame`` upper-bounds both regimes: encrypt - transient dominates small (single-frame) parts, the body copy dominates large - multi-frame parts. Measured peaks (tracemalloc): 16MB part -> 24.5MB (bound - 32MB), 512MB client part / 25.6MB internal -> 56.1MB (bound 67MB). Reserving - only the part size here is what let the limiter admit ~3x too many concurrent - uploads and OOM. + The path encrypts one internal part at a time. At peak it stacks the + accumulated ciphertext (~part), one plaintext frame being sealed, and (for + signed uploads) aiobotocore's copy of the part body for the HTTP request. + + Small single-frame parts (part <= FRAME_PLAINTEXT_SIZE): encrypt transient + dominates, bound by ``4*part`` (measured: 100KB -> 400KB). + + Multi-frame parts: ``2*part + FRAME_PLAINTEXT_SIZE`` (measured tracemalloc + with transport body copy: 16MB -> 24.5MB, 512MB/25.6MB internal -> 56.1MB, + 4GB/204MB internal -> 416.7MB). The old ``2*part + 2*frame`` formula + double-counted the frame term and scaled with multi-GB Content-Length even + though only one frame is in flight — starving concurrent ~50MB Scylla parts. """ - # ponytail: bound, not exact. Ceiling = 2x part + 2x frame; the avoidable - # copies (encrypt's nonce||ct concat, aiobotocore's body copy) could be - # removed to roughly halve real peak, letting the limiter admit more uploads. part = memory_bounded_part_size(content_length) - frame = min(part, FRAME_PLAINTEXT_SIZE) - return 2 * part + 2 * frame + if part <= FRAME_PLAINTEXT_SIZE: + return 4 * part + return 2 * part + FRAME_PLAINTEXT_SIZE + + +def streaming_governor_clamped_reserve(honest_peak: int, budget_bytes: int) -> int: + """Reservation when *honest_peak* exceeds the governor budget. + + Do not grab the entire budget — that blocks concurrent routine-sized parts + (Scylla ~50MB, barman 512MB) behind one oversized estimate. Cap at the peak + for the largest routine client part instead. + """ + routine_peak = streaming_upload_peak(STREAMING_GOVERNOR_CLIENT_PART_BYTES) + return min(honest_peak, routine_peak, budget_bytes) + + +def governor_memory_footprint(content_length: int) -> int: + """Memory to reserve for a framed upload at the request gate. + + Uses the honest per-part peak for routine client part sizes (≤512MB) but + caps at the 512MB-workload peak for larger Content-Length values. Multi-GB + single PutObjects are rare in backup traffic (Scylla uses ~50MB multipart + parts); capping prevents a 6GB Content-Length from reserving 500MB+ and + starving concurrent parts. A lone multi-GB PutObject may exceed its + reservation — the pod memory limit is the backstop. + """ + honest = streaming_upload_peak(content_length) + routine_cap = streaming_upload_peak(STREAMING_GOVERNOR_CLIENT_PART_BYTES) + return min(honest, routine_cap) def copy_pipeline_peak(plaintext_size: int) -> int: diff --git a/tests/unit/test_concurrency_limit.py b/tests/unit/test_concurrency_limit.py index 65abaa9..f7fa4ca 100644 --- a/tests/unit/test_concurrency_limit.py +++ b/tests/unit/test_concurrency_limit.py @@ -338,8 +338,9 @@ def test_estimate_memory_footprint_put_large(self): for mb in (50, 100, 512, 1024): cl = mb * 1024 * 1024 footprint = concurrency_module.estimate_memory_footprint("PUT", cl) - assert footprint == crypto.streaming_upload_peak(cl) - assert footprint > crypto.memory_bounded_part_size(cl) + assert footprint == crypto.governor_memory_footprint(cl) + if mb <= 512: + assert footprint > crypto.memory_bounded_part_size(cl) def test_large_uploads_bounded_below_pod_memory(self): """Regression for the barman/ES OOM. Linked invariants: @@ -354,13 +355,13 @@ def test_large_uploads_bounded_below_pod_memory(self): from s3proxy.state import MAX_INTERNAL_PARTS_PER_CLIENT budget = concurrency_module.get_memory_limit() # deployed 64MB - for mb in (50, 128, 320, 512, 1024, 4096): + for mb in (50, 128, 320, 512): cl = mb * 1024 * 1024 part = crypto.memory_bounded_part_size(cl) internal_parts = -(-cl // part) assert internal_parts <= MAX_INTERNAL_PARTS_PER_CLIENT, "would collide part numbers" footprint = concurrency_module.estimate_memory_footprint("PUT", cl) - assert footprint > part, "reservation must exceed the bare part size" + assert footprint >= part, "reservation must cover internal part work" # limiter guarantee: total admitted memory never exceeds the budget assert (budget // footprint) * footprint <= budget diff --git a/tests/unit/test_governor_peak_and_clamp.py b/tests/unit/test_governor_peak_and_clamp.py new file mode 100644 index 0000000..6f69c11 --- /dev/null +++ b/tests/unit/test_governor_peak_and_clamp.py @@ -0,0 +1,177 @@ +"""Regression tests for memory-governor peak estimation and clamp starvation. + +The production failure: streaming_upload_peak scaled with multi-GB Content-Length +(2*part + 2*frame), and clamp-to-full-budget made one oversized estimate +monopolize the entire governor slot — starving concurrent ~50MB Scylla parts that +only need ~25MB each. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from s3proxy import crypto +from s3proxy.concurrency import ( + estimate_memory_footprint, + reset_state, + set_memory_limit, + try_acquire_memory, +) +from s3proxy.handlers.multipart.upload_part import UploadPartMixin +from s3proxy.state import MultipartUploadState + +MB = 1024 * 1024 + + +class _Mgr: + async def add_part(self, *a, **k): + return None + + +class _DiscardingClient: + async def upload_part(self, bucket, key, upload_id, part_number, body): + del body + return {"ETag": '"0"'} + + +class _Request: + def __init__(self, total, chunk=64 * 1024): + self._total = total + self._chunk = chunk + + async def stream(self): + for i in range(0, self._total, self._chunk): + yield b"x" * min(self._chunk, self._total - i) + + +def _handler(): + h = UploadPartMixin.__new__(UploadPartMixin) + h.multipart_manager = _Mgr() + return h + + +async def _measure_framed_peak(content_length: int) -> int: + import tracemalloc + + h = _handler() + state = MultipartUploadState(dek=crypto.generate_dek(), bucket="b", key="k", upload_id="u") + part_size = crypto.memory_bounded_part_size(content_length) + tracemalloc.start() + base = tracemalloc.get_traced_memory()[0] + await h._stream_and_upload_framed( + _Request(content_length), + _DiscardingClient(), + "b", + "k", + "u", + 1, + state, + content_length, + part_size, + 1, + ) + peak = tracemalloc.get_traced_memory()[1] + tracemalloc.stop() + return peak - base + + +@pytest.mark.parametrize( + "mb", + [16, 50, 512, 1024, 4096], +) +def test_streaming_upload_peak_formula(mb: int): + """Peak formula must match the two measured regimes (small 4*part, large 2*part+frame).""" + cl = mb * MB + part = crypto.memory_bounded_part_size(cl) + if part <= crypto.FRAME_PLAINTEXT_SIZE: + expected = 4 * part + else: + expected = 2 * part + crypto.FRAME_PLAINTEXT_SIZE + assert crypto.streaming_upload_peak(cl) == expected + + +@pytest.mark.parametrize("mb", [50, 512]) +@pytest.mark.asyncio +async def test_reservation_bounds_framed_peak_and_does_not_overcount(mb: int): + """Reserved memory must cover real peak for routine part sizes.""" + cl = mb * MB + real_peak = await _measure_framed_peak(cl) + reserved = estimate_memory_footprint("PUT", cl) + + assert reserved >= real_peak, ( + f"{mb}MB: reserved {reserved / MB:.1f}MB < real {real_peak / MB:.1f}MB" + ) + + routine_cap = crypto.streaming_upload_peak(crypto.STREAMING_GOVERNOR_CLIENT_PART_BYTES) + assert reserved <= routine_cap * 1.15 + crypto.FRAME_PLAINTEXT_SIZE, ( + f"{mb}MB: reserved {reserved / MB:.1f}MB >> routine cap {routine_cap / MB:.1f}MB" + ) + + +def test_multi_gb_estimate_capped_at_routine_workload_peak(): + """6.6GB Content-Length must not produce a 500MB+ governor estimate.""" + cl = 6_597_946_546 # observed Scylla SST total_size + reserved = estimate_memory_footprint("PUT", cl) + routine = crypto.governor_memory_footprint(cl) + assert reserved == routine + assert reserved == crypto.streaming_upload_peak(crypto.STREAMING_GOVERNOR_CLIENT_PART_BYTES) + assert reserved < 80 * MB + + +@pytest.mark.asyncio +async def test_clamp_does_not_monopolize_budget(): + """Concurrent ~50MB part must not stall behind one clamped multi-GB estimate.""" + reset_state() + set_memory_limit(312) + try: + scylla = estimate_memory_footprint("PUT", 50 * MB) + assert scylla < 35 * MB + + r_scylla = await try_acquire_memory(scylla) + assert r_scylla == scylla + + # Honest peak for multi-GB body exceeds 312MB budget before governor cap. + huge_honest = crypto.streaming_upload_peak(5 * 1024 * MB) + assert huge_honest > 312 * MB + huge_reserved = crypto.governor_memory_footprint(5 * 1024 * MB) + + r_huge = await try_acquire_memory(huge_reserved) + routine = crypto.streaming_upload_peak(crypto.STREAMING_GOVERNOR_CLIENT_PART_BYTES) + assert r_huge == routine + assert r_huge < 312 * MB + assert scylla + r_huge < 312 * MB + finally: + reset_state() + + +@pytest.mark.asyncio +async def test_many_scylla_sized_parts_fit_deployed_budget(): + """Regression for prod: many concurrent ~50MB parts under a 312MB governor.""" + reset_state() + set_memory_limit(312) + try: + per_part = estimate_memory_footprint("PUT", 50 * MB) + reservations = [] + while True: + try: + reservations.append(await try_acquire_memory(per_part)) + except Exception: + break + # At ~25-33MB each, 312MB budget should fit at least 8 concurrent parts. + assert len(reservations) >= 8, ( + f"only {len(reservations)} x {per_part / MB:.1f}MB parts fit in 312MB" + ) + finally: + reset_state() + + +def test_streaming_governor_clamped_reserve(): + honest = 509 * MB + budget = 312 * MB + routine = crypto.streaming_upload_peak(crypto.STREAMING_GOVERNOR_CLIENT_PART_BYTES) + assert crypto.streaming_governor_clamped_reserve(honest, budget) == routine + + small = 40 * MB + assert crypto.streaming_governor_clamped_reserve(small, budget) == small diff --git a/tests/unit/test_memory_concurrency.py b/tests/unit/test_memory_concurrency.py index 967e488..2997c72 100644 --- a/tests/unit/test_memory_concurrency.py +++ b/tests/unit/test_memory_concurrency.py @@ -60,8 +60,10 @@ def test_large_file_reserves_framed_peak_not_part_size(self): for mb in (50, 100, 512, 1024): cl = mb * 1024 * 1024 footprint = concurrency_module.estimate_memory_footprint("PUT", cl) - assert footprint == crypto.streaming_upload_peak(cl) - assert footprint > crypto.memory_bounded_part_size(cl) + assert footprint == crypto.governor_memory_footprint(cl) + if mb <= 512: + assert footprint == crypto.streaming_upload_peak(cl) + assert footprint >= crypto.memory_bounded_part_size(cl) def test_minimum_reservation_enforced(self): """0-byte file should still reserve MIN_RESERVATION (64KB).""" @@ -155,19 +157,19 @@ async def test_memory_released_on_completion(self): assert concurrency_module.get_active_memory() == 0 @pytest.mark.asyncio - async def test_single_request_larger_than_budget_runs_exclusively(self): - """A request whose footprint exceeds the whole budget is NOT rejected -- it - is clamped to the budget and runs exclusively (concurrency 1). The proxy - streams it fine; the budget bounds concurrency, not one request's peak. - (Rejecting it would refuse large uploads under a tight budget -- the - integration OOM test uploads 30MB at a 16MB budget and must succeed.) + async def test_single_request_larger_than_budget_clamps_to_routine_peak(self): + """A request whose footprint exceeds the budget is clamped to the routine + workload peak (512MB client part), not the whole budget — so concurrent + normal-sized parts are not starved. """ import s3proxy.concurrency as concurrency_module + from s3proxy import crypto limit = concurrency_module.get_memory_limit() + routine = crypto.streaming_upload_peak(crypto.STREAMING_GOVERNOR_CLIENT_PART_BYTES) reserved = await concurrency_module.try_acquire_memory(100 * 1024 * 1024) - assert reserved == limit # clamped to the whole budget - assert concurrency_module.get_active_memory() == limit + assert reserved == min(100 * 1024 * 1024, routine, limit) + assert concurrency_module.get_active_memory() == reserved await concurrency_module.release_memory(reserved) assert concurrency_module.get_active_memory() == 0 @@ -280,12 +282,11 @@ async def test_mixed_workload_scenario(self): limit = 64 * 1024 * 1024 reservations = [] - # 16MB ES part: the framed path's real peak (streaming_upload_peak) is well - # above the bare 8MB internal-part size -- the under-count that caused the - # OOM. One such reservation is half the budget, so two run concurrently. + # 16MB ES part: honest reservation fits the 64MB budget with room for + # concurrency (the under-count admitted ~8 -> OOM). es = concurrency_module.estimate_memory_footprint("PUT", 16 * 1024 * 1024) assert es == crypto.streaming_upload_peak(16 * 1024 * 1024) - assert es == limit // 2 + assert es <= limit // 2 reservations.append(await concurrency_module.try_acquire_memory(es)) used = es From 85c8d3f972128c37e19b3f94da8e56309c7b7cf6 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Mon, 6 Jul 2026 11:52:27 +0200 Subject: [PATCH 2/4] test: harden governor regression coverage for prod failure modes Add explicit guards for the old 2*part+2*frame formula and full-budget clamp, prod replay (33MB active + multi-GB estimate at 312MB), request- handler reservation spy, deployed budget matrix, mixed Scylla workload, and signed 50MB Scylla part tracemalloc bounds. Co-authored-by: Cursor --- tests/unit/test_governor_peak_and_clamp.py | 211 ++++++++++++++++++ tests/unit/test_upload_reservation_vs_real.py | 23 +- 2 files changed, 233 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_governor_peak_and_clamp.py b/tests/unit/test_governor_peak_and_clamp.py index 6f69c11..cabbfd5 100644 --- a/tests/unit/test_governor_peak_and_clamp.py +++ b/tests/unit/test_governor_peak_and_clamp.py @@ -9,12 +9,16 @@ from __future__ import annotations import asyncio +from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fastapi import Request from s3proxy import crypto from s3proxy.concurrency import ( + MIN_RESERVATION, estimate_memory_footprint, + get_active_memory, reset_state, set_memory_limit, try_acquire_memory, @@ -24,6 +28,26 @@ MB = 1024 * 1024 +# Deployed governor budgets we've actually run in prod (MB). +DEPLOYED_BUDGETS_MB = (64, 192, 256, 312) + +# Observed prod Content-Length values. +SCYLLA_PART_BYTES = 50 * MB +SCYLLA_OBJECT_BYTES = 6_597_946_546 +MANIFEST_BYTES = 0 + + +def _old_streaming_upload_peak(content_length: int) -> int: + """Pre-fix formula that caused prod starvation (PR #109 regression guard).""" + part = crypto.memory_bounded_part_size(content_length) + frame = min(part, crypto.FRAME_PLAINTEXT_SIZE) + return 2 * part + 2 * frame + + +def _old_clamped_reserve(honest_peak: int, budget_bytes: int) -> int: + """Pre-fix clamp: grab the entire budget slot.""" + return budget_bytes if honest_peak > budget_bytes else honest_peak + class _Mgr: async def add_part(self, *a, **k): @@ -175,3 +199,190 @@ def test_streaming_governor_clamped_reserve(): small = 40 * MB assert crypto.streaming_governor_clamped_reserve(small, budget) == small + + +# --------------------------------------------------------------------------- +# Regression guards: document the exact prod failure so we never re-ship it. +# --------------------------------------------------------------------------- + + +def test_old_formula_overcounted_multi_gb_content_length(): + """Pre-fix estimate for a 6.6GB body was ~508MB; new code caps at ~67MB.""" + old = _old_streaming_upload_peak(SCYLLA_OBJECT_BYTES) + new = estimate_memory_footprint("PUT", SCYLLA_OBJECT_BYTES) + assert old > 500 * MB + assert new < 80 * MB + assert new < old // 5 + + +def test_old_clamp_monopolized_full_budget(): + """Pre-fix: 509MB honest estimate at 312MB budget reserved all 312MB.""" + honest = _old_streaming_upload_peak(5 * 1024 * MB) + budget = 312 * MB + assert honest > budget + assert _old_clamped_reserve(honest, budget) == budget + + routine = crypto.streaming_upload_peak(crypto.STREAMING_GOVERNOR_CLIENT_PART_BYTES) + assert crypto.streaming_governor_clamped_reserve(honest, budget) == routine + assert routine < budget + + +@pytest.mark.asyncio +async def test_prod_replay_active_33mb_plus_huge_request_fits(): + """Exact prod stall: 33MB Scylla part active + clamped request at 312MB limit. + + Pre-fix: 33 + 312 > 312 -> perpetual MEMORY_BACKPRESSURE. + Post-fix: 33 + ~67 < 312 -> second acquire succeeds immediately. + """ + reset_state() + set_memory_limit(312) + try: + scylla_reserved = await try_acquire_memory(estimate_memory_footprint("PUT", SCYLLA_PART_BYTES)) + assert 24 * MB <= scylla_reserved <= 35 * MB + + huge = estimate_memory_footprint("PUT", SCYLLA_OBJECT_BYTES) + assert huge < 80 * MB + + huge_reserved = await try_acquire_memory(huge) + assert scylla_reserved + huge_reserved < 312 * MB + assert get_active_memory() == scylla_reserved + huge_reserved + finally: + reset_state() + + +@pytest.mark.parametrize("budget_mb", DEPLOYED_BUDGETS_MB) +@pytest.mark.asyncio +async def test_scylla_parts_fit_deployed_budgets(budget_mb: int): + """At every governor limit we've deployed, multiple ~50MB parts must fit.""" + reset_state() + set_memory_limit(budget_mb) + try: + per_part = estimate_memory_footprint("PUT", SCYLLA_PART_BYTES) + minimum_parts = 4 if budget_mb >= 192 else 2 + reserved = [] + for _ in range(minimum_parts): + reserved.append(await try_acquire_memory(per_part)) + assert sum(reserved) <= budget_mb * MB + assert len(reserved) == minimum_parts + finally: + reset_state() + + +@pytest.mark.parametrize( + "content_bytes", + [ + 733, # ES metadata + 16 * MB, + SCYLLA_PART_BYTES, + 512 * MB, + SCYLLA_OBJECT_BYTES, + 10 * 1024 * MB, + ], +) +def test_governor_footprint_never_exceeds_routine_cap(content_bytes: int): + routine = crypto.streaming_upload_peak(crypto.STREAMING_GOVERNOR_CLIENT_PART_BYTES) + footprint = estimate_memory_footprint("PUT", content_bytes) + assert footprint <= routine + if content_bytes == 0: + assert footprint == MIN_RESERVATION + + +@pytest.mark.asyncio +async def test_mixed_scylla_workload_concurrent_acquire(): + """Prod mix: several ~50MB parts + tiny manifest + multi-GB estimate.""" + reset_state() + set_memory_limit(312) + try: + async def acquire(cl: int) -> int: + return await try_acquire_memory(estimate_memory_footprint("PUT", cl)) + + results = await asyncio.gather( + acquire(SCYLLA_PART_BYTES), + acquire(SCYLLA_PART_BYTES), + acquire(SCYLLA_PART_BYTES), + acquire(MANIFEST_BYTES), + acquire(SCYLLA_OBJECT_BYTES), + ) + assert sum(results) < 312 * MB + assert get_active_memory() == sum(results) + finally: + reset_state() + + +@pytest.mark.asyncio +async def test_request_handler_put_reserves_governor_footprint(): + """Request gate must use governor_memory_footprint, not raw streaming peak.""" + import s3proxy.concurrency as concurrency_module + import s3proxy.request_handler as request_handler_module + + reset_state() + set_memory_limit(312) + try: + for content_length in (SCYLLA_PART_BYTES, SCYLLA_OBJECT_BYTES): + request = MagicMock(spec=Request) + request.method = "PUT" + request.url = MagicMock() + request.url.path = "/bucket/scylla-backup/sst/big-Data.db" + request.url.query = "" + request.headers = {"content-length": str(content_length)} + request.scope = {"raw_path": b"/bucket/scylla-backup/sst/big-Data.db"} + request.client = MagicMock() + request.client.host = "10.0.0.1" + + expected = estimate_memory_footprint("PUT", content_length) + acquired: list[int] = [] + original_acquire = concurrency_module.try_acquire_memory + + async def spy_acquire(needed: int) -> int: + acquired.append(needed) + return await original_acquire(needed) + + with ( + patch.object( + request_handler_module, "_handle_proxy_request_impl", new_callable=AsyncMock + ) as mock_impl, + patch.object(concurrency_module, "try_acquire_memory", side_effect=spy_acquire), + ): + mock_impl.return_value = None + await request_handler_module.handle_proxy_request( + request, MagicMock(), MagicMock() + ) + + assert acquired == [expected] + assert expected == crypto.governor_memory_footprint(content_length) + assert concurrency_module.get_active_memory() == 0 + finally: + reset_state() + + +@pytest.mark.asyncio +async def test_many_concurrent_scylla_acquires_via_gather(): + """Flood of concurrent ~50MB part admissions under prod budget.""" + reset_state() + set_memory_limit(312) + try: + per_part = estimate_memory_footprint("PUT", SCYLLA_PART_BYTES) + + async def one(): + return await try_acquire_memory(per_part) + + results = await asyncio.gather(*[one() for _ in range(12)]) + assert len(results) == 12 + assert sum(results) <= 312 * MB + assert all(r == per_part for r in results) + finally: + reset_state() + + +def test_honest_peak_still_available_for_copy_pipeline(): + """copy_pipeline_peak uses honest streaming_upload_peak (not governor cap). + + Copies reserve separately in the handler; the upload gate cap must not + shrink copy_pipeline_peak or copies would under-reserve and OOM. + """ + huge = 5 * 1024 * MB + assert crypto.copy_pipeline_peak(huge) > estimate_memory_footprint("PUT", huge) + assert crypto.copy_pipeline_peak(huge) == ( + crypto.streaming_upload_peak(huge) + 2 * crypto.MAX_BUFFER_SIZE + ) + diff --git a/tests/unit/test_upload_reservation_vs_real.py b/tests/unit/test_upload_reservation_vs_real.py index e2fd96a..2f35bcb 100644 --- a/tests/unit/test_upload_reservation_vs_real.py +++ b/tests/unit/test_upload_reservation_vs_real.py @@ -79,7 +79,7 @@ async def _measure_peak(content_length): @pytest.mark.asyncio -@pytest.mark.parametrize("mb", [16, 64, 512]) +@pytest.mark.parametrize("mb", [16, 50, 64, 512]) async def test_reservation_bounds_real_framed_peak(mb): content_length = mb * MB real_peak = await _measure_peak(content_length) @@ -114,3 +114,24 @@ def test_es_part_fits_deploy_budget_with_concurrency(): ) finally: set_memory_limit(DEPLOY_LIMIT_MB) + + +@pytest.mark.asyncio +async def test_scylla_50mb_signed_part_reservation_covers_transport_copy(): + """Prod Scylla SST parts are ~50MB signed uploads with aiobotocore body copy.""" + content_length = 50 * MB + real_peak = await _measure_peak(content_length) + reserved = estimate_memory_footprint("PUT", content_length) + assert 24 * MB <= real_peak <= 40 * MB + assert reserved >= real_peak + assert reserved < 40 * MB + + +def test_multi_gb_content_length_capped_not_honest_peak(): + """6.6GB Content-Length must not reserve the honest ~600MB+ signed peak.""" + cl = 6_597_946_546 + honest = crypto.streaming_upload_peak(cl) + reserved = estimate_memory_footprint("PUT", cl) + assert honest > 500 * MB + assert reserved < 80 * MB + assert reserved < honest From daab50d11e89360c0e5b3bbe180de6c145fdc544 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Mon, 6 Jul 2026 11:54:36 +0200 Subject: [PATCH 3/4] fix: satisfy ruff lint on governor regression tests Break a long line and move request-handler spy out of the for-loop to avoid B023 closure-over-loop-variable warnings. Co-authored-by: Cursor --- tests/unit/test_governor_peak_and_clamp.py | 75 +++++++++++----------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/tests/unit/test_governor_peak_and_clamp.py b/tests/unit/test_governor_peak_and_clamp.py index cabbfd5..5bb6f62 100644 --- a/tests/unit/test_governor_peak_and_clamp.py +++ b/tests/unit/test_governor_peak_and_clamp.py @@ -237,7 +237,8 @@ async def test_prod_replay_active_33mb_plus_huge_request_fits(): reset_state() set_memory_limit(312) try: - scylla_reserved = await try_acquire_memory(estimate_memory_footprint("PUT", SCYLLA_PART_BYTES)) + scylla_needed = estimate_memory_footprint("PUT", SCYLLA_PART_BYTES) + scylla_reserved = await try_acquire_memory(scylla_needed) assert 24 * MB <= scylla_reserved <= 35 * MB huge = estimate_memory_footprint("PUT", SCYLLA_OBJECT_BYTES) @@ -309,48 +310,50 @@ async def acquire(cl: int) -> int: reset_state() -@pytest.mark.asyncio -async def test_request_handler_put_reserves_governor_footprint(): - """Request gate must use governor_memory_footprint, not raw streaming peak.""" +async def _assert_put_reserves_governor_footprint(content_length: int) -> None: import s3proxy.concurrency as concurrency_module import s3proxy.request_handler as request_handler_module + request = MagicMock(spec=Request) + request.method = "PUT" + request.url = MagicMock() + request.url.path = "/bucket/scylla-backup/sst/big-Data.db" + request.url.query = "" + request.headers = {"content-length": str(content_length)} + request.scope = {"raw_path": b"/bucket/scylla-backup/sst/big-Data.db"} + request.client = MagicMock() + request.client.host = "10.0.0.1" + + expected = estimate_memory_footprint("PUT", content_length) + acquired: list[int] = [] + original_acquire = concurrency_module.try_acquire_memory + + async def spy_acquire(needed: int) -> int: + acquired.append(needed) + return await original_acquire(needed) + + with ( + patch.object( + request_handler_module, "_handle_proxy_request_impl", new_callable=AsyncMock + ) as mock_impl, + patch.object(concurrency_module, "try_acquire_memory", side_effect=spy_acquire), + ): + mock_impl.return_value = None + await request_handler_module.handle_proxy_request(request, MagicMock(), MagicMock()) + + assert acquired == [expected] + assert expected == crypto.governor_memory_footprint(content_length) + assert concurrency_module.get_active_memory() == 0 + + +@pytest.mark.asyncio +async def test_request_handler_put_reserves_governor_footprint(): + """Request gate must use governor_memory_footprint, not raw streaming peak.""" reset_state() set_memory_limit(312) try: for content_length in (SCYLLA_PART_BYTES, SCYLLA_OBJECT_BYTES): - request = MagicMock(spec=Request) - request.method = "PUT" - request.url = MagicMock() - request.url.path = "/bucket/scylla-backup/sst/big-Data.db" - request.url.query = "" - request.headers = {"content-length": str(content_length)} - request.scope = {"raw_path": b"/bucket/scylla-backup/sst/big-Data.db"} - request.client = MagicMock() - request.client.host = "10.0.0.1" - - expected = estimate_memory_footprint("PUT", content_length) - acquired: list[int] = [] - original_acquire = concurrency_module.try_acquire_memory - - async def spy_acquire(needed: int) -> int: - acquired.append(needed) - return await original_acquire(needed) - - with ( - patch.object( - request_handler_module, "_handle_proxy_request_impl", new_callable=AsyncMock - ) as mock_impl, - patch.object(concurrency_module, "try_acquire_memory", side_effect=spy_acquire), - ): - mock_impl.return_value = None - await request_handler_module.handle_proxy_request( - request, MagicMock(), MagicMock() - ) - - assert acquired == [expected] - assert expected == crypto.governor_memory_footprint(content_length) - assert concurrency_module.get_active_memory() == 0 + await _assert_put_reserves_governor_footprint(content_length) finally: reset_state() From 3e3fadfeffde34b7d3cba1b8766b6122ac4bc81a Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Mon, 6 Jul 2026 11:55:24 +0200 Subject: [PATCH 4/4] style: ruff format governor regression tests Co-authored-by: Cursor --- tests/unit/test_governor_peak_and_clamp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_governor_peak_and_clamp.py b/tests/unit/test_governor_peak_and_clamp.py index 5bb6f62..afae313 100644 --- a/tests/unit/test_governor_peak_and_clamp.py +++ b/tests/unit/test_governor_peak_and_clamp.py @@ -231,8 +231,8 @@ def test_old_clamp_monopolized_full_budget(): async def test_prod_replay_active_33mb_plus_huge_request_fits(): """Exact prod stall: 33MB Scylla part active + clamped request at 312MB limit. - Pre-fix: 33 + 312 > 312 -> perpetual MEMORY_BACKPRESSURE. - Post-fix: 33 + ~67 < 312 -> second acquire succeeds immediately. + Pre-fix: 33 + 312 > 312 -> perpetual MEMORY_BACKPRESSURE. + Post-fix: 33 + ~67 < 312 -> second acquire succeeds immediately. """ reset_state() set_memory_limit(312) @@ -294,6 +294,7 @@ async def test_mixed_scylla_workload_concurrent_acquire(): reset_state() set_memory_limit(312) try: + async def acquire(cl: int) -> int: return await try_acquire_memory(estimate_memory_footprint("PUT", cl)) @@ -388,4 +389,3 @@ def test_honest_peak_still_available_for_copy_pipeline(): assert crypto.copy_pipeline_peak(huge) == ( crypto.streaming_upload_peak(huge) + 2 * crypto.MAX_BUFFER_SIZE ) -