From 65b87e3819d3d7b417a01b3033ec5efc36904d2d Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Wed, 1 Jul 2026 09:20:45 +0200 Subject: [PATCH] fix: cap UploadPartCopy pump chunk at MAX_BUFFER_SIZE (the prod OOM) Root cause of the concurrent-backup OOM, found via memray on a live pod: the top resident allocations were copy.py:398 (64MB) + crypto.py:354 (64MB) -- the scylla dedup UploadPartCopy path. _pump_copy_chunks sized its buffer from calculate_optimal_part_size, which returns up to 64MB for large sources. So each copy of a large SSTable buffered a 64MB chunk, copied it (bytes(buf[:chunk_size])) and re-encrypted it (~150-190MB resident) while the request limiter only reserved copy_pipeline_peak (~32MB, it assumes 8MB streaming). A handful of concurrent dedup copies of >=80MB parts therefore blew past the pod memory limit even though the governor read well under budget -- exactly the prod signature (RSS 512, governed ~48). Cap the pump chunk at MAX_BUFFER_SIZE so copies stream in 8MB chunks like every other path and stay matched to their reservation. Reproduced locally at prod config (512Mi/64MB): concurrent UploadPartCopy of 90-120MB sources pinned RSS at 511.9MiB (the wall); with the cap the same load peaks 321MiB. Streaming-copy round-trip tests still pass. --- s3proxy/handlers/multipart/copy.py | 8 ++++++- tests/unit/test_copy_chunk_bounded.py | 30 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_copy_chunk_bounded.py diff --git a/s3proxy/handlers/multipart/copy.py b/s3proxy/handlers/multipart/copy.py index f11c835..61a2775 100644 --- a/s3proxy/handlers/multipart/copy.py +++ b/s3proxy/handlers/multipart/copy.py @@ -254,7 +254,13 @@ async def _streaming_copy_part_inner( plaintext_size: int, ) -> Response: """Stream-decrypt the source and encrypt+upload each chunk as an internal S3 part.""" - chunk_size = crypto.calculate_optimal_part_size(plaintext_size) + # Cap the pump chunk at MAX_BUFFER_SIZE: calculate_optimal_part_size returns + # up to 64MB for large sources, which made _pump_copy_chunks buffer a 64MB + # chunk + copy it + re-encrypt (~150MB/copy) while the limiter only reserved + # copy_pipeline_peak (~32MB). Under a scylla dedup flood of large SSTables + # that under-reservation OOMed the pod. 8MB chunks keep the copy truly + # streaming and matched to the reservation. + chunk_size = min(crypto.calculate_optimal_part_size(plaintext_size), crypto.MAX_BUFFER_SIZE) estimated_parts = max(1, math.ceil(plaintext_size / chunk_size)) internal_part_start = await self.multipart_manager.allocate_internal_parts( diff --git a/tests/unit/test_copy_chunk_bounded.py b/tests/unit/test_copy_chunk_bounded.py new file mode 100644 index 0000000..d832f04 --- /dev/null +++ b/tests/unit/test_copy_chunk_bounded.py @@ -0,0 +1,30 @@ +"""UploadPartCopy must pump the source in MAX_BUFFER_SIZE chunks, not 64MB ones. + +The dedup copy path (_pump_copy_chunks) sized its buffer from +calculate_optimal_part_size, which returns up to 64MB for large sources. So a +copy of a large SSTable buffered a 64MB chunk + copied it + re-encrypted it +(~150MB resident) while the limiter only reserved copy_pipeline_peak (~32MB). +Under a scylla dedup flood that under-reservation OOMed the pod (reproduced +locally: concurrent UploadPartCopy of >=80MB sources pinned RSS at the 512Mi +wall; capping the chunk to 8MB dropped peak to ~320MiB). + +This pins the invariant: the pump chunk never exceeds MAX_BUFFER_SIZE, so the +copy stays streaming and matched to its reservation, regardless of source size. +""" + +from s3proxy import crypto + + +def _pump_chunk_size(plaintext_size: int) -> int: + # Mirror the sizing in _streaming_copy_part_inner (copy.py). + return min(crypto.calculate_optimal_part_size(plaintext_size), crypto.MAX_BUFFER_SIZE) + + +def test_optimal_part_size_is_large_for_big_sources(): + # Guard the premise: without the cap, big sources buffer huge chunks. + assert crypto.calculate_optimal_part_size(80 * 1024 * 1024) > crypto.MAX_BUFFER_SIZE + + +def test_pump_chunk_capped_at_max_buffer(): + for size_mb in (1, 8, 52, 80, 120, 5 * 1024, 100 * 1024): + assert _pump_chunk_size(size_mb * 1024 * 1024) <= crypto.MAX_BUFFER_SIZE