diff --git a/s3proxy/crypto.py b/s3proxy/crypto.py index 1bdc36f..7d2e7a5 100644 --- a/s3proxy/crypto.py +++ b/s3proxy/crypto.py @@ -191,16 +191,17 @@ def streaming_upload_peak(content_length: int) -> int: def copy_pipeline_peak(plaintext_size: int) -> int: """Peak memory a server-side copy holds while decrypting + re-encrypting. - A copy request has no body, so the request-level memory limiter reserves - ~nothing for it -- yet the copy reads the source, decrypts it and re-encrypts - it. Large copies stream in MAX_BUFFER_SIZE chunks, so their peak is the - pipeline (source chunk + plaintext buffer + dest ciphertext), independent of - object size; small copies buffer the whole object (~3x). Reserving this lets - the limiter BOUND concurrent copies instead of admitting an unbounded dedup - flood that OOMs the pod. + A copy request has no body, so the request-level limiter reserves ~nothing + for it -- yet the copy reads the source, decrypts it and re-encrypts it. The + streaming copy path now frames exactly like the framed UploadPart path + (encrypt_frame per FRAME_PLAINTEXT_SIZE, one internal part at a time, sized by + memory_bounded_part_size), so its per-part peak matches the upload path. It + adds one pipeline stage the body-fed upload path lacks: the source is read in + MAX_BUFFER_SIZE chunks into a _PlaintextReader buffer, so reserve two extra + buffers for it. Small copies buffer the whole object and re-encrypt it (~3x). """ if plaintext_size > STREAMING_THRESHOLD: - return 4 * MAX_BUFFER_SIZE + return streaming_upload_peak(plaintext_size) + 2 * MAX_BUFFER_SIZE return max(MAX_BUFFER_SIZE, 3 * plaintext_size) diff --git a/s3proxy/handlers/multipart/copy.py b/s3proxy/handlers/multipart/copy.py index f11c835..b84afff 100644 --- a/s3proxy/handlers/multipart/copy.py +++ b/s3proxy/handlers/multipart/copy.py @@ -1,8 +1,8 @@ """UploadPartCopy handler for multipart uploads.""" -import asyncio import hashlib import math +import time from collections.abc import AsyncIterator from datetime import UTC, datetime @@ -22,11 +22,10 @@ ) from ...utils import format_iso8601 from ..base import BaseHandler +from .upload_part import _PlaintextReader logger: BoundLogger = structlog.get_logger(__name__) -MAX_PARALLEL_INTERNAL_UPLOADS = 2 - class CopyPartMixin(BaseHandler): async def handle_upload_part_copy(self, request: Request, creds: S3Credentials) -> Response: @@ -60,21 +59,25 @@ async def handle_upload_part_copy(self, request: Request, creds: S3Credentials) ) if plaintext_size <= crypto.STREAMING_THRESHOLD: - return await self._simple_copy_part( - client, - bucket, - key, - upload_id, - part_num, - state, - src_bucket, - src_key, - copy_source_range, - head_resp, - src_metadata, - src_wrapped_dek, - src_multipart_meta, - ) + # Small copies buffer the whole object + re-encrypt it; gate them + # by the limiter too (they carry no body, so the request-level + # reservation was ~nothing and a small-object flood ran unbounded). + async with concurrency.reserve_memory(crypto.copy_pipeline_peak(plaintext_size)): + return await self._simple_copy_part( + client, + bucket, + key, + upload_id, + part_num, + state, + src_bucket, + src_key, + copy_source_range, + head_resp, + src_metadata, + src_wrapped_dek, + src_multipart_meta, + ) return await self._streaming_copy_part( client, bucket, @@ -254,7 +257,10 @@ async def _streaming_copy_part_inner( plaintext_size: int, ) -> Response: """Stream-decrypt the source and encrypt+upload each chunk as an internal S3 part.""" - chunk_size = crypto.calculate_optimal_part_size(plaintext_size) + # Same part sizing as the framed UploadPart path: bounds the internal part + # count to MAX_INTERNAL_PARTS_PER_CLIENT (so a large copy can never overflow + # the per-client internal-part range) and keeps parts small. + chunk_size = crypto.memory_bounded_part_size(plaintext_size) estimated_parts = max(1, math.ceil(plaintext_size / chunk_size)) internal_part_start = await self.multipart_manager.allocate_internal_parts( @@ -381,69 +387,69 @@ async def _pump_copy_chunks( chunk_size: int, internal_part_start: int, ) -> tuple[list[InternalPartMetadata], int, int, object]: - """Buffer plaintext from src_iter into chunk_size pieces and upload as internal parts.""" - buf = bytearray() - current_internal = internal_part_start - upload_tasks: dict[int, asyncio.Task] = {} - semaphore = asyncio.Semaphore(MAX_PARALLEL_INTERNAL_UPLOADS) + """Frame-encrypt the copy source into internal S3 parts, one at a time. + + Mirrors UploadPartMixin._stream_and_upload_framed: reads FRAME_PLAINTEXT_SIZE + plaintext frames from the source, encrypts each with encrypt_frame, + accumulates one internal part's ciphertext, then uploads it before starting + the next. Peak memory is O(one internal part ciphertext + one frame) -- it + never holds the whole part's plaintext and ciphertext at once and runs one + part at a time, which is what makes copy_pipeline_peak == streaming_upload_peak. + Parts are byte-identical to the framed upload path and read back via + decrypt_framed / _iter_multipart_plaintext. + """ + reader = _PlaintextReader(src_iter) md5 = hashlib.md5(usedforsecurity=False) + internal_parts: list[InternalPartMetadata] = [] total_plaintext = 0 - - async for raw in src_iter: - buf.extend(raw) - md5.update(raw) - total_plaintext += len(raw) - - while len(buf) >= chunk_size: - data = bytes(buf[:chunk_size]) - del buf[:chunk_size] - await semaphore.acquire() - task = asyncio.create_task( - self._upload_internal_part_with_semaphore( # type: ignore[attr-defined] - client, - bucket, - key, - upload_id, - part_num, - state, - data, - current_internal, - semaphore, + total_ciphertext = 0 + internal_part_num = internal_part_start + + while True: + ciphertext = bytearray() + part_plaintext = 0 + frame_idx = 0 + while part_plaintext < chunk_size: + frame_pt = await reader.read( + min(crypto.FRAME_PLAINTEXT_SIZE, chunk_size - part_plaintext) + ) + if not frame_pt: + break + md5.update(frame_pt) + part_plaintext += len(frame_pt) + ciphertext.extend( + crypto.encrypt_frame( + frame_pt, state.dek, upload_id, internal_part_num, frame_idx ) ) - upload_tasks[current_internal] = task - current_internal += 1 - - if buf: - await semaphore.acquire() - data = bytes(buf) - task = asyncio.create_task( - self._upload_internal_part_with_semaphore( # type: ignore[attr-defined] - client, - bucket, - key, - upload_id, - part_num, - state, - data, - current_internal, - semaphore, + frame_idx += 1 + + if part_plaintext == 0: + break + + upload_start = time.monotonic() + resp = await client.upload_part(bucket, key, upload_id, internal_part_num, ciphertext) + del ciphertext + ct_size = crypto.framed_ciphertext_size(part_plaintext) + internal_parts.append( + InternalPartMetadata( + internal_part_number=internal_part_num, + plaintext_size=part_plaintext, + ciphertext_size=ct_size, + etag=resp["ETag"].strip('"'), ) ) - upload_tasks[current_internal] = task - - results = await asyncio.gather(*upload_tasks.values(), return_exceptions=True) - self._check_upload_results(results, bucket, key, upload_id, part_num) # type: ignore[attr-defined] - - results_by_part: dict[int, InternalPartMetadata] = { - r.internal_part_number: r - for r in results # type: ignore[union-attr] - } - internal_parts = [] - total_ciphertext = 0 - for pn in sorted(results_by_part): - meta = results_by_part[pn] - internal_parts.append(meta) - total_ciphertext += meta.ciphertext_size + logger.info( + "INTERNAL_PART_UPLOADED", + bucket=bucket, + key=key, + client_part=part_num, + internal_part=internal_part_num, + plaintext_mb=f"{part_plaintext / 1024 / 1024:.2f}MB", + elapsed_sec=f"{time.monotonic() - upload_start:.2f}s", + ) + total_plaintext += part_plaintext + total_ciphertext += ct_size + internal_part_num += 1 return internal_parts, total_plaintext, total_ciphertext, md5 diff --git a/tests/unit/test_copy_memory_governing.py b/tests/unit/test_copy_memory_governing.py index 76b0936..82f7a06 100644 --- a/tests/unit/test_copy_memory_governing.py +++ b/tests/unit/test_copy_memory_governing.py @@ -21,13 +21,17 @@ MB = 1024 * 1024 -def test_copy_pipeline_peak_streamed_is_bounded(): - # Large copies stream in MAX_BUFFER_SIZE chunks, so their peak is the - # pipeline (source chunk + plaintext + dest ciphertext), independent of size. +def test_copy_pipeline_peak_matches_framed_upload(): + # The streaming copy path now frames exactly like the framed UploadPart path + # (one internal part at a time), so its peak is identical: the old + # size-independent 4*MAX_BUFFER_SIZE under-counted the real peak ~6x -- when + # it un-framed a whole part -- and OOMed the pod. for size in (64 * MB, 512 * MB, 5 * 1024 * MB): - peak = crypto.copy_pipeline_peak(size) - assert peak == 4 * crypto.MAX_BUFFER_SIZE - assert peak < size # never scales with object size + # Same framed writer peak, plus two MAX_BUFFER_SIZE buffers for the copy + # source read pipeline the body-fed upload path doesn't have. + assert crypto.copy_pipeline_peak(size) == ( + crypto.streaming_upload_peak(size) + 2 * crypto.MAX_BUFFER_SIZE + ) def test_copy_pipeline_peak_small_is_three_x(): @@ -48,9 +52,10 @@ async def test_reserve_memory_bounds_concurrent_copies(): original_timeout = concurrency.BACKPRESSURE_TIMEOUT concurrency.BACKPRESSURE_TIMEOUT = 30 limiter = concurrency._default - limiter.set_memory_limit(64) # 64MB budget limiter.active_bytes = 0 - per_copy = crypto.copy_pipeline_peak(64 * MB) # 32MB -> budget fits 2 + per_copy = crypto.copy_pipeline_peak(64 * MB) + limiter.set_memory_limit((per_copy * 2) // MB) # budget sized to fit ~2 + budget_fits = limiter.limit_bytes // per_copy peak_active = 0 inside = 0 @@ -71,5 +76,5 @@ async def one_copy(): concurrency.BACKPRESSURE_TIMEOUT = original_timeout assert peak_active <= limiter.limit_bytes # never overran the budget - assert max_inside <= 2 # 64MB / 32MB == at most 2 copies at once + assert max_inside <= budget_fits # limiter admits only what the budget fits assert limiter.active_bytes == 0 # everything released diff --git a/tests/unit/test_copy_reservation_vs_real.py b/tests/unit/test_copy_reservation_vs_real.py new file mode 100644 index 0000000..b53be39 --- /dev/null +++ b/tests/unit/test_copy_reservation_vs_real.py @@ -0,0 +1,119 @@ +"""The copy reservation must bound the streaming UploadPartCopy path's REAL peak. + +Analogue of test_upload_reservation_vs_real.py, for the copy path. This is the +test that would have caught the ScyllaDB backup OOM: copy_pipeline_peak() +reserved a size-independent 32MB (4*MAX_BUFFER_SIZE) on the false premise that +the copy "streams in MAX_BUFFER_SIZE chunks", while _pump_copy_chunks actually +buffers a full internal part and _upload_internal_part_with_semaphore holds +plaintext + ciphertext of that part at once, times MAX_PARALLEL_INTERNAL_UPLOADS. +The governor therefore admitted ~6x too many concurrent copies -> OOMKilled. + +It drives the ACTUAL handler method (not a re-implementation) under tracemalloc, +so it fails if the reservation drifts below reality OR the copy path starts +allocating more. The mock client copies the uploaded body the way aiobotocore +does for the signed HTTP request, which is part of the real peak. +""" + +import hashlib +import tracemalloc + +import pytest + +from s3proxy import crypto +from s3proxy.handlers.multipart import MultipartHandlerMixin +from s3proxy.state import MultipartUploadState + +MB = 1024 * 1024 + + +class _Mgr: + async def allocate_internal_parts(self, bucket, key, upload_id, count, client_part_number): + return 1 + + async def add_part(self, *a, **k): + return None + + +class _Body: + """aiohttp-like streaming body: `async with body`, `body.content.read(n)`.""" + + def __init__(self, total, chunk): + self._total = total + self._chunk = chunk + self._sent = 0 + self.content = self + + async def read(self, n): + if self._sent >= self._total: + return b"" + take = min(n, self._total - self._sent) + self._sent += take + return b"x" * take + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + +class _Client: + def __init__(self, total): + self._total = total + + async def get_object(self, bucket, key, range_header=None): + return {"Body": _Body(self._total, crypto.MAX_BUFFER_SIZE)} + + async def upload_part(self, bucket, key, upload_id, part_number, body): + # aiobotocore copies the body to sign and send it; mirror that so the + # measured peak reflects what the real transport holds. + sent = bytes(body) + return {"ETag": hashlib.md5(sent).hexdigest()} + + +def _handler(): + h = MultipartHandlerMixin.__new__(MultipartHandlerMixin) # bypass BaseHandler.__init__ + h.multipart_manager = _Mgr() + return h + + +async def _measure_peak(plaintext_size): + h = _handler() + state = MultipartUploadState(dek=crypto.generate_dek(), bucket="b", key="k", upload_id="u") + tracemalloc.start() + base = tracemalloc.get_traced_memory()[0] + # Plaintext (unencrypted) source: src_wrapped_dek=None, src_multipart_meta=None. + await h._streaming_copy_part_inner( + _Client(plaintext_size), + "b", + "k", + "u", + 1, + state, + "b", + "src", + None, + None, + None, + {}, + {}, + plaintext_size, + ) + peak = tracemalloc.get_traced_memory()[1] + tracemalloc.stop() + return peak - base + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mb", [64, 128, 512]) +async def test_reservation_bounds_real_copy_peak(mb): + plaintext_size = mb * MB + real_peak = await _measure_peak(plaintext_size) + reserved = crypto.copy_pipeline_peak(plaintext_size) + + # The whole point: the reservation must cover the REAL peak. The old + # size-independent 4*MAX_BUFFER_SIZE reservation fails here. + assert reserved >= real_peak, ( + f"{mb}MB copy: reserved {reserved / MB:.1f}MB < real peak " + f"{real_peak / MB:.1f}MB -- governor under-counts, will OOM" + ) diff --git a/tests/unit/test_streaming_copy.py b/tests/unit/test_streaming_copy.py index f023d89..8e15b46 100644 --- a/tests/unit/test_streaming_copy.py +++ b/tests/unit/test_streaming_copy.py @@ -193,6 +193,61 @@ async def test_large_unencrypted_source_splits_into_multiple_parts( assert bytes(recovered) == plaintext + @pytest.mark.asyncio + async def test_streamed_parts_are_framed_and_roundtrip( + self, mock_s3, settings, manager, credentials, monkeypatch + ): + """An internal part larger than one frame must be written framed + (encrypt_frame per FRAME_PLAINTEXT_SIZE) and round-trip via decrypt_framed. + Guards the framed copy writer -- the old writer sealed whole parts, so + this many-frames-per-part case was never exercised.""" + monkeypatch.setattr(crypto, "MAX_BUFFER_SIZE", 1024) + monkeypatch.setattr(crypto, "FRAME_PLAINTEXT_SIZE", 256) + handler = _make_handler(settings, manager) + await mock_s3.create_bucket("bucket") + + # 20 * MAX_BUFFER_SIZE -> memory_bounded_part_size caps at 20 parts of + # ~1KB each, and each 1KB part spans 4 x 256B frames. + plaintext = bytes((i * 31) % 256 for i in range(20 * 1024)) + await mock_s3.put_object("bucket", "src", plaintext) + + resp_create = await mock_s3.create_multipart_upload("bucket", "dst") + upload_id = resp_create["UploadId"] + kid, _ = settings.keyring.key_for(credentials.access_key) + dek = crypto.generate_dek() + await manager.create_upload("bucket", "dst", upload_id, dek, kid) + state = await manager.get_upload("bucket", "dst", upload_id) + + head_resp = {"Metadata": {}, "ContentLength": len(plaintext)} + await handler._streaming_copy_part( + mock_s3, + "bucket", + "dst", + upload_id, + 1, + state, + "bucket", + "src", + None, + None, + None, + head_resp, + {}, + len(plaintext), + ) + + updated = await manager.get_upload("bucket", "dst", upload_id) + part = updated.parts[1] + assert part.plaintext_size == len(plaintext) + # at least one internal part is genuinely multi-frame + assert any(crypto.frame_count(ip.plaintext_size) > 1 for ip in part.internal_parts) + + recovered = bytearray() + for ip in sorted(part.internal_parts, key=lambda x: x.internal_part_number): + ct = mock_s3.multipart_uploads[upload_id]["Parts"][ip.internal_part_number]["Body"] + recovered.extend(crypto.decrypt_framed(ct, dek, ip.plaintext_size)) + assert bytes(recovered) == plaintext + @pytest.mark.asyncio async def test_large_unencrypted_source_with_range( self, mock_s3, settings, manager, credentials