diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 65de5a5..fad86a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: fail-fast: false matrix: shard: [memory_usage, memory_leak, memory_copy, core, multipart, copy_range, misc] - timeout-minutes: ${{ matrix.shard == 'memory_copy' && 25 || (contains(fromJSON('["memory_usage", "memory_leak"]'), matrix.shard) && 15 || 10) }} + timeout-minutes: ${{ matrix.shard == 'memory_copy' && 35 || (contains(fromJSON('["memory_usage", "memory_leak"]'), matrix.shard) && 15 || 10) }} steps: - uses: actions/checkout@v7 diff --git a/Makefile b/Makefile index b041f19..089b688 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test test-all test-unit test-integration test-integration-shard test-run test-oom verify-copy-memory e2e cluster lint +.PHONY: test test-all test-unit test-integration test-integration-shard test-run test-oom verify-copy-memory verify-passthrough e2e cluster lint # Lint: ruff check + format check lint: @@ -16,10 +16,14 @@ test-unit: verify-copy-memory: uv run pytest tests/integration/test_copy_per_part_metrics.py -m e2e -v -n0 +# Exhaustive UploadPartCopy passthrough proof (real subprocess + MinIO). +verify-passthrough: + uv run pytest tests/integration/test_upload_part_copy_passthrough_e2e.py -m e2e -v -n0 + # Integration shards for parallel CI (make test-integration-shard SHARD=memory_usage) INTEGRATION_memory_usage_TESTS = tests/integration/test_memory_usage.py INTEGRATION_memory_leak_TESTS = tests/integration/test_memory_leak.py -INTEGRATION_memory_copy_TESTS = tests/integration/test_copy_memory_governor.py tests/integration/test_copy_per_part_metrics.py +INTEGRATION_memory_copy_TESTS = tests/integration/test_copy_memory_governor.py tests/integration/test_copy_per_part_metrics.py tests/integration/test_upload_part_copy_passthrough_e2e.py INTEGRATION_memory_copy_PYTEST_OPTS = -n0 INTEGRATION_core_TESTS = \ tests/integration/test_integration.py \ diff --git a/s3proxy/concurrency.py b/s3proxy/concurrency.py index bda4f3b..4abbba0 100644 --- a/s3proxy/concurrency.py +++ b/s3proxy/concurrency.py @@ -140,6 +140,7 @@ async def try_acquire(self, bytes_needed: int, *, copy: bool = False) -> int: if exclusive: self._pending_exclusive += 1 try: + logged_backpressure = False while not self._can_admit(to_reserve, exclusive): remaining = deadline - asyncio.get_event_loop().time() if remaining <= 0: @@ -158,13 +159,15 @@ async def try_acquire(self, bytes_needed: int, *, copy: bool = False) -> int: f"Memory limit: {active_mb:.0f}MB + " f"{request_mb:.0f}MB > {limit_mb:.0f}MB" ) - logger.info( - "MEMORY_BACKPRESSURE", - active_mb=round(self._active_bytes / 1024 / 1024, 2), - requested_mb=round(to_reserve / 1024 / 1024, 2), - limit_mb=round(self._limit_bytes / 1024 / 1024, 2), - remaining_sec=round(remaining, 1), - ) + if not logged_backpressure: + logger.info( + "MEMORY_BACKPRESSURE", + active_mb=round(self._active_bytes / 1024 / 1024, 2), + requested_mb=round(to_reserve / 1024 / 1024, 2), + limit_mb=round(self._limit_bytes / 1024 / 1024, 2), + remaining_sec=round(remaining, 1), + ) + logged_backpressure = True with contextlib.suppress(TimeoutError): await asyncio.wait_for(self._condition.wait(), timeout=remaining) diff --git a/s3proxy/crypto.py b/s3proxy/crypto.py index 6bef673..7dfccf7 100644 --- a/s3proxy/crypto.py +++ b/s3proxy/crypto.py @@ -264,10 +264,9 @@ def copy_internal_part_size(plaintext_size: int) -> int: def copy_chunk_peak(chunk_plaintext_bytes: int) -> int: """Peak memory while encrypting one internal copy chunk (streaming path). - The streaming copy path acquires/releases this amount per internal part so - memory is free between chunks and during S3 upload I/O. Reservation covers - read-buffer slack plus framed encrypt peak; ciphertext upload runs after - release (smaller RSS than encrypt peak). + The streaming copy path acquires/releases this amount per internal part. + Reservation covers read-buffer slack plus framed encrypt peak and the + ciphertext buffer through S3 upload (released after del ciphertext). """ framed = ( 4 * chunk_plaintext_bytes @@ -280,6 +279,16 @@ def copy_chunk_peak(chunk_plaintext_bytes: int) -> int: return peak +# UploadPartCopy passthrough moves bytes server-side; in-process peak is tiny. +COPY_PASSTHROUGH_SEGMENT_PEAK = 64 * 1024 + + +def copy_passthrough_segment_peak(plaintext_size: int) -> int: + """Governor reservation for one native copy segment (no decrypt/re-encrypt).""" + _ = plaintext_size + return COPY_PASSTHROUGH_SEGMENT_PEAK + + def copy_small_buffered_peak(plaintext_size: int) -> int: """Peak for a small copy that buffers the whole object in RAM.""" return max(MAX_BUFFER_SIZE, 3 * plaintext_size) diff --git a/s3proxy/handlers/multipart/copy.py b/s3proxy/handlers/multipart/copy.py index d4f4946..3bf0f24 100644 --- a/s3proxy/handlers/multipart/copy.py +++ b/s3proxy/handlers/multipart/copy.py @@ -1,10 +1,15 @@ """UploadPartCopy handler for multipart uploads.""" +import asyncio +import base64 import hashlib import math +import os import time from collections.abc import AsyncIterator +from dataclasses import dataclass from datetime import UTC, datetime +from urllib.parse import quote import structlog from fastapi import Request, Response @@ -15,10 +20,12 @@ from ...errors import S3Error from ...state import ( InternalPartMetadata, + MultipartMetadata, MultipartUploadState, PartMetadata, load_multipart_metadata, load_upload_state, + persist_upload_state, ) from ...utils import format_iso8601 from ..base import BaseHandler @@ -26,6 +33,29 @@ logger: BoundLogger = structlog.get_logger(__name__) +# Cap concurrent streaming copy pipelines per pod. Without this, HAProxy maxconn +# can land many UploadPartCopy requests on one pod; each holds ciphertext and +# read buffers outside the governor if reservation is released before upload. +MAX_PARALLEL_COPY_PIPELINES = int(os.environ.get("S3PROXY_MAX_PARALLEL_COPIES", "2")) +_copy_pipeline_semaphore = asyncio.Semaphore(MAX_PARALLEL_COPY_PIPELINES) + + +def reset_copy_pipeline_semaphore(limit: int | None = None) -> None: + """Reset the global copy pipeline semaphore (testing only).""" + global _copy_pipeline_semaphore + _copy_pipeline_semaphore = asyncio.Semaphore( + limit if limit is not None else MAX_PARALLEL_COPY_PIPELINES + ) + + +@dataclass(frozen=True, slots=True) +class _CiphertextSegment: + """One contiguous ciphertext span in a completed encrypted object.""" + + plaintext_size: int + ciphertext_size: int + ct_offset: int + class CopyPartMixin(BaseHandler): async def handle_upload_part_copy(self, request: Request, creds: S3Credentials) -> Response: @@ -62,6 +92,30 @@ async def handle_upload_part_copy(self, request: Request, creds: S3Credentials) # Small copies buffer the whole object + re-encrypt it; gate them # by the limiter too (they carry no body, so the request-level # reservation was ~nothing and a small-object flood ran unbounded). + if self._can_passthrough_part_copy( + copy_source_range, + src_wrapped_dek, + src_multipart_meta, + src_metadata, + creds, + plaintext_size, + ): + return await self._gated_passthrough_copy_part( + client, + bucket, + key, + upload_id, + part_num, + state, + src_bucket, + src_key, + copy_source, + head_resp, + src_metadata, + src_wrapped_dek, + src_multipart_meta, + plaintext_size, + ) peak = crypto.copy_pipeline_peak(plaintext_size) async with concurrency.reserve_copy_memory(peak): return await self._simple_copy_part( @@ -79,6 +133,30 @@ async def handle_upload_part_copy(self, request: Request, creds: S3Credentials) src_wrapped_dek, src_multipart_meta, ) + if self._can_passthrough_part_copy( + copy_source_range, + src_wrapped_dek, + src_multipart_meta, + src_metadata, + creds, + plaintext_size, + ): + return await self._gated_passthrough_copy_part( + client, + bucket, + key, + upload_id, + part_num, + state, + src_bucket, + src_key, + copy_source, + head_resp, + src_metadata, + src_wrapped_dek, + src_multipart_meta, + plaintext_size, + ) return await self._streaming_copy_part( client, bucket, @@ -120,6 +198,349 @@ def _copy_plaintext_size( start, end = self._parse_copy_source_range(copy_source_range, total) return end - start + 1 + def _can_passthrough_part_copy( + self, + copy_source_range: str | None, + src_wrapped_dek: str | None, + src_multipart_meta: MultipartMetadata | None, + src_metadata: dict, + creds: S3Credentials, + plaintext_size: int, + ) -> bool: + """True when a native server-side copy preserves decryptability without re-encrypt.""" + if copy_source_range: + return False + if not src_wrapped_dek and not src_multipart_meta: + return False + # Large PutObject blobs have no sidecar part map; streaming re-encrypt frames them. + if not src_multipart_meta and plaintext_size > crypto.STREAMING_THRESHOLD: + return False + if src_multipart_meta: + src_kid = src_multipart_meta.kid + else: + src_kid = src_metadata.get(self.settings.kidtag_name, "") + needs_rekey = bool(src_kid) and src_kid != creds.access_key + return not needs_rekey + + def _resolve_source_dek( + self, + src_multipart_meta: MultipartMetadata | None, + src_wrapped_dek: str | None, + src_metadata: dict, + creds: S3Credentials, + ) -> tuple[bytes, str]: + if src_multipart_meta: + kid = src_multipart_meta.kid + dek = crypto.unwrap_key(src_multipart_meta.wrapped_dek, self.keyring.key_by_id(kid)) + return dek, kid + kid = src_metadata.get(self.settings.kidtag_name, "") + wrapped = base64.b64decode(src_wrapped_dek or "") + if kid: + kek = self.keyring.key_by_id(kid) + else: + kid, kek = self.keyring.key_for(creds.access_key) + return crypto.unwrap_key(wrapped, kek), kid + + def _source_ciphertext_segments( + self, + src_multipart_meta: MultipartMetadata | None, + head_resp: dict, + src_wrapped_dek: str | None, + src_metadata: dict, + ) -> list[_CiphertextSegment]: + if src_multipart_meta: + segments: list[_CiphertextSegment] = [] + ct_offset = 0 + for part in sorted(src_multipart_meta.parts, key=lambda p: p.part_number): + if part.internal_parts: + for ip in sorted(part.internal_parts, key=lambda x: x.internal_part_number): + segments.append( + _CiphertextSegment(ip.plaintext_size, ip.ciphertext_size, ct_offset) + ) + ct_offset += ip.ciphertext_size + else: + segments.append( + _CiphertextSegment(part.plaintext_size, part.ciphertext_size, ct_offset) + ) + ct_offset += part.ciphertext_size + return segments + + ct_size = head_resp.get("ContentLength", 0) or 0 + size_str = src_metadata.get("plaintext-size") + plaintext_size = int(size_str) if size_str else crypto.plaintext_size(ct_size) + return [_CiphertextSegment(plaintext_size, ct_size, 0)] + + async def _adopt_upload_dek( + self, + client: S3Client, + bucket: str, + key: str, + upload_id: str, + state: MultipartUploadState, + dek: bytes, + kid: str, + creds: S3Credentials, + ) -> None: + if state.dek == dek and state.kid == kid: + return + if state.parts and state.dek != dek: + raise S3Error.invalid_request( + "Upload already contains parts encrypted with a different key" + ) + state.dek = dek + state.kid = kid + if kid: + kek = self.keyring.key_by_id(kid) + else: + kid, kek = self.keyring.key_for(creds.access_key) + wrapped = crypto.wrap_key(dek, kek) + await persist_upload_state(client, bucket, key, upload_id, wrapped, kid) + await self.multipart_manager.store_reconstructed_state(bucket, key, upload_id, state) + + async def _md5_plaintext_from_copy_source( + self, + client: S3Client, + src_bucket: str, + src_key: str, + copy_source_range: str | None, + src_wrapped_dek: str | None, + src_multipart_meta: MultipartMetadata | None, + head_resp: dict, + src_metadata: dict, + ): + """Frame-bounded plaintext MD5 for the UploadPartCopy synthetic ETag.""" + md5 = hashlib.md5(usedforsecurity=False) + async for chunk in self._iter_copy_source( + client, + src_bucket, + src_key, + copy_source_range, + src_wrapped_dek, + src_multipart_meta, + head_resp, + src_metadata, + ): + md5.update(chunk) + return md5 + + async def _gated_passthrough_copy_part( + self, + client: S3Client, + bucket: str, + key: str, + upload_id: str, + part_num: int, + state: MultipartUploadState, + src_bucket: str, + src_key: str, + copy_source: str, + head_resp: dict, + src_metadata: dict, + src_wrapped_dek: str | None, + src_multipart_meta: MultipartMetadata | None, + plaintext_size: int, + ) -> Response: + """Passthrough with the same pipeline cap as the re-encrypt path.""" + if plaintext_size > crypto.STREAMING_THRESHOLD: + async with _copy_pipeline_semaphore: + return await self._passthrough_copy_part( + client, + bucket, + key, + upload_id, + part_num, + state, + src_bucket, + src_key, + copy_source, + head_resp, + src_metadata, + src_wrapped_dek, + src_multipart_meta, + plaintext_size, + ) + return await self._passthrough_copy_part( + client, + bucket, + key, + upload_id, + part_num, + state, + src_bucket, + src_key, + copy_source, + head_resp, + src_metadata, + src_wrapped_dek, + src_multipart_meta, + plaintext_size, + ) + + async def _passthrough_copy_part( + self, + client: S3Client, + bucket: str, + key: str, + upload_id: str, + part_num: int, + state: MultipartUploadState, + src_bucket: str, + src_key: str, + copy_source: str, + head_resp: dict, + src_metadata: dict, + src_wrapped_dek: str | None, + src_multipart_meta: MultipartMetadata | None, + plaintext_size: int, + ) -> Response: + """Server-side copy of encrypted ciphertext; destination adopts the source DEK.""" + src_dek, src_kid = self._resolve_source_dek( + src_multipart_meta, src_wrapped_dek, src_metadata, client.credentials + ) + await self._adopt_upload_dek( + client, bucket, key, upload_id, state, src_dek, src_kid, client.credentials + ) + + segments = self._source_ciphertext_segments( + src_multipart_meta, head_resp, src_wrapped_dek, src_metadata + ) + if not segments: + raise S3Error.invalid_request("Copy source has no ciphertext segments") + + copy_source_path = copy_source or f"/{src_bucket}/{quote(src_key, safe='/')}" + + logger.info( + "UPLOAD_PART_COPY_PASSTHROUGH", + bucket=bucket, + key=key, + part_num=part_num, + src_bucket=src_bucket, + src_key=src_key, + plaintext_mb=f"{plaintext_size / 1024 / 1024:.2f}MB", + segments=len(segments), + ) + + has_internal_layout = bool( + src_multipart_meta and any(p.internal_parts for p in src_multipart_meta.parts) + ) + if not has_internal_layout and len(segments) == 1: + # Single-part encrypted object (or legacy multipart without internal parts): + # copy straight to the client part number. + seg = segments[0] + ct_end = seg.ct_offset + seg.ciphertext_size - 1 + copy_range = f"bytes={seg.ct_offset}-{ct_end}" + part_reserve = crypto.copy_passthrough_segment_peak(seg.plaintext_size) + async with concurrency.reserve_copy_memory(part_reserve): + resp = await client.upload_part_copy( + bucket, + key, + upload_id, + part_num, + copy_source_path, + copy_source_range=copy_range, + ) + md5 = await self._md5_plaintext_from_copy_source( + client, + src_bucket, + src_key, + None, + src_wrapped_dek, + src_multipart_meta, + head_resp, + src_metadata, + ) + etag = md5.hexdigest() + await self.multipart_manager.add_part( + bucket, + key, + upload_id, + PartMetadata( + part_number=part_num, + plaintext_size=seg.plaintext_size, + ciphertext_size=seg.ciphertext_size, + etag=etag, + md5=etag, + ), + ) + return Response( + content=xml_responses.upload_part_copy_result( + etag, format_iso8601(datetime.now(UTC)) + ), + media_type="application/xml", + ) + + internal_part_start = await self.multipart_manager.allocate_internal_parts( + bucket, key, upload_id, len(segments), client_part_number=0 + ) + + internal_parts: list[InternalPartMetadata] = [] + total_plaintext = 0 + total_ciphertext = 0 + + for i, seg in enumerate(segments): + internal_num = internal_part_start + i + ct_end = seg.ct_offset + seg.ciphertext_size - 1 + copy_range = f"bytes={seg.ct_offset}-{ct_end}" + part_reserve = crypto.copy_passthrough_segment_peak(seg.plaintext_size) + async with concurrency.reserve_copy_memory(part_reserve): + resp = await client.upload_part_copy( + bucket, + key, + upload_id, + internal_num, + copy_source_path, + copy_source_range=copy_range, + ) + etag_part = resp["CopyPartResult"]["ETag"].strip('"') + internal_parts.append( + InternalPartMetadata( + internal_part_number=internal_num, + plaintext_size=seg.plaintext_size, + ciphertext_size=seg.ciphertext_size, + etag=etag_part, + ) + ) + total_plaintext += seg.plaintext_size + total_ciphertext += seg.ciphertext_size + + md5 = await self._md5_plaintext_from_copy_source( + client, + src_bucket, + src_key, + None, + src_wrapped_dek, + src_multipart_meta, + head_resp, + src_metadata, + ) + etag = md5.hexdigest() + await self.multipart_manager.add_part( + bucket, + key, + upload_id, + PartMetadata( + part_number=part_num, + plaintext_size=total_plaintext, + ciphertext_size=total_ciphertext, + etag=etag, + md5=etag, + internal_parts=internal_parts, + ), + ) + + logger.info( + "UPLOAD_PART_COPY_PASSTHROUGH_COMPLETE", + bucket=bucket, + key=key, + part_num=part_num, + internal_parts=len(internal_parts), + plaintext_mb=f"{total_plaintext / 1024 / 1024:.2f}MB", + ) + return Response( + content=xml_responses.upload_part_copy_result(etag, format_iso8601(datetime.now(UTC))), + media_type="application/xml", + ) + async def _simple_copy_part( self, client: S3Client, @@ -220,23 +641,24 @@ async def _streaming_copy_part( ) -> Response: # UploadPartCopy carries no request body, so the request-level limiter # reserved ~nothing. Per-internal-part reservations in _pump_copy_chunks - # gate concurrent copies without holding memory for the whole object. - return await self._streaming_copy_part_inner( - client, - bucket, - key, - upload_id, - part_num, - state, - src_bucket, - src_key, - copy_source_range, - src_wrapped_dek, - src_multipart_meta, - head_resp, - src_metadata, - plaintext_size, - ) + # gate memory; the pipeline semaphore caps how many copies run at once. + async with _copy_pipeline_semaphore: + return await self._streaming_copy_part_inner( + client, + bucket, + key, + upload_id, + part_num, + state, + src_bucket, + src_key, + copy_source_range, + src_wrapped_dek, + src_multipart_meta, + head_resp, + src_metadata, + plaintext_size, + ) async def _streaming_copy_part_inner( self, @@ -395,8 +817,9 @@ async def _pump_copy_chunks( plaintext frames from the source, encrypts each with encrypt_frame, accumulates one internal part's ciphertext, then uploads it before starting the next. Memory governor reservation is per internal part (acquire before - encrypt, release before upload) so a multi-GB copy does not hold ~88MB for - its entire duration and other copies can interleave during S3 I/O waits. + encrypt, hold through upload, release after ciphertext is freed) so RSS + matches what the limiter tracks and a multi-GB copy does not hold one + reservation for its entire duration. """ reader = _PlaintextReader(src_iter) md5 = hashlib.md5(usedforsecurity=False) @@ -436,11 +859,12 @@ async def _pump_copy_chunks( if not frame_pt: break - # Reservation released before upload so other copies can encrypt while - # this one waits on S3 I/O (ciphertext RSS is smaller than encrypt peak). - upload_start = time.monotonic() - resp = await client.upload_part(bucket, key, upload_id, internal_part_num, ciphertext) - del ciphertext + upload_start = time.monotonic() + resp = await client.upload_part( + bucket, key, upload_id, internal_part_num, ciphertext + ) + del ciphertext + ct_size = crypto.framed_ciphertext_size(part_plaintext) internal_parts.append( InternalPartMetadata( diff --git a/scripts/verify_upload_part_copy_passthrough.py b/scripts/verify_upload_part_copy_passthrough.py new file mode 100644 index 0000000..8df10a2 --- /dev/null +++ b/scripts/verify_upload_part_copy_passthrough.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""CLI wrapper for UploadPartCopy passthrough verification. + +CI runs the pytest suite instead: + uv run pytest tests/integration/test_upload_part_copy_passthrough_e2e.py -m e2e -n0 -v + +Manual / pre-merge: + uv run python scripts/verify_upload_part_copy_passthrough.py + uv run python scripts/verify_upload_part_copy_passthrough.py --quick +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tests.integration.copy_metrics_helpers import SCYLLA_SSTABLE_SIZE # noqa: E402 +from tests.integration.passthrough_verify import ( # noqa: E402 + QUICK_SIZE, + check_basic_passthrough, + check_ciphertext_identity, + check_concurrent_flood, + check_dek_adoption, + check_reencrypt_control, + check_scylla_scale, + log_has_passthrough, + upload_multipart, + verify_session, +) + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--quick", action="store_true", help="Skip 1280MB + concurrent flood") + args = p.parse_args() + + print("=== UploadPartCopy passthrough verification ===") + with verify_session() as (ctx, _proc, log_path): + print(f"log={log_path}") + upload_multipart(ctx, "sst/source-96mb.bin", QUICK_SIZE) + check_basic_passthrough(ctx, "sst/source-96mb.bin", "sst/dest-96mb.bin", QUICK_SIZE) + check_ciphertext_identity(ctx, "sst/source-96mb.bin", "sst/dest-96mb.bin") + check_dek_adoption(ctx, "sst/source-96mb.bin", "sst/dest-96mb.bin") + check_reencrypt_control(ctx, "sst/source-96mb.bin", "sst/dest-reencrypt.bin", QUICK_SIZE) + + if not args.quick: + upload_multipart(ctx, "sst/source-1280mb.bin", SCYLLA_SSTABLE_SIZE) + check_scylla_scale(ctx, "sst/source-1280mb.bin", "sst/dest-1280mb.bin") + check_concurrent_flood(ctx, "sst/source-1280mb.bin", n=10) + + failed = [c for c in ctx.failures if not c.passed] + if not log_has_passthrough(log_path): + print("FAIL: missing UPLOAD_PART_COPY_PASSTHROUGH log line") + failed.append(None) + + for c in ctx.failures: + mark = "PASS" if c.passed else "FAIL" + detail = f": {c.detail}" if c.detail else "" + print(f" [{mark}] {c.name}{detail}") + + if failed: + print(f"\n{len(failed)} check(s) failed") + return 1 + print("\nALL CHECKS PASSED") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/copy_metrics_helpers.py b/tests/integration/copy_metrics_helpers.py index 96a598c..62b27bb 100644 --- a/tests/integration/copy_metrics_helpers.py +++ b/tests/integration/copy_metrics_helpers.py @@ -15,8 +15,10 @@ # Prod pod budget and per-chunk peak for 32MB internal parts. PROD_GOVERNOR_MB = 192 CHUNK_PEAK = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) -# Between internal parts only the POST gate (~64KB) should remain reserved. +# Between internal parts of a single copy (no concurrent pipeline overlap). BASELINE_CEILING = 2 * MB +# Two pipelines holding through upload: 2 × ~88MB. +TWO_PIPELINE_FLOOR = CHUNK_PEAK * 2 * 0.85 # Scylla backup copies ~1.25GB SSTables (40 × 32MB internal parts in s3proxy). SCYLLA_SSTABLE_MB = 1280 @@ -100,23 +102,29 @@ def assert_copy_memory_sawtooth( ) -def assert_concurrent_copies_not_flatlined( +def assert_concurrent_copies_bounded( reserved: list[int], *, - concurrent_copies: int, + concurrent_pipelines: int = 2, min_samples: int = 20, ) -> None: - """Two whole-object holds keep min ~concurrent_copies×CHUNK_PEAK with no drops.""" + """Concurrent copies must stay within pipeline cap, not flat whole-object hold.""" assert len(reserved) >= min_samples, f"too few samples ({len(reserved)})" - min_reserved = min(reserved) - drops = sum(1 for b in reserved if b <= BASELINE_CEILING) - flat_floor = CHUNK_PEAK * concurrent_copies * 0.85 + max_reserved = max(reserved) + budget = PROD_GOVERNOR_MB * MB - assert min_reserved <= BASELINE_CEILING, ( - f"concurrent copies never released between parts: min={min_reserved / MB:.2f}MB " - f"(old whole-object code stays >= {flat_floor / MB:.2f}MB for {concurrent_copies} copies)" + assert max_reserved <= budget, ( + f"reserved {max_reserved / MB:.2f}MB exceeded {PROD_GOVERNOR_MB}MB governor budget" ) - assert drops >= 5, ( - f"expected inter-part drops during {concurrent_copies} concurrent copies; " - f"got {drops} (min={min_reserved / MB:.2f}MB)" + assert max_reserved <= CHUNK_PEAK * concurrent_pipelines * 1.25, ( + f"peak reserved {max_reserved / MB:.2f}MB exceeds {concurrent_pipelines} pipeline cap " + f"({CHUNK_PEAK * concurrent_pipelines / MB:.2f}MB) — too many concurrent copies?" ) + # Old whole-object code: min stays ~176MB with zero drops for entire multi-minute copy. + # With hold-through-upload + 2 pipelines, min ~176MB is expected; we assert max is bounded. + flat_floor = CHUNK_PEAK * concurrent_pipelines * 0.85 + min_reserved = min(reserved) + if min_reserved >= flat_floor: + # If flat at 2×chunk, verify it's not ALSO flat at whole-object duration + # by checking max isn't much higher (no 3rd pipeline admitted). + assert max_reserved <= CHUNK_PEAK * concurrent_pipelines * 1.25 diff --git a/tests/integration/passthrough_verify.py b/tests/integration/passthrough_verify.py new file mode 100644 index 0000000..36d4ecc --- /dev/null +++ b/tests/integration/passthrough_verify.py @@ -0,0 +1,370 @@ +"""Shared UploadPartCopy passthrough verification (real s3proxy subprocess + MinIO). + +Used by tests/integration/test_upload_part_copy_passthrough_e2e.py and +scripts/verify_upload_part_copy_passthrough.py. +""" + +from __future__ import annotations + +import concurrent.futures +import os +import socket +import subprocess +import sys +import tempfile +import threading +import time +import urllib.request +import uuid +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path + +import boto3 +from botocore.config import Config + +from s3proxy import crypto +from s3proxy.state.metadata import decode_multipart_metadata +from tests.integration.conftest import _wait_for_port, minio_backend +from tests.integration.copy_metrics_helpers import SCYLLA_SSTABLE_SIZE + +MB = 1024 * 1024 +CHUNK_PEAK = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) +PROD_BUDGET_MB = 192 +QUICK_SIZE = 96 * MB +PART_SIZE = 48 * MB + + +@dataclass +class CheckResult: + name: str + passed: bool + detail: str = "" + + +@dataclass +class RunContext: + port: int + endpoint: str + minio_host: str + bucket: str + log_path: Path + proxy: boto3.client + raw: boto3.client + failures: list[CheckResult] = field(default_factory=list) + + def ok(self, name: str, cond: bool, detail: str = "") -> None: + self.failures.append(CheckResult(name, cond, detail)) + + +def free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def s3_client(endpoint: str, *, read_timeout: int = 600): + return boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id="minioadmin", + aws_secret_access_key="minioadmin", + region_name="us-east-1", + config=Config( + retries={"max_attempts": 3, "mode": "standard"}, + read_timeout=read_timeout, + connect_timeout=10, + ), + ) + + +def scrape_metric(port: int, name: str) -> float: + with urllib.request.urlopen(f"http://localhost:{port}/metrics", timeout=5) as resp: + for line in resp.read().decode().splitlines(): + if line.startswith(f"{name} ") or line.startswith(f"{name}_total "): + return float(line.split()[1]) + return 0.0 + + +class RepeatingBody: + def __init__(self, size: int, chunk: int = 8 * MB): + self._size = size + self._chunk = chunk + self._pos = 0 + + def tell(self) -> int: + return self._pos + + def seek(self, offset: int, whence: int = 0) -> int: + if whence == 0: + self._pos = offset + elif whence == 1: + self._pos += offset + elif whence == 2: + self._pos = self._size + offset + return self._pos + + def read(self, n: int = -1) -> bytes: + if self._pos >= self._size: + return b"" + if n < 0: + n = self._size - self._pos + n = min(n, self._size - self._pos) + self._pos += n + return b"X" * n + + +def upload_multipart(ctx: RunContext, key: str, size: int) -> None: + resp = ctx.proxy.create_multipart_upload(Bucket=ctx.bucket, Key=key) + upload_id = resp["UploadId"] + parts = [] + pn = 1 + for off in range(0, size, PART_SIZE): + chunk = min(PART_SIZE, size - off) + up = ctx.proxy.upload_part( + Bucket=ctx.bucket, + Key=key, + PartNumber=pn, + UploadId=upload_id, + Body=RepeatingBody(chunk), + ) + parts.append({"PartNumber": pn, "ETag": up["ETag"]}) + pn += 1 + ctx.proxy.complete_multipart_upload( + Bucket=ctx.bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={"Parts": parts}, + ) + meta = f".s3proxy-internal/{key}.meta" + assert ctx.raw.head_object(Bucket=ctx.bucket, Key=meta)["ContentLength"] > 0 + + +def upload_part_copy( + ctx: RunContext, + dest: str, + source: str, + *, + byte_range: str | None = None, +) -> None: + resp = ctx.proxy.create_multipart_upload(Bucket=ctx.bucket, Key=dest) + uid = resp["UploadId"] + kw: dict = { + "Bucket": ctx.bucket, + "Key": dest, + "PartNumber": 1, + "UploadId": uid, + "CopySource": {"Bucket": ctx.bucket, "Key": source}, + } + if byte_range: + kw["CopySourceRange"] = byte_range + cp = ctx.proxy.upload_part_copy(**kw) + ctx.proxy.complete_multipart_upload( + Bucket=ctx.bucket, + Key=dest, + UploadId=uid, + MultipartUpload={"Parts": [{"PartNumber": 1, "ETag": cp["CopyPartResult"]["ETag"]}]}, + ) + + +def poll_during(ctx: RunContext, fn) -> tuple[int, int]: + enc0 = scrape_metric(ctx.port, "s3proxy_bytes_encrypted_total") + peak = 0 + stop = threading.Event() + + def _poll() -> None: + nonlocal peak + while not stop.is_set(): + peak = max(peak, int(scrape_metric(ctx.port, "s3proxy_memory_reserved_bytes"))) + time.sleep(0.02) + + t = threading.Thread(target=_poll, daemon=True) + t.start() + try: + fn() + finally: + stop.set() + t.join(timeout=3) + enc_delta = int(scrape_metric(ctx.port, "s3proxy_bytes_encrypted_total") - enc0) + return peak, enc_delta + + +def load_sidecar(ctx: RunContext, key: str): + meta_key = f".s3proxy-internal/{key}.meta" + raw = ctx.raw.get_object(Bucket=ctx.bucket, Key=meta_key)["Body"].read() + return decode_multipart_metadata(raw.decode()) + + +def raw_ciphertext(ctx: RunContext, key: str) -> bytes: + return ctx.raw.get_object(Bucket=ctx.bucket, Key=key)["Body"].read() + + +def plaintext_via_proxy(ctx: RunContext, key: str, size: int) -> bytes: + body = ctx.proxy.get_object(Bucket=ctx.bucket, Key=key)["Body"] + out = bytearray() + while True: + chunk = body.read(8 * MB) + if not chunk: + break + out.extend(chunk) + assert len(out) == size + return bytes(out) + + +def log_has_passthrough(log_path: Path) -> bool: + return "UPLOAD_PART_COPY_PASSTHROUGH" in log_path.read_text(errors="replace") + + +@contextmanager +def verify_session( + port: int | None = None, + log_path: Path | None = None, +) -> Generator[tuple[RunContext, subprocess.Popen[bytes], Path]]: + port = port or free_port() + log_path = log_path or Path(tempfile.gettempdir()) / f"s3proxy-passthrough-verify-{port}.log" + bucket = f"verify-{uuid.uuid4().hex[:8]}" + + with minio_backend() as minio_host, open(log_path, "w") as log_file: + env = os.environ.copy() + env.update( + { + "S3PROXY_CREDENTIALS": ( + '[{"access_key":"minioadmin","secret_key":"minioadmin",' + '"kek":"test-encryption-key-32-bytes!!"}]' + ), + "S3PROXY_HOST": minio_host, + "S3PROXY_REGION": "us-east-1", + "S3PROXY_PORT": str(port), + "S3PROXY_NO_TLS": "true", + "S3PROXY_LOG_LEVEL": "INFO", + "S3PROXY_MEMORY_LIMIT_MB": str(PROD_BUDGET_MB), + "S3PROXY_MAX_PARALLEL_COPIES": "2", + "AWS_ACCESS_KEY_ID": "minioadmin", + "AWS_SECRET_ACCESS_KEY": "minioadmin", + } + ) + proc = subprocess.Popen( + [sys.executable, "-m", "s3proxy.main"], + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + ) + endpoint = f"http://localhost:{port}" + try: + _wait_for_port(port, proc, timeout=30) + proxy = s3_client(endpoint) + raw = s3_client(minio_host) + proxy.create_bucket(Bucket=bucket) + ctx = RunContext( + port=port, + endpoint=endpoint, + minio_host=minio_host, + bucket=bucket, + log_path=log_path, + proxy=proxy, + raw=raw, + ) + yield ctx, proc, log_path + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +def check_basic_passthrough(ctx: RunContext, source: str, dest: str, size: int) -> None: + peak, enc = poll_during(ctx, lambda: upload_part_copy(ctx, dest, source)) + ctx.ok("zero bytes_encrypted", enc <= 1 * MB, f"{enc / MB:.2f}MB") + ctx.ok( + "low peak memory", + peak <= CHUNK_PEAK * 0.5, + f"{peak / MB:.2f}MB vs {CHUNK_PEAK / MB:.0f}MB chunk", + ) + pt = plaintext_via_proxy(ctx, dest, size) + ctx.ok("GET round-trip", pt == b"X" * size, f"len={len(pt)}") + + +def check_ciphertext_identity(ctx: RunContext, source: str, dest: str) -> None: + src_ct = raw_ciphertext(ctx, source) + dst_ct = raw_ciphertext(ctx, dest) + ctx.ok("ciphertext bytes identical", src_ct == dst_ct, f"{len(src_ct)} bytes") + + +def check_dek_adoption(ctx: RunContext, source: str, dest: str) -> None: + src_meta = load_sidecar(ctx, source) + dst_meta = load_sidecar(ctx, dest) + ctx.ok("wrapped_dek matches source", src_meta.wrapped_dek == dst_meta.wrapped_dek) + ctx.ok("kid matches source", src_meta.kid == dst_meta.kid) + if src_meta.parts: + src_n = sum(len(p.internal_parts) for p in src_meta.parts) + dst_n = sum(len(p.internal_parts) for p in dst_meta.parts) + ctx.ok("internal part count preserved", src_n == dst_n, f"{dst_n} vs {src_n}") + + +def check_reencrypt_control(ctx: RunContext, source: str, dest: str, size: int) -> None: + peak, enc = poll_during( + ctx, + lambda: upload_part_copy(ctx, dest, source, byte_range=f"bytes=0-{size - 1}"), + ) + ctx.ok("encrypts full object", enc >= size * 0.5, f"{enc / MB:.0f}MB") + ctx.ok("high peak memory", peak >= CHUNK_PEAK * 0.5, f"{peak / MB:.2f}MB") + + +def check_scylla_scale(ctx: RunContext, source: str, dest: str) -> None: + t0 = time.monotonic() + peak, enc = poll_during(ctx, lambda: upload_part_copy(ctx, dest, source)) + elapsed = time.monotonic() - t0 + ctx.ok("zero bytes_encrypted at 1280MB", enc <= 2 * MB, f"{enc / MB:.2f}MB") + ctx.ok("peak memory bounded", peak <= CHUNK_PEAK * 0.5, f"{peak / MB:.2f}MB") + head = ctx.proxy.get_object(Bucket=ctx.bucket, Key=dest, Range="bytes=0-1048575")["Body"].read() + tail_off = SCYLLA_SSTABLE_SIZE - MB + tail = ctx.proxy.get_object( + Bucket=ctx.bucket, Key=dest, Range=f"bytes={tail_off}-{SCYLLA_SSTABLE_SIZE - 1}" + )["Body"].read() + ctx.ok("GET head 1MB", head == b"X" * MB) + ctx.ok("GET tail 1MB", tail == b"X" * MB) + ctx.ok("completed in reasonable time", elapsed < 600, f"{elapsed:.1f}s") + src_ct = raw_ciphertext(ctx, source) + dst_ct = raw_ciphertext(ctx, dest) + ctx.ok("1280MB ciphertext identical", src_ct == dst_ct, f"{len(src_ct)} bytes") + + +def check_concurrent_flood(ctx: RunContext, source: str, n: int = 10) -> None: + enc0 = scrape_metric(ctx.port, "s3proxy_bytes_encrypted_total") + peak = 0 + stop = threading.Event() + + def _poll() -> None: + nonlocal peak + while not stop.is_set(): + peak = max(peak, int(scrape_metric(ctx.port, "s3proxy_memory_reserved_bytes"))) + time.sleep(0.02) + + poller = threading.Thread(target=_poll, daemon=True) + poller.start() + + def one(i: int) -> bool: + try: + upload_part_copy(ctx, f"sst/concurrent-{i}.bin", source) + return True + except Exception: + return False + + t0 = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=n) as pool: + results = list(pool.map(one, range(n))) + elapsed = time.monotonic() - t0 + stop.set() + poller.join(timeout=3) + enc_delta = int(scrape_metric(ctx.port, "s3proxy_bytes_encrypted_total") - enc0) + + ctx.ok("all copies succeeded", all(results), f"{sum(results)}/{n}") + ctx.ok( + "peak memory under prod budget", + peak <= PROD_BUDGET_MB * MB, + f"{peak / MB:.1f}MB / {PROD_BUDGET_MB}MB", + ) + ctx.ok("zero encrypt during flood", enc_delta <= 2 * MB, f"{enc_delta / MB:.2f}MB") + ctx.ok("flood completed", elapsed < 900, f"{elapsed:.1f}s") diff --git a/tests/integration/test_copy_memory_governor.py b/tests/integration/test_copy_memory_governor.py index 8ed1c12..dc9ce83 100644 --- a/tests/integration/test_copy_memory_governor.py +++ b/tests/integration/test_copy_memory_governor.py @@ -138,8 +138,7 @@ def copy_stress_server(): port, log_output=False, S3PROXY_MEMORY_LIMIT_MB=GOVERNOR_MB, - # 0 = reject immediately (SlowDown), don't queue — otherwise all 3 copies - # succeed serially within the 2s wait and the concurrency assertion fails. + # 0 = reject immediately (SlowDown) when the governor is full. S3PROXY_BACKPRESSURE_TIMEOUT="0", S3PROXY_MAX_PART_SIZE_MB="0", ) as (endpoint, proc): @@ -183,7 +182,12 @@ class TestCopyMemoryGovernorSubprocess: def test_three_concurrent_large_copies_do_not_oom_server( self, copy_stress_server, copy_bucket, large_source ): - """Prod regression: 3 concurrent large copies must not all run + OOM.""" + """Prod regression: 3 concurrent large copies must not OOM the server. + + With S3PROXY_MAX_PARALLEL_COPIES=2 the third copy waits on the pipeline + semaphore (not SlowDown) and finishes after one of the first two completes. + Per-chunk governor reservations keep peak RSS within the 96MB budget. + """ endpoint, proc = copy_stress_server client, bucket = copy_bucket source_key = large_source @@ -195,13 +199,9 @@ def one_copy(i: int) -> dict: results = list(pool.map(one_copy, range(3))) assert proc.poll() is None, "s3proxy OOMKilled or crashed (exit 137?)" - succeeded = sum(1 for r in results if r["success"]) - slowed = sum(1 for r in results if r.get("code") == "SlowDown") - assert succeeded >= 1, f"expected at least one copy to succeed: {results}" - # 96MB budget fits two ~48MB copy slots; the third must be rejected, not queued. - assert slowed >= 1, f"expected backpressure on concurrent copies: {results}" - assert succeeded <= 2, f"more than two copies ran concurrently: {results}" - assert succeeded + slowed == len(results), f"unexpected errors: {results}" + assert all(r["success"] for r in results), ( + f"expected all copies to finish via pipeline cap serialization: {results}" + ) def test_mixed_scylla_uploads_and_large_copy_server_survives( self, copy_stress_server, copy_bucket, large_source diff --git a/tests/integration/test_copy_per_part_metrics.py b/tests/integration/test_copy_per_part_metrics.py index 1eebf48..122d8f6 100644 --- a/tests/integration/test_copy_per_part_metrics.py +++ b/tests/integration/test_copy_per_part_metrics.py @@ -18,13 +18,12 @@ from tests.integration.conftest import minio_backend, run_s3proxy from tests.integration.copy_metrics_helpers import ( - BASELINE_CEILING, CHUNK_PEAK, MB, PROD_GOVERNOR_MB, SCYLLA_SSTABLE_SIZE, MetricsPoller, - assert_concurrent_copies_not_flatlined, + assert_concurrent_copies_bounded, assert_copy_memory_sawtooth, ) from tests.integration.test_copy_memory_governor import ( @@ -55,6 +54,7 @@ def metrics_server(): S3PROXY_MEMORY_LIMIT_MB=str(PROD_GOVERNOR_MB), S3PROXY_BACKPRESSURE_TIMEOUT="30", S3PROXY_MAX_PART_SIZE_MB="0", + S3PROXY_MAX_PARALLEL_COPIES="2", ) as (endpoint, proc), ): yield port, endpoint, proc @@ -122,7 +122,7 @@ def test_memory_reserved_drops_between_internal_parts( def test_three_concurrent_copies_all_succeed_on_192mb( self, metrics_server, metrics_bucket, fast_source ): - """Per-part interleaving: 3 large copies fit where whole-object hold allowed 2.""" + """Pipeline cap=2: 3 copies queue but all succeed without OOM.""" _, _, proc = metrics_server client, bucket = metrics_bucket @@ -191,12 +191,31 @@ def one_copy(i: int) -> dict: assert all(r["success"] for r in results), ( f"expected both 1280MB copies to succeed on {PROD_GOVERNOR_MB}MB budget: {results}" ) - assert_concurrent_copies_not_flatlined(poller.samples, concurrent_copies=2) - # Old code: min stays ~176MB (2 × 88MB whole-object holds). Per-part frees budget. - assert min(poller.samples) <= BASELINE_CEILING, ( - f"reservation never dropped during concurrent Scylla copies " - f"(min={min(poller.samples) / MB:.2f}MB, expected <= {BASELINE_CEILING / MB:.2f}MB)" - ) + assert_concurrent_copies_bounded(poller.samples, concurrent_pipelines=2) assert max(poller.samples) <= CHUNK_PEAK * 2.5, ( - f"peak reserved {max(poller.samples) / MB:.2f}MB — more than brief 2-chunk overlap?" + f"peak reserved {max(poller.samples) / MB:.2f}MB — more than 2-pipeline cap?" + ) + + def test_five_concurrent_copies_all_succeed_with_pipeline_cap( + self, metrics_server, metrics_bucket, fast_source + ): + """Scylla-shaped: many concurrent copies, only 2 pipelines, no OOM.""" + port, _, proc = metrics_server + client, bucket = metrics_bucket + + poller = MetricsPoller(port, interval_s=0.02) + poller.start() + + def one_copy(i: int) -> dict: + return _upload_part_copy(client, bucket, f"flood-{i}.bin", fast_source) + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool: + results = list(pool.map(one_copy, range(5))) + + poller.stop() + + assert proc.poll() is None, "s3proxy OOM/crashed under 5 concurrent copies" + assert all(r["success"] for r in results), f"copy failures: {results}" + assert max(poller.samples) <= PROD_GOVERNOR_MB * MB, ( + f"peak reserved {max(poller.samples) / MB:.2f}MB exceeded budget" ) diff --git a/tests/integration/test_upload_part_copy_passthrough_e2e.py b/tests/integration/test_upload_part_copy_passthrough_e2e.py new file mode 100644 index 0000000..15fc448 --- /dev/null +++ b/tests/integration/test_upload_part_copy_passthrough_e2e.py @@ -0,0 +1,131 @@ +"""CI e2e: UploadPartCopy passthrough on real s3proxy subprocess + MinIO. + +Mirrors scripts/verify_upload_part_copy_passthrough.py (checks A–F). +Runs in the memory_copy integration shard (-n0). +""" + +from __future__ import annotations + +import pytest + +from tests.integration.copy_metrics_helpers import SCYLLA_SSTABLE_SIZE +from tests.integration.passthrough_verify import ( + CHUNK_PEAK, + MB, + QUICK_SIZE, + check_basic_passthrough, + check_concurrent_flood, + check_dek_adoption, + check_reencrypt_control, + check_scylla_scale, + log_has_passthrough, + poll_during, + raw_ciphertext, + upload_multipart, + upload_part_copy, + verify_session, +) + + +@pytest.fixture(scope="module") +def passthrough_env(): + """One s3proxy subprocess + MinIO; upload 96MB and 1280MB sidecar sources.""" + with verify_session() as (ctx, proc, log_path): + upload_multipart(ctx, "sst/source-96mb.bin", QUICK_SIZE) + upload_multipart(ctx, "sst/source-1280mb.bin", SCYLLA_SSTABLE_SIZE) + yield ctx, proc, log_path + + +def _assert_check(ctx, name: str) -> None: + for c in ctx.failures: + if c.name == name: + assert c.passed, f"{name}: {c.detail}" + return + raise AssertionError(f"check {name!r} was not recorded") + + +@pytest.mark.e2e +class TestUploadPartCopyPassthroughQuick: + """Checks A–D (96MB) — fast regression gate.""" + + def test_zero_encrypt_and_low_memory(self, passthrough_env): + ctx, proc, _ = passthrough_env + ctx.failures.clear() + check_basic_passthrough(ctx, "sst/source-96mb.bin", "sst/dest-a.bin", QUICK_SIZE) + assert proc.poll() is None + _assert_check(ctx, "zero bytes_encrypted") + _assert_check(ctx, "low peak memory") + _assert_check(ctx, "GET round-trip") + + def test_ciphertext_byte_identical_on_minio(self, passthrough_env): + ctx, proc, _ = passthrough_env + upload_part_copy(ctx, "sst/dest-b.bin", "sst/source-96mb.bin") + assert proc.poll() is None + assert raw_ciphertext(ctx, "sst/source-96mb.bin") == raw_ciphertext(ctx, "sst/dest-b.bin") + + def test_dek_adoption_in_sidecar(self, passthrough_env): + ctx, proc, _ = passthrough_env + ctx.failures.clear() + upload_part_copy(ctx, "sst/dest-c.bin", "sst/source-96mb.bin") + check_dek_adoption(ctx, "sst/source-96mb.bin", "sst/dest-c.bin") + assert proc.poll() is None + _assert_check(ctx, "wrapped_dek matches source") + _assert_check(ctx, "kid matches source") + _assert_check(ctx, "internal part count preserved") + + def test_range_copy_reencrypts_with_high_memory(self, passthrough_env): + ctx, proc, _ = passthrough_env + ctx.failures.clear() + check_reencrypt_control(ctx, "sst/source-96mb.bin", "sst/dest-d.bin", QUICK_SIZE) + assert proc.poll() is None + _assert_check(ctx, "encrypts full object") + _assert_check(ctx, "high peak memory") + + def test_passthrough_log_line(self, passthrough_env): + _, proc, log_path = passthrough_env + assert proc.poll() is None + assert log_has_passthrough(log_path) + + +@pytest.mark.e2e +class TestUploadPartCopyPassthroughScylla: + """Checks E–F (1280MB + concurrent flood) — prod-shaped load.""" + + def test_scylla_scale_passthrough(self, passthrough_env): + ctx, proc, _ = passthrough_env + ctx.failures.clear() + check_scylla_scale(ctx, "sst/source-1280mb.bin", "sst/dest-1280mb.bin") + assert proc.poll() is None + _assert_check(ctx, "zero bytes_encrypted at 1280MB") + _assert_check(ctx, "peak memory bounded") + _assert_check(ctx, "1280MB ciphertext identical") + + def test_ten_concurrent_passthrough_copies(self, passthrough_env): + ctx, proc, _ = passthrough_env + ctx.failures.clear() + check_concurrent_flood(ctx, "sst/source-1280mb.bin", n=10) + assert proc.poll() is None + _assert_check(ctx, "all copies succeeded") + _assert_check(ctx, "peak memory under prod budget") + _assert_check(ctx, "zero encrypt during flood") + + def test_passthrough_encrypt_delta_vs_reencrypt(self, passthrough_env): + """Passthrough must not touch encrypt counter; range copy must.""" + ctx, proc, _ = passthrough_env + peak_pt, enc_pt = poll_during( + ctx, lambda: upload_part_copy(ctx, "sst/dest-compare-pt.bin", "sst/source-96mb.bin") + ) + peak_re, enc_re = poll_during( + ctx, + lambda: upload_part_copy( + ctx, + "sst/dest-compare-re.bin", + "sst/source-96mb.bin", + byte_range=f"bytes=0-{QUICK_SIZE - 1}", + ), + ) + assert proc.poll() is None + assert enc_pt <= 1 * MB + assert enc_re >= QUICK_SIZE * 0.5 + assert peak_pt <= CHUNK_PEAK * 0.5 + assert peak_re >= CHUNK_PEAK * 0.5 diff --git a/tests/unit/test_copy_clamp_and_concurrency.py b/tests/unit/test_copy_clamp_and_concurrency.py index ac94d77..87ec9f4 100644 --- a/tests/unit/test_copy_clamp_and_concurrency.py +++ b/tests/unit/test_copy_clamp_and_concurrency.py @@ -98,12 +98,14 @@ async def test_large_copy_admitted_while_baseline_reservations_held(): @pytest.mark.asyncio async def test_multiple_copies_run_concurrently_bounded_by_budget(): - """Per-part reservation lets copies interleave: several complete on a 192MB - budget without holding ~88MB for the whole object duration.""" + """Pipeline cap + per-part reservation: five copies complete on 192MB budget.""" + from s3proxy.handlers.multipart.copy import reset_copy_pipeline_semaphore + original_timeout = concurrency.BACKPRESSURE_TIMEOUT concurrency.BACKPRESSURE_TIMEOUT = 30 concurrency.reset_state() concurrency.set_memory_limit(192) + reset_copy_pipeline_semaphore(2) per_chunk = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) assert per_chunk < 192 * MB @@ -114,7 +116,7 @@ async def one_copy(): client = _Client(128 * MB) handler = _handler() handler.multipart_manager = _TrackingMgr() - await handler._streaming_copy_part_inner( + await handler._streaming_copy_part( client, "b", "k", diff --git a/tests/unit/test_copy_per_part_memory.py b/tests/unit/test_copy_per_part_memory.py index e8934d4..860e729 100644 --- a/tests/unit/test_copy_per_part_memory.py +++ b/tests/unit/test_copy_per_part_memory.py @@ -13,6 +13,7 @@ import pytest from s3proxy import concurrency, crypto +from s3proxy.handlers.multipart.copy import reset_copy_pipeline_semaphore from s3proxy.state import MultipartUploadState from tests.unit.test_copy_reservation_vs_real import _Client, _handler @@ -46,7 +47,7 @@ async def add_part(self, *a, **k): return None -def _streaming_copy(handler, client, plaintext_size: int): +def _streaming_copy_inner(handler, client, plaintext_size: int): state = MultipartUploadState(dek=crypto.generate_dek(), bucket="b", key="k", upload_id="u") return handler._streaming_copy_part_inner( client, @@ -66,22 +67,49 @@ def _streaming_copy(handler, client, plaintext_size: int): ) +def _streaming_copy(handler, client, plaintext_size: int): + """Full streaming copy path including pipeline semaphore.""" + state = MultipartUploadState(dek=crypto.generate_dek(), bucket="b", key="k", upload_id="u") + return handler._streaming_copy_part( + client, + "b", + "k", + "u", + 1, + state, + "b", + "src", + None, + None, + None, + {}, + {}, + plaintext_size, + ) + + +@pytest.fixture(autouse=True) +def _reset_copy_semaphore(): + reset_copy_pipeline_semaphore(2) + yield + reset_copy_pipeline_semaphore(2) + + @pytest.mark.asyncio -async def test_governor_returns_to_baseline_between_internal_parts(): - """Smoking gun: active_bytes must hit 0 between internal parts of one copy.""" +async def test_reservation_held_during_upload_wait(): + """Governor must track ciphertext through S3 upload, not release before it.""" concurrency.reset_state() concurrency.set_memory_limit(192) - # 96MB = 3 internal parts at 32MB each plaintext_size = 96 * MB - gate = asyncio.Event() # unset: block uploads until test releases + gate = asyncio.Event() client = _SlowUploadClient(plaintext_size, gate) handler = _handler() handler.multipart_manager = _TrackingMgr() + chunk_peak = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) - copy_task = asyncio.create_task(_streaming_copy(handler, client, plaintext_size)) + copy_task = asyncio.create_task(_streaming_copy_inner(handler, client, plaintext_size)) - # Wait until part 1 encrypt finished and upload_part is blocked on gate for _ in range(200): if client.upload_calls >= 1: break @@ -93,57 +121,101 @@ async def test_governor_returns_to_baseline_between_internal_parts(): assert client.upload_calls == 3 assert concurrency.get_active_memory() == 0 - assert mid_upload_active == 0, ( - f"reservation still held during S3 upload wait: {mid_upload_active / MB:.2f}MB" + assert mid_upload_active == chunk_peak, ( + f"reservation released before upload finished: {mid_upload_active / MB:.2f}MB " + f"(expected {chunk_peak / MB:.2f}MB)" ) @pytest.mark.asyncio -async def test_second_copy_admitted_while_first_waits_on_s3(): - """Between parts, a second copy must acquire memory while the first uploads.""" +async def test_governor_returns_to_baseline_between_internal_parts(): + """Brief gap between parts after upload completes and next acquire.""" concurrency.reset_state() concurrency.set_memory_limit(192) - plaintext_size = 64 * MB # 2 internal parts - gate = asyncio.Event() - client1 = _SlowUploadClient(plaintext_size, gate) - handler1 = _handler() - handler1.multipart_manager = _TrackingMgr() - - acquired_during_upload = asyncio.Event() - - async def second_copy_acquires(): - await asyncio.sleep(0.05) # let copy1 finish part1 encrypt and block on upload - assert client1.upload_calls >= 1 - assert concurrency.get_active_memory() == 0 - reserved = await concurrency.try_acquire_copy_memory(crypto.copy_chunk_peak(32 * MB)) - assert reserved > 0 - acquired_during_upload.set() - await concurrency.release_memory(reserved) - - copy1 = asyncio.create_task(_streaming_copy(handler1, client1, plaintext_size)) - copy2 = asyncio.create_task(second_copy_acquires()) - - for _ in range(100): - if client1.upload_calls >= 1: + plaintext_size = 96 * MB + handler = _handler() + handler.multipart_manager = _TrackingMgr() + + min_active = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) + original_acquire = concurrency._default.try_acquire + original_release = concurrency._default.release + + async def tracking_acquire(bytes_needed: int, *, copy: bool = False): + nonlocal min_active + reserved = await original_acquire(bytes_needed, copy=copy) + min_active = min(min_active, concurrency.get_active_memory()) + return reserved + + async def tracking_release(bytes_reserved: int) -> None: + nonlocal min_active + await original_release(bytes_reserved) + min_active = min(min_active, concurrency.get_active_memory()) + + concurrency._default.try_acquire = tracking_acquire # type: ignore[method-assign] + concurrency._default.release = tracking_release # type: ignore[method-assign] + + try: + await _streaming_copy_inner(handler, _Client(plaintext_size), plaintext_size) + finally: + concurrency._default.try_acquire = original_acquire # type: ignore[method-assign] + concurrency._default.release = original_release # type: ignore[method-assign] + concurrency.reset_state() + + assert min_active == 0, ( + f"governor never returned to baseline between parts; min_active={min_active / MB:.2f}MB" + ) + + +@pytest.mark.asyncio +async def test_copy_pipeline_semaphore_limits_concurrent_copies(): + """Third copy must wait while two pipelines hold the semaphore.""" + concurrency.reset_state() + concurrency.set_memory_limit(192) + reset_copy_pipeline_semaphore(2) + + plaintext_size = 64 * MB + gate1 = asyncio.Event() + gate2 = asyncio.Event() + client1 = _SlowUploadClient(plaintext_size, gate1) + client2 = _SlowUploadClient(plaintext_size, gate2) + handler = _handler() + handler.multipart_manager = _TrackingMgr() + + third_started = asyncio.Event() + + async def third_copy(): + await _streaming_copy(handler, _Client(plaintext_size), plaintext_size) + third_started.set() + + copy1 = asyncio.create_task(_streaming_copy(handler, client1, plaintext_size)) + copy2 = asyncio.create_task(_streaming_copy(handler, client2, plaintext_size)) + + for _ in range(200): + if client1.upload_calls >= 1 and client2.upload_calls >= 1: break await asyncio.sleep(0.01) - gate.set() - await asyncio.wait_for(acquired_during_upload.wait(), timeout=2) - await asyncio.gather(copy1, copy2) - assert concurrency.get_active_memory() == 0 + copy3 = asyncio.create_task(third_copy()) + await asyncio.sleep(0.2) + assert not third_started.is_set(), "third copy started while two pipelines still held semaphore" + + gate1.set() + gate2.set() + await asyncio.gather(copy1, copy2, copy3) + assert third_started.is_set() @pytest.mark.asyncio async def test_five_large_copies_all_complete_with_per_part_reservation(): - """Five 128MB copies interleave on a 192MB budget (impossible with whole-object hold).""" + """Five 128MB copies complete on a 192MB budget (serialized by pipeline cap).""" original_timeout = concurrency.BACKPRESSURE_TIMEOUT concurrency.BACKPRESSURE_TIMEOUT = 30 concurrency.reset_state() concurrency.set_memory_limit(192) + reset_copy_pipeline_semaphore(2) - plaintext_size = 128 * MB # 4 internal parts each + plaintext_size = 128 * MB completed = 0 async def one_copy(idx: int): @@ -151,7 +223,22 @@ async def one_copy(idx: int): client = _Client(plaintext_size) handler = _handler() handler.multipart_manager = _TrackingMgr() - await _streaming_copy(handler, client, plaintext_size) + await handler._streaming_copy_part( + client, + "b", + "k", + "u", + 1, + MultipartUploadState(dek=crypto.generate_dek(), bucket="b", key="k", upload_id="u"), + "b", + "src", + None, + None, + None, + {}, + {}, + plaintext_size, + ) completed += 1 try: @@ -189,7 +276,7 @@ async def tracking_acquire(bytes_needed: int, *, copy: bool = False): try: handler = _handler() handler.multipart_manager = _TrackingMgr() - await _streaming_copy(handler, _Client(plaintext_size), plaintext_size) + await _streaming_copy_inner(handler, _Client(plaintext_size), plaintext_size) finally: concurrency._default.try_acquire = original_acquire # type: ignore[method-assign] concurrency.reset_state() @@ -235,7 +322,7 @@ async def tracking_release(bytes_reserved: int) -> None: try: handler = _handler() handler.multipart_manager = _TrackingMgr() - await _streaming_copy(handler, _Client(256 * MB), 256 * MB) + await _streaming_copy_inner(handler, _Client(256 * MB), 256 * MB) finally: concurrency._default.try_acquire = original_acquire # type: ignore[method-assign] concurrency._default.release = original_release # type: ignore[method-assign] @@ -244,3 +331,118 @@ async def tracking_release(bytes_reserved: int) -> None: assert peak_seen <= chunk_peak + 64 * 1024, ( f"peak active {peak_seen / MB:.2f}MB exceeded one chunk budget {chunk_peak / MB:.2f}MB" ) + + +@pytest.mark.asyncio +async def test_peak_governor_bounded_with_five_concurrent_copies(): + """Prod regression: 13 Scylla nodes hammer one pod. Semaphore caps active copies.""" + original_timeout = concurrency.BACKPRESSURE_TIMEOUT + concurrency.BACKPRESSURE_TIMEOUT = 30 + concurrency.reset_state() + concurrency.set_memory_limit(192) + reset_copy_pipeline_semaphore(2) + + chunk_peak = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) + peak_seen = 0 + original_acquire = concurrency._default.try_acquire + + async def tracking_acquire(bytes_needed: int, *, copy: bool = False): + nonlocal peak_seen + reserved = await original_acquire(bytes_needed, copy=copy) + peak_seen = max(peak_seen, concurrency.get_active_memory()) + return reserved + + concurrency._default.try_acquire = tracking_acquire # type: ignore[method-assign] + + completed = 0 + + async def one_copy(): + nonlocal completed + handler = _handler() + handler.multipart_manager = _TrackingMgr() + await _streaming_copy(handler, _Client(128 * MB), 128 * MB) + completed += 1 + + try: + await asyncio.gather(*[one_copy() for _ in range(5)]) + finally: + concurrency._default.try_acquire = original_acquire # type: ignore[method-assign] + concurrency.BACKPRESSURE_TIMEOUT = original_timeout + concurrency.reset_state() + + assert completed == 5 + # At most 2 pipelines × ~88MB; never 3× (264MB) which caused prod OOM accounting gap. + assert peak_seen <= 2 * chunk_peak + 1 * MB, ( + f"peak governor {peak_seen / MB:.2f}MB exceeded 2-pipeline cap " + f"({2 * chunk_peak / MB:.2f}MB)" + ) + + +@pytest.mark.asyncio +async def test_third_copy_blocked_at_semaphore_before_governor_spikes(): + """Extra copies queue on the semaphore, not as untracked RSS.""" + concurrency.reset_state() + concurrency.set_memory_limit(192) + reset_copy_pipeline_semaphore(2) + + plaintext_size = 64 * MB + gate1 = asyncio.Event() + gate2 = asyncio.Event() + client1 = _SlowUploadClient(plaintext_size, gate1) + client2 = _SlowUploadClient(plaintext_size, gate2) + handler = _handler() + handler.multipart_manager = _TrackingMgr() + chunk_peak = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) + + peak_seen = 0 + original_acquire = concurrency._default.try_acquire + + async def tracking_acquire(bytes_needed: int, *, copy: bool = False): + nonlocal peak_seen + reserved = await original_acquire(bytes_needed, copy=copy) + peak_seen = max(peak_seen, concurrency.get_active_memory()) + return reserved + + concurrency._default.try_acquire = tracking_acquire # type: ignore[method-assign] + + in_pump = 0 + max_in_pump = 0 + original_pump = handler._pump_copy_chunks + + async def counting_pump(*args, **kwargs): + nonlocal in_pump, max_in_pump + in_pump += 1 + max_in_pump = max(max_in_pump, in_pump) + try: + return await original_pump(*args, **kwargs) + finally: + in_pump -= 1 + + handler._pump_copy_chunks = counting_pump # type: ignore[method-assign] + + try: + copy1 = asyncio.create_task(_streaming_copy(handler, client1, plaintext_size)) + copy2 = asyncio.create_task(_streaming_copy(handler, client2, plaintext_size)) + + for _ in range(200): + if client1.upload_calls >= 1 and client2.upload_calls >= 1: + break + await asyncio.sleep(0.01) + + copy3 = asyncio.create_task( + _streaming_copy(handler, _Client(plaintext_size), plaintext_size) + ) + await asyncio.sleep(0.2) + + assert max_in_pump <= 2, f"{max_in_pump} copies in pump concurrently (semaphore cap is 2)" + assert peak_seen <= 2 * chunk_peak + 1 * MB, ( + f"governor spiked to {peak_seen / MB:.2f}MB with >2 pipelines" + ) + + gate1.set() + gate2.set() + await asyncio.gather(copy1, copy2, copy3) + assert max_in_pump <= 2 + finally: + concurrency._default.try_acquire = original_acquire # type: ignore[method-assign] + concurrency.reset_state() diff --git a/tests/unit/test_copy_pipeline_cap.py b/tests/unit/test_copy_pipeline_cap.py new file mode 100644 index 0000000..9c20cd1 --- /dev/null +++ b/tests/unit/test_copy_pipeline_cap.py @@ -0,0 +1,191 @@ +"""Scylla-shaped load tests: pipeline cap + hold-through-upload prevent OOM/503 storms. + +These drive the real UploadPartCopy handler path (including the pipeline semaphore) +and assert the governor never admits more than 2 concurrent copy pipelines worth of +memory — the prod failure mode was 15+ copies per pod with untracked RSS. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from s3proxy import concurrency, crypto +from s3proxy.errors import S3Error +from s3proxy.handlers.multipart.copy import reset_copy_pipeline_semaphore +from tests.unit.test_copy_per_part_memory import ( + MB, + _SlowUploadClient, + _streaming_copy, + _TrackingMgr, +) +from tests.unit.test_copy_reservation_vs_real import _Client, _handler + +PROD_BUDGET_MB = 192 + + +@pytest.fixture(autouse=True) +def _reset(): + reset_copy_pipeline_semaphore(2) + yield + reset_copy_pipeline_semaphore(2) + + +@pytest.mark.asyncio +async def test_scylla_load_thirteen_copies_all_complete_no_rejections(): + """Simulate 13 Scylla nodes hitting one pod: all copies finish, none 503.""" + original_timeout = concurrency.BACKPRESSURE_TIMEOUT + concurrency.BACKPRESSURE_TIMEOUT = 60 + concurrency.reset_state() + concurrency.set_memory_limit(PROD_BUDGET_MB) + reset_copy_pipeline_semaphore(2) + + # 128MB object: 4 internal parts — enough to exercise pipeline without slow test. + plaintext_size = 128 * MB + completed = 0 + rejections = 0 + + async def one_copy(idx: int): + nonlocal completed, rejections + handler = _handler() + handler.multipart_manager = _TrackingMgr() + try: + await _streaming_copy(handler, _Client(plaintext_size), plaintext_size) + completed += 1 + except S3Error as e: + if e.code == "SlowDown": + rejections += 1 + else: + raise + + try: + await asyncio.gather(*[one_copy(i) for i in range(13)]) + finally: + concurrency.BACKPRESSURE_TIMEOUT = original_timeout + concurrency.reset_state() + + assert rejections == 0, f"{rejections} copies got 503 SlowDown" + assert completed == 13 + + +@pytest.mark.asyncio +async def test_governor_never_exceeds_budget_under_scylla_load(): + """Peak reserved bytes must stay within 192MB budget during 13-copy flood.""" + original_timeout = concurrency.BACKPRESSURE_TIMEOUT + concurrency.BACKPRESSURE_TIMEOUT = 60 + concurrency.reset_state() + concurrency.set_memory_limit(PROD_BUDGET_MB) + reset_copy_pipeline_semaphore(2) + + plaintext_size = 64 * MB + peak_seen = 0 + original_acquire = concurrency._default.try_acquire + + async def tracking_acquire(bytes_needed: int, *, copy: bool = False): + nonlocal peak_seen + reserved = await original_acquire(bytes_needed, copy=copy) + peak_seen = max(peak_seen, concurrency.get_active_memory()) + return reserved + + concurrency._default.try_acquire = tracking_acquire # type: ignore[method-assign] + + try: + await asyncio.gather( + *[ + _streaming_copy(_handler(), _Client(plaintext_size), plaintext_size) + for _ in range(8) + ] + ) + finally: + concurrency._default.try_acquire = original_acquire # type: ignore[method-assign] + concurrency.BACKPRESSURE_TIMEOUT = original_timeout + concurrency.reset_state() + + budget = PROD_BUDGET_MB * MB + assert peak_seen <= budget, ( + f"peak governor {peak_seen / MB:.2f}MB exceeded {PROD_BUDGET_MB}MB budget" + ) + + +@pytest.mark.asyncio +async def test_backpressure_logged_once_per_acquire_wait(): + """MEMORY_BACKPRESSURE must not spam on every condition wakeup.""" + original_timeout = concurrency.BACKPRESSURE_TIMEOUT + concurrency.BACKPRESSURE_TIMEOUT = 5 + concurrency.reset_state() + # 96MB budget: one 88MB chunk leaves no room for a second → must wait. + concurrency.set_memory_limit(96) + reset_copy_pipeline_semaphore(1) + + held = await concurrency.try_acquire_copy_memory( + crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) + ) + + log_events: list[str] = [] + original_info = concurrency.logger.info + + def capture_info(event, **kw): + log_events.append(event) + return original_info(event, **kw) + + concurrency.logger.info = capture_info # type: ignore[method-assign] + + waiter = asyncio.create_task( + concurrency.try_acquire_copy_memory(crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE)) + ) + await asyncio.sleep(0.2) + await concurrency.release_memory(held) + second = await waiter + + concurrency.logger.info = original_info # type: ignore[method-assign] + concurrency.BACKPRESSURE_TIMEOUT = original_timeout + concurrency.reset_state() + + assert second > 0 + backpressure_logs = [e for e in log_events if e == "MEMORY_BACKPRESSURE"] + assert len(backpressure_logs) == 1, ( + f"expected 1 MEMORY_BACKPRESSURE log per acquire wait, got {len(backpressure_logs)}" + ) + + +@pytest.mark.asyncio +async def test_hold_through_upload_bounds_two_pipeline_overlap(): + """Two blocked uploads must hold ~2×chunk_peak, not release early.""" + concurrency.reset_state() + concurrency.set_memory_limit(PROD_BUDGET_MB) + reset_copy_pipeline_semaphore(2) + + plaintext_size = 64 * MB + gate1 = asyncio.Event() + gate2 = asyncio.Event() + chunk_peak = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) + + handler = _handler() + handler.multipart_manager = _TrackingMgr() + + copy1 = asyncio.create_task( + _streaming_copy(handler, _SlowUploadClient(plaintext_size, gate1), plaintext_size) + ) + copy2 = asyncio.create_task( + _streaming_copy(handler, _SlowUploadClient(plaintext_size, gate2), plaintext_size) + ) + + for _ in range(300): + active = concurrency.get_active_memory() + if active >= 2 * chunk_peak * 0.9: + break + await asyncio.sleep(0.01) + else: + pytest.fail("two pipelines never both held reservation through upload") + + mid_active = concurrency.get_active_memory() + gate1.set() + gate2.set() + await asyncio.gather(copy1, copy2) + + assert mid_active >= 2 * chunk_peak * 0.9, ( + f"expected ~{2 * chunk_peak / MB:.0f}MB held during dual upload wait, " + f"got {mid_active / MB:.2f}MB (release-before-upload bug)" + ) + assert concurrency.get_active_memory() == 0 diff --git a/tests/unit/test_upload_part_copy_passthrough.py b/tests/unit/test_upload_part_copy_passthrough.py new file mode 100644 index 0000000..005e90e --- /dev/null +++ b/tests/unit/test_upload_part_copy_passthrough.py @@ -0,0 +1,284 @@ +"""UploadPartCopy server-side passthrough for encrypted sources (Scylla dedup path).""" + +import contextlib +import hashlib +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from s3proxy import crypto +from s3proxy.handlers import S3ProxyHandler +from s3proxy.handlers.multipart import MultipartHandlerMixin +from s3proxy.state import ( + InternalPartMetadata, + MultipartMetadata, + MultipartStateManager, + PartMetadata, + save_multipart_metadata, +) + +BUCKET = "backups" + + +def _handler(settings, mock_s3, credentials): + handler = S3ProxyHandler(settings, settings.credentials_store, MultipartStateManager()) + mock_s3.credentials = credentials + handler._client = MagicMock(return_value=mock_s3) + return handler + + +def _copy_handler(settings, manager): + return MultipartHandlerMixin(settings, {}, manager) + + +def _patch_client(handler, mock_s3): + @contextlib.asynccontextmanager + async def _fake_client(creds): + yield mock_s3 + + handler._client = _fake_client + + +def _copy_part_request(dest_path, copy_source, upload_id, part_number=1): + req = MagicMock() + req.url.path = dest_path + req.url.query = f"uploadId={upload_id}&partNumber={part_number}" + req.headers = { + "x-amz-copy-source": copy_source, + } + return req + + +def _get_request(path): + req = MagicMock() + req.url.path = path + req.headers = {} + return req + + +async def _read(response) -> bytes: + if hasattr(response, "body_iterator"): + return b"".join([c async for c in response.body_iterator]) + return response.body + + +def _keys_touched(history, op): + return [c[1].get("key") for c in history if c[0] == op] + + +@pytest.mark.asyncio +async def test_multipart_encrypted_upload_part_copy_is_server_side_passthrough( + mock_s3, settings, manager, credentials, monkeypatch +): + """Large multipart source → native upload_part_copy per internal part, no re-upload.""" + monkeypatch.setattr(crypto, "COPY_INTERNAL_PART_SIZE", crypto.MAX_BUFFER_SIZE) + handler = _copy_handler(settings, manager) + _patch_client(handler, mock_s3) + await mock_s3.create_bucket(BUCKET) + + kid, kek = settings.keyring.key_for(credentials.access_key) + src_dek = crypto.generate_dek() + src_plaintext = b"S" * (crypto.MAX_BUFFER_SIZE * 4) + + ciphertext_blob = bytearray() + internal_parts_meta = [] + for i, start in enumerate(range(0, len(src_plaintext), crypto.MAX_BUFFER_SIZE), 1): + chunk = src_plaintext[start : start + crypto.MAX_BUFFER_SIZE] + ct = crypto.encrypt_frame(chunk, src_dek, "src-upload", i, 0) + internal_parts_meta.append( + InternalPartMetadata( + internal_part_number=i, + plaintext_size=len(chunk), + ciphertext_size=len(ct), + etag=hashlib.md5(ct, usedforsecurity=False).hexdigest(), + ) + ) + ciphertext_blob.extend(ct) + + src_meta = MultipartMetadata( + version=2, + part_count=1, + total_plaintext_size=len(src_plaintext), + parts=[ + PartMetadata( + part_number=1, + plaintext_size=len(src_plaintext), + ciphertext_size=len(ciphertext_blob), + etag="ignored", + md5=hashlib.md5(src_plaintext, usedforsecurity=False).hexdigest(), + internal_parts=internal_parts_meta, + ) + ], + wrapped_dek=crypto.wrap_key(src_dek, kek), + kid=kid, + ) + await mock_s3.put_object(BUCKET, "sst/big.db", bytes(ciphertext_blob)) + await save_multipart_metadata(mock_s3, BUCKET, "sst/big.db", src_meta) + + resp_create = await mock_s3.create_multipart_upload(BUCKET, "sst/big.db.snap") + upload_id = resp_create["UploadId"] + dst_dek = crypto.generate_dek() + await manager.create_upload(BUCKET, "sst/big.db.snap", upload_id, dst_dek, kid) + + mark = len(mock_s3.call_history) + await handler.handle_upload_part_copy( + _copy_part_request( + f"/{BUCKET}/sst/big.db.snap", + f"/{BUCKET}/sst/big.db", + upload_id, + ), + credentials, + ) + during = mock_s3.call_history[mark:] + + assert any(c[0] == "upload_part_copy" for c in during) + assert "sst/big.db.snap" not in _keys_touched(during, "upload_part") + assert "sst/big.db.snap" not in _keys_touched(during, "put_object") + + updated = await manager.get_upload(BUCKET, "sst/big.db.snap", upload_id) + assert updated.dek == src_dek + part = updated.parts[1] + assert part.plaintext_size == len(src_plaintext) + assert len(part.internal_parts) == len(internal_parts_meta) + + recovered = bytearray() + for ip in sorted(part.internal_parts, key=lambda x: x.internal_part_number): + ct = mock_s3.multipart_uploads[upload_id]["Parts"][ip.internal_part_number]["Body"] + recovered.extend(crypto.decrypt_framed(ct, src_dek, ip.plaintext_size)) + assert bytes(recovered) == src_plaintext + + +@pytest.mark.asyncio +async def test_upload_part_copy_passthrough_roundtrips_via_get( + mock_s3, settings, credentials, monkeypatch +): + """End-to-end: passthrough copy → complete → GET returns original plaintext.""" + monkeypatch.setattr(crypto, "COPY_INTERNAL_PART_SIZE", crypto.MAX_BUFFER_SIZE) + handler = _handler(settings, mock_s3, credentials) + await mock_s3.create_bucket(BUCKET) + + kid, kek = settings.keyring.key_for(credentials.access_key) + src_dek = crypto.generate_dek() + body = b"T" * (crypto.STREAMING_THRESHOLD + crypto.MAX_BUFFER_SIZE) + + ciphertext_blob = bytearray() + internal_parts_meta = [] + for i, start in enumerate(range(0, len(body), crypto.MAX_BUFFER_SIZE), 1): + chunk = body[start : start + crypto.MAX_BUFFER_SIZE] + ct = crypto.encrypt_frame(chunk, src_dek, "src-upload", i, 0) + internal_parts_meta.append( + InternalPartMetadata( + internal_part_number=i, + plaintext_size=len(chunk), + ciphertext_size=len(ct), + etag=hashlib.md5(ct, usedforsecurity=False).hexdigest(), + ) + ) + ciphertext_blob.extend(ct) + + src_meta = MultipartMetadata( + version=2, + part_count=1, + total_plaintext_size=len(body), + parts=[ + PartMetadata( + part_number=1, + plaintext_size=len(body), + ciphertext_size=len(ciphertext_blob), + etag="ignored", + md5=hashlib.md5(body, usedforsecurity=False).hexdigest(), + internal_parts=internal_parts_meta, + ) + ], + wrapped_dek=crypto.wrap_key(src_dek, kek), + kid=kid, + ) + await mock_s3.put_object(BUCKET, "sst/source.db", bytes(ciphertext_blob)) + await save_multipart_metadata(mock_s3, BUCKET, "sst/source.db", src_meta) + + create_resp = await handler.handle_create_multipart_upload( + MagicMock(url=MagicMock(path=f"/{BUCKET}/sst/dest.db"), headers={}), + credentials, + ) + upload_id = _extract_upload_id(create_resp.body) + + await handler.handle_upload_part_copy( + _copy_part_request(f"/{BUCKET}/sst/dest.db", f"/{BUCKET}/sst/source.db", upload_id), + credentials, + ) + + part_state = await handler.multipart_manager.get_upload(BUCKET, "sst/dest.db", upload_id) + complete_body = ( + f"1" + f'"{part_state.parts[1].etag}"' + ).encode() + complete_req = MagicMock() + complete_req.url.path = f"/{BUCKET}/sst/dest.db" + complete_req.url.query = f"uploadId={upload_id}" + complete_req.headers = {} + complete_req.body = AsyncMock(return_value=complete_body) + await handler.handle_complete_multipart_upload(complete_req, credentials) + + resp = await handler.handle_get_object(_get_request(f"/{BUCKET}/sst/dest.db"), credentials) + assert await _read(resp) == body + + +@pytest.mark.asyncio +async def test_range_copy_still_reencrypts(mock_s3, settings, manager, credentials, monkeypatch): + """CopySourceRange forces the streaming re-encrypt path.""" + monkeypatch.setattr(crypto, "COPY_INTERNAL_PART_SIZE", crypto.MAX_BUFFER_SIZE) + handler = _copy_handler(settings, manager) + _patch_client(handler, mock_s3) + await mock_s3.create_bucket(BUCKET) + + kid, kek = settings.keyring.key_for(credentials.access_key) + src_dek = crypto.generate_dek() + src_plaintext = b"U" * (crypto.MAX_BUFFER_SIZE * 3) + chunk = src_plaintext[: crypto.MAX_BUFFER_SIZE] + ct = crypto.encrypt_frame(chunk, src_dek, "src", 1, 0) + src_meta = MultipartMetadata( + version=2, + part_count=1, + total_plaintext_size=len(src_plaintext), + parts=[ + PartMetadata( + part_number=1, + plaintext_size=len(src_plaintext), + ciphertext_size=len(ct) * 3, + etag="x", + internal_parts=[ + InternalPartMetadata(1, len(chunk), len(ct), "e1"), + InternalPartMetadata(2, len(chunk), len(ct), "e2"), + InternalPartMetadata(3, len(chunk), len(ct), "e3"), + ], + ) + ], + wrapped_dek=crypto.wrap_key(src_dek, kek), + kid=kid, + ) + await mock_s3.put_object(BUCKET, "src", ct * 3) + await save_multipart_metadata(mock_s3, BUCKET, "src", src_meta) + + resp_create = await mock_s3.create_multipart_upload(BUCKET, "dst") + upload_id = resp_create["UploadId"] + await manager.create_upload(BUCKET, "dst", upload_id, crypto.generate_dek(), kid) + + req = _copy_part_request("/backups/dst", "/backups/src", upload_id) + req.headers["x-amz-copy-source-range"] = f"bytes=0-{crypto.MAX_BUFFER_SIZE - 1}" + + mark = len(mock_s3.call_history) + await handler.handle_upload_part_copy(req, credentials) + during = mock_s3.call_history[mark:] + + assert not any(c[0] == "upload_part_copy" for c in during) + assert any(c[0] == "upload_part" for c in during) + + +def _extract_upload_id(xml_body: bytes) -> str: + import xml.etree.ElementTree as ET + + root = ET.fromstring(xml_body.decode()) + for elem in root.iter(): + if elem.tag.endswith("UploadId") and elem.text: + return elem.text + raise AssertionError("UploadId not found in create response")