From 84091416c1a9be1dc008b41828c9e4e78316bbbe Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Tue, 7 Jul 2026 10:09:59 +0200 Subject: [PATCH] fix: reserve copy memory per internal part, not whole object Streaming UploadPartCopy held ~88MB for the entire multi-part pipeline, blocking concurrent copies on the 192MB budget. Acquire/release per 32MB chunk so memory is free during S3 I/O. Add unit + subprocess Prometheus tests including Scylla-shaped 1280MB concurrent copy load. Co-authored-by: Cursor --- .github/workflows/test.yml | 2 +- Makefile | 8 +- s3proxy/crypto.py | 53 ++-- s3proxy/handlers/multipart/copy.py | 90 ++++--- s3proxy/handlers/objects/misc.py | 24 +- tests/integration/conftest.py | 85 +++++- tests/integration/copy_metrics_helpers.py | 122 +++++++++ tests/integration/test_copy_passthrough.py | 26 ++ .../integration/test_copy_per_part_metrics.py | 202 ++++++++++++++ tests/unit/test_copy_clamp_and_concurrency.py | 57 ++-- tests/unit/test_copy_per_part_memory.py | 246 ++++++++++++++++++ 11 files changed, 812 insertions(+), 103 deletions(-) create mode 100644 tests/integration/copy_metrics_helpers.py create mode 100644 tests/integration/test_copy_per_part_metrics.py create mode 100644 tests/unit/test_copy_per_part_memory.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29637bf..65de5a5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: fail-fast: false matrix: shard: [memory_usage, memory_leak, memory_copy, core, multipart, copy_range, misc] - timeout-minutes: ${{ contains(fromJSON('["memory_usage", "memory_leak"]'), matrix.shard) && 15 || 10 }} + timeout-minutes: ${{ matrix.shard == 'memory_copy' && 25 || (contains(fromJSON('["memory_usage", "memory_leak"]'), matrix.shard) && 15 || 10) }} steps: - uses: actions/checkout@v7 diff --git a/Makefile b/Makefile index 50c37b5..b041f19 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test test-all test-unit test-integration test-integration-shard test-run test-oom e2e cluster lint +.PHONY: test test-all test-unit test-integration test-integration-shard test-run test-oom verify-copy-memory e2e cluster lint # Lint: ruff check + format check lint: @@ -12,10 +12,14 @@ test: test-unit test-unit: uv run pytest -m "not e2e and not ha" -v -n auto +# Pre-merge gate: subprocess Prometheus proof of per-part copy memory release. +verify-copy-memory: + uv run pytest tests/integration/test_copy_per_part_metrics.py -m e2e -v -n0 + # Integration shards for parallel CI (make test-integration-shard SHARD=memory_usage) INTEGRATION_memory_usage_TESTS = tests/integration/test_memory_usage.py INTEGRATION_memory_leak_TESTS = tests/integration/test_memory_leak.py -INTEGRATION_memory_copy_TESTS = tests/integration/test_copy_memory_governor.py +INTEGRATION_memory_copy_TESTS = tests/integration/test_copy_memory_governor.py tests/integration/test_copy_per_part_metrics.py INTEGRATION_memory_copy_PYTEST_OPTS = -n0 INTEGRATION_core_TESTS = \ tests/integration/test_integration.py \ diff --git a/s3proxy/crypto.py b/s3proxy/crypto.py index 6f615e4..6bef673 100644 --- a/s3proxy/crypto.py +++ b/s3proxy/crypto.py @@ -261,31 +261,40 @@ def copy_internal_part_size(plaintext_size: int) -> int: return COPY_INTERNAL_PART_SIZE +def copy_chunk_peak(chunk_plaintext_bytes: int) -> int: + """Peak memory while encrypting one internal copy chunk (streaming path). + + The streaming copy path acquires/releases this amount per internal part so + memory is free between chunks and during S3 upload I/O. Reservation covers + read-buffer slack plus framed encrypt peak; ciphertext upload runs after + release (smaller RSS than encrypt peak). + """ + framed = ( + 4 * chunk_plaintext_bytes + if chunk_plaintext_bytes <= FRAME_PLAINTEXT_SIZE + else 2 * chunk_plaintext_bytes + FRAME_PLAINTEXT_SIZE + ) + peak = framed + 2 * MAX_BUFFER_SIZE + if chunk_plaintext_bytes > 32 * 1024 * 1024: + peak += chunk_plaintext_bytes // 7 + return peak + + +def copy_small_buffered_peak(plaintext_size: int) -> int: + """Peak for a small copy that buffers the whole object in RAM.""" + return max(MAX_BUFFER_SIZE, 3 * plaintext_size) + + def copy_pipeline_peak(plaintext_size: int) -> int: - """Peak memory a server-side copy holds while decrypting + re-encrypting. - - A copy request has no body, so the request-level limiter reserves ~nothing - for it -- yet the copy reads the source, decrypts it and re-encrypts it. The - streaming copy path frames exactly like the framed UploadPart path - (encrypt_frame per FRAME_PLAINTEXT_SIZE, one internal part at a time) but uses - a FIXED copy_internal_part_size, so the per-part peak -- and thus the whole - reservation -- is O(1) in object size (~90MB) rather than object_size/20 - (~535MB for a 4.7GB copy, which monopolized the budget and deadlocked). It - adds one pipeline stage the body-fed upload path lacks: the source is read in - MAX_BUFFER_SIZE chunks into a _PlaintextReader buffer, so reserve two extra - buffers for it. Small copies buffer the whole object and re-encrypt it (~3x). - - aiobotocore copies the ciphertext body for signing; tracemalloc shows - ~part/7 slack once parts exceed 32MB. + """Peak memory for one copy chunk (streaming) or a small buffered copy. + + Streaming copies reserve per internal part via copy_chunk_peak; this returns + the per-chunk peak (O(1) in object size, ~88MB for 32MB parts). Small + copies hold the whole object under one reservation. """ if plaintext_size > STREAMING_THRESHOLD: - part = copy_internal_part_size(plaintext_size) - framed = 4 * part if part <= FRAME_PLAINTEXT_SIZE else 2 * part + FRAME_PLAINTEXT_SIZE - peak = framed + 2 * MAX_BUFFER_SIZE - if part > 32 * 1024 * 1024: - peak += part // 7 - return peak - return max(MAX_BUFFER_SIZE, 3 * plaintext_size) + return copy_chunk_peak(copy_internal_part_size(plaintext_size)) + return copy_small_buffered_peak(plaintext_size) @dataclass(slots=True) diff --git a/s3proxy/handlers/multipart/copy.py b/s3proxy/handlers/multipart/copy.py index 3bdccca..d4f4946 100644 --- a/s3proxy/handlers/multipart/copy.py +++ b/s3proxy/handlers/multipart/copy.py @@ -219,27 +219,24 @@ async def _streaming_copy_part( plaintext_size: int, ) -> Response: # UploadPartCopy carries no request body, so the request-level limiter - # reserved ~nothing -- but this streams the source through decrypt + - # re-encrypt. Reserve the pipeline peak so concurrent copies are bounded - # (a dedup flood otherwise runs unbounded and OOMs the pod). - peak = crypto.copy_pipeline_peak(plaintext_size) - async with concurrency.reserve_copy_memory(peak): - return await self._streaming_copy_part_inner( - client, - bucket, - key, - upload_id, - part_num, - state, - src_bucket, - src_key, - copy_source_range, - src_wrapped_dek, - src_multipart_meta, - head_resp, - src_metadata, - plaintext_size, - ) + # reserved ~nothing. Per-internal-part reservations in _pump_copy_chunks + # gate concurrent copies without holding memory for the whole object. + return await self._streaming_copy_part_inner( + client, + bucket, + key, + upload_id, + part_num, + state, + src_bucket, + src_key, + copy_source_range, + src_wrapped_dek, + src_multipart_meta, + head_resp, + src_metadata, + plaintext_size, + ) async def _streaming_copy_part_inner( self, @@ -397,11 +394,9 @@ async def _pump_copy_chunks( Mirrors UploadPartMixin._stream_and_upload_framed: reads FRAME_PLAINTEXT_SIZE plaintext frames from the source, encrypts each with encrypt_frame, accumulates one internal part's ciphertext, then uploads it before starting - the next. Peak memory is O(one internal part ciphertext + one frame) -- it - never holds the whole part's plaintext and ciphertext at once and runs one - part at a time, which is what makes copy_pipeline_peak == streaming_upload_peak. - Parts are byte-identical to the framed upload path and read back via - decrypt_framed / _iter_multipart_plaintext. + the next. Memory governor reservation is per internal part (acquire before + encrypt, release before upload) so a multi-GB copy does not hold ~88MB for + its entire duration and other copies can interleave during S3 I/O waits. """ reader = _PlaintextReader(src_iter) md5 = hashlib.md5(usedforsecurity=False) @@ -410,28 +405,39 @@ async def _pump_copy_chunks( total_ciphertext = 0 internal_part_num = internal_part_start + prefetch: bytes | None = None while True: + if prefetch is None: + prefetch = await reader.read(min(crypto.FRAME_PLAINTEXT_SIZE, chunk_size)) + if not prefetch: + break + + part_reserve = crypto.copy_chunk_peak(chunk_size) ciphertext = bytearray() part_plaintext = 0 frame_idx = 0 - while part_plaintext < chunk_size: - frame_pt = await reader.read( - min(crypto.FRAME_PLAINTEXT_SIZE, chunk_size - part_plaintext) - ) - if not frame_pt: - break - md5.update(frame_pt) - part_plaintext += len(frame_pt) - ciphertext.extend( - crypto.encrypt_frame( - frame_pt, state.dek, upload_id, internal_part_num, frame_idx + async with concurrency.reserve_copy_memory(part_reserve): + frame_pt = prefetch + prefetch = None + while True: + md5.update(frame_pt) + part_plaintext += len(frame_pt) + ciphertext.extend( + crypto.encrypt_frame( + frame_pt, state.dek, upload_id, internal_part_num, frame_idx + ) ) - ) - frame_idx += 1 - - if part_plaintext == 0: - break + frame_idx += 1 + if part_plaintext >= chunk_size: + break + frame_pt = await reader.read( + min(crypto.FRAME_PLAINTEXT_SIZE, chunk_size - part_plaintext) + ) + if not frame_pt: + break + # Reservation released before upload so other copies can encrypt while + # this one waits on S3 I/O (ciphertext RSS is smaller than encrypt peak). upload_start = time.monotonic() resp = await client.upload_part(bucket, key, upload_id, internal_part_num, ciphertext) del ciphertext diff --git a/s3proxy/handlers/objects/misc.py b/s3proxy/handlers/objects/misc.py index ab44229..75b9d18 100644 --- a/s3proxy/handlers/objects/misc.py +++ b/s3proxy/handlers/objects/misc.py @@ -205,23 +205,27 @@ async def handle_copy_object(self, request: Request, creds: S3Credentials) -> Re request, ) - # Encrypted source. A same-credential plain COPY needs no re-encrypt: - # the ciphertext is self-describing and key-independent (GCM AAD is - # None, the DEK is random and stored in metadata / the sidecar, - # nonces are embedded), so a native server-side CopyObject yields a - # byte-identical, decryptable destination and skips the - # download+re-encrypt+re-upload amplification (Scylla Manager dedup). - # A cross-credential copy must re-key under the copier's KEK, and - # REPLACE / oversized objects also fall back to the re-encrypt path. + # Encrypted source. A plain COPY needs no re-encrypt: the ciphertext + # is self-describing and key-independent (GCM AAD is None, the DEK is + # random and stored in metadata / the sidecar, nonces are embedded), + # so a native server-side CopyObject yields a byte-identical + # destination and skips the download+re-encrypt+re-upload + # amplification (Scylla Manager dedup versioning). + # + # Re-encrypt only to re-key to a *different* owning credential. A + # same-kid copy is byte-identical, and a legacy object with no kid + # cannot be decrypted to re-encrypt anyway (re-encrypt would fail with + # UnknownKidError) -- both must pass through as-is. REPLACE and + # objects above the single-op CopyObject limit still re-encrypt. if src_multipart_meta: src_kid = src_multipart_meta.kid else: src_kid = head_resp.get("Metadata", {}).get(self.settings.kidtag_name, "") - same_credential = bool(src_kid) and src_kid == client.credentials.access_key + needs_rekey = bool(src_kid) and src_kid != client.credentials.access_key ciphertext_size = head_resp.get("ContentLength", 0) or 0 if ( metadata_directive == "COPY" - and same_credential + and not needs_rekey and ciphertext_size <= MAX_SERVER_SIDE_COPY_BYTES ): return await self._copy_passthrough_encrypted( diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8de0ca4..6bf0d87 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,12 +6,95 @@ import subprocess import sys import time +import urllib.error +import urllib.request from collections.abc import Generator from contextlib import contextmanager import boto3 import pytest +# === MINIO BACKEND === + + +def _is_minio(endpoint: str) -> bool: + try: + req = urllib.request.Request(f"{endpoint}/minio/health/live") + with urllib.request.urlopen(req, timeout=2) as resp: + return resp.status == 200 + except urllib.error.URLError, TimeoutError, OSError: + return False + + +def _wait_for_minio(endpoint: str, timeout: float = 30) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if _is_minio(endpoint): + return True + time.sleep(0.5) + return False + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +@contextlib.contextmanager +def minio_backend() -> Generator[str]: + """Yield a MinIO HTTP endpoint, starting a throwaway container if needed.""" + for port in (9000, 19000, 19001): + endpoint = f"http://localhost:{port}" + if _is_minio(endpoint): + yield endpoint + return + + port = 19000 + while port < 19100: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + if s.connect_ex(("localhost", port)) == 0: + port += 1 + continue + break + else: + raise RuntimeError("no free port for temporary MinIO") + + name = f"s3proxy-test-minio-{port}" + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + proc = subprocess.Popen( + [ + "docker", + "run", + "--rm", + "--name", + name, + "-p", + f"{port}:9000", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "minio/minio", + "server", + "/data", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + endpoint = f"http://localhost:{port}" + try: + if not _wait_for_minio(endpoint): + raise RuntimeError(f"MinIO container failed to become healthy on port {port}") + yield endpoint + finally: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + subprocess.run(["docker", "kill", name], capture_output=True, check=False) + + # === SHARED S3PROXY HELPER === @@ -61,7 +144,7 @@ def run_s3proxy( output = sys.stderr if log_output else subprocess.DEVNULL proc = subprocess.Popen( - ["python", "-m", "s3proxy.main"], + [sys.executable, "-m", "s3proxy.main"], env=env, stdout=output, stderr=output, diff --git a/tests/integration/copy_metrics_helpers.py b/tests/integration/copy_metrics_helpers.py new file mode 100644 index 0000000..96a598c --- /dev/null +++ b/tests/integration/copy_metrics_helpers.py @@ -0,0 +1,122 @@ +"""Shared helpers for copy-path governor integration tests (Prometheus scraping).""" + +from __future__ import annotations + +import contextlib +import threading +import time +import urllib.request +from dataclasses import dataclass, field + +from s3proxy import crypto + +MB = 1024 * 1024 + +# Prod pod budget and per-chunk peak for 32MB internal parts. +PROD_GOVERNOR_MB = 192 +CHUNK_PEAK = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) +# Between internal parts only the POST gate (~64KB) should remain reserved. +BASELINE_CEILING = 2 * MB + +# Scylla backup copies ~1.25GB SSTables (40 × 32MB internal parts in s3proxy). +SCYLLA_SSTABLE_MB = 1280 +SCYLLA_SSTABLE_SIZE = SCYLLA_SSTABLE_MB * MB + + +def scrape_reserved_bytes(port: int) -> int: + with urllib.request.urlopen(f"http://localhost:{port}/metrics", timeout=3) as resp: + for line in resp.read().decode().splitlines(): + if line.startswith("s3proxy_memory_reserved_bytes "): + return int(float(line.split()[1])) + return 0 + + +@dataclass +class MetricsPoller: + """Poll s3proxy_memory_reserved_bytes in a background thread.""" + + port: int + interval_s: float = 0.02 + samples: list[int] = field(default_factory=list) + _stop: threading.Event = field(default_factory=threading.Event, repr=False) + _thread: threading.Thread | None = field(default=None, repr=False) + + def start(self) -> None: + self._stop.clear() + + def _poll() -> None: + while not self._stop.is_set(): + with contextlib.suppress(OSError): + self.samples.append(scrape_reserved_bytes(self.port)) + time.sleep(self.interval_s) + + self._thread = threading.Thread(target=_poll, daemon=True) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2) + + +def internal_part_count(plaintext_size: int) -> int: + return -(-plaintext_size // crypto.COPY_INTERNAL_PART_SIZE) + + +def assert_copy_memory_sawtooth( + reserved: list[int], + *, + plaintext_size: int, + min_drops: int | None = None, + min_samples: int = 10, +) -> None: + """Fail if reserved bytes stayed flat (whole-object hold) during a streaming copy.""" + parts = internal_part_count(plaintext_size) + if min_drops is None: + # Need to observe several inter-part gaps; scale with part count. + min_drops = max(3, parts // 4) + + assert len(reserved) >= min_samples, ( + f"too few metric samples ({len(reserved)}); copy finished before polling?" + ) + + min_reserved = min(reserved) + max_reserved = max(reserved) + drops = sum(1 for b in reserved if b <= BASELINE_CEILING) + + assert min_reserved <= BASELINE_CEILING, ( + f"reserved never dropped between internal parts: min={min_reserved / MB:.2f}MB " + f"(expected <= {BASELINE_CEILING / MB:.2f}MB). " + f"max={max_reserved / MB:.2f}MB samples={len(reserved)} parts={parts}" + ) + assert max_reserved <= CHUNK_PEAK * 1.2, ( + f"peak reserved {max_reserved / MB:.2f}MB exceeds one-chunk budget " + f"{CHUNK_PEAK / MB:.2f}MB — whole-object hold?" + ) + assert drops >= min_drops, ( + f"sawtooth too shallow: only {drops} samples <= {BASELINE_CEILING / MB:.2f}MB " + f"(need >= {min_drops} for {parts} internal parts; " + f"min={min_reserved / MB:.2f}MB max={max_reserved / MB:.2f}MB)" + ) + + +def assert_concurrent_copies_not_flatlined( + reserved: list[int], + *, + concurrent_copies: int, + min_samples: int = 20, +) -> None: + """Two whole-object holds keep min ~concurrent_copies×CHUNK_PEAK with no drops.""" + assert len(reserved) >= min_samples, f"too few samples ({len(reserved)})" + min_reserved = min(reserved) + drops = sum(1 for b in reserved if b <= BASELINE_CEILING) + flat_floor = CHUNK_PEAK * concurrent_copies * 0.85 + + assert min_reserved <= BASELINE_CEILING, ( + f"concurrent copies never released between parts: min={min_reserved / MB:.2f}MB " + f"(old whole-object code stays >= {flat_floor / MB:.2f}MB for {concurrent_copies} copies)" + ) + assert drops >= 5, ( + f"expected inter-part drops during {concurrent_copies} concurrent copies; " + f"got {drops} (min={min_reserved / MB:.2f}MB)" + ) diff --git a/tests/integration/test_copy_passthrough.py b/tests/integration/test_copy_passthrough.py index 1cc523a..04df925 100644 --- a/tests/integration/test_copy_passthrough.py +++ b/tests/integration/test_copy_passthrough.py @@ -127,6 +127,32 @@ async def test_multipart_object_copy_copies_sidecar_and_roundtrips(settings, moc assert await _read(resp) == body +@pytest.mark.asyncio +async def test_legacy_no_kid_object_passes_through_not_reencrypt(settings, mock_s3, credentials): + """Legacy encrypted objects (isec set, no isec-kid) can't be decrypted to + re-encrypt; the copy must pass through byte-identically, not fail.""" + handler = _handler(settings, mock_s3, credentials) + await mock_s3.create_bucket(BUCKET) + + # Seed an object with a wrapped DEK but NO kid tag (pre-kid-tracking era). + src_meta = {settings.dektag_name: "d2hhdGV2ZXI=", "client-etag": "deadbeef"} + await mock_s3.put_object(BUCKET, "legacy/x.db", b"opaque-ciphertext", metadata=src_meta) + + mark = len(mock_s3.call_history) + await handler.handle_copy_object( + _copy_request(f"/{BUCKET}/legacy/x.db.snap", f"/{BUCKET}/legacy/x.db"), credentials + ) + during = mock_s3.call_history[mark:] + + assert "legacy/x.db.snap" in _keys_touched(during, "copy_object") + assert "legacy/x.db" not in _keys_touched(during, "get_object") # no decrypt attempt + assert "legacy/x.db.snap" not in _keys_touched(during, "put_object") + # Destination is a faithful, byte-identical copy incl. the wrapped DEK. + dst = mock_s3.objects[mock_s3._key(BUCKET, "legacy/x.db.snap")] + assert dst["Body"] == b"opaque-ciphertext" + assert dst["Metadata"].get(settings.dektag_name) == src_meta[settings.dektag_name] + + @pytest.mark.asyncio async def test_replace_directive_still_reencrypts(settings, mock_s3, credentials): handler = _handler(settings, mock_s3, credentials) diff --git a/tests/integration/test_copy_per_part_metrics.py b/tests/integration/test_copy_per_part_metrics.py new file mode 100644 index 0000000..1eebf48 --- /dev/null +++ b/tests/integration/test_copy_per_part_metrics.py @@ -0,0 +1,202 @@ +"""Subprocess + Prometheus proof: copy governor releases memory per internal part. + +Starts a real s3proxy subprocess, drives real boto3 UploadPartCopy against MinIO, +and polls ``s3proxy_memory_reserved_bytes`` — the same signal Grafana charts in prod. + +Includes a Scylla-backup workload simulation: 1280MB SSTable copies (40 internal +parts), 192MB governor, two concurrent copies (the prod regression scenario). +""" + +from __future__ import annotations + +import concurrent.futures +import socket +import uuid + +import pytest +from botocore.exceptions import ClientError + +from tests.integration.conftest import minio_backend, run_s3proxy +from tests.integration.copy_metrics_helpers import ( + BASELINE_CEILING, + CHUNK_PEAK, + MB, + PROD_GOVERNOR_MB, + SCYLLA_SSTABLE_SIZE, + MetricsPoller, + assert_concurrent_copies_not_flatlined, + assert_copy_memory_sawtooth, +) +from tests.integration.test_copy_memory_governor import ( + _client, + _put_large_source, + _upload_part_copy, +) + +# 256MB → 8 internal parts; fast sawtooth check. +FAST_SOURCE_SIZE = 256 * MB + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +@pytest.fixture(scope="module") +def metrics_server(): + port = _find_free_port() + with ( + minio_backend() as minio_host, + run_s3proxy( + port, + log_output=False, + S3PROXY_HOST=minio_host, + S3PROXY_MEMORY_LIMIT_MB=str(PROD_GOVERNOR_MB), + S3PROXY_BACKPRESSURE_TIMEOUT="30", + S3PROXY_MAX_PART_SIZE_MB="0", + ) as (endpoint, proc), + ): + yield port, endpoint, proc + + +@pytest.fixture(scope="module") +def metrics_bucket(metrics_server): + _, endpoint, _ = metrics_server + client = _client(endpoint) + bucket = f"copy-metrics-{uuid.uuid4().hex[:8]}" + try: + client.create_bucket(Bucket=bucket) + except ClientError as exc: + pytest.skip(f"MinIO/S3 backend not available: {exc}") + yield client, bucket + try: + resp = client.list_objects_v2(Bucket=bucket) + if "Contents" in resp: + client.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": o["Key"]} for o in resp["Contents"]]}, + ) + client.delete_bucket(Bucket=bucket) + except Exception: + pass + + +@pytest.fixture(scope="module") +def fast_source(metrics_bucket): + client, bucket = metrics_bucket + key = "fast-source.bin" + _put_large_source(client, bucket, key, FAST_SOURCE_SIZE) + return key + + +@pytest.fixture(scope="module") +def scylla_sstable_source(metrics_bucket): + """1280MB encrypted object — same size as prod Scylla backup SSTable copies.""" + client, bucket = metrics_bucket + key = "scylla-sstable-1280mb.bin" + _put_large_source(client, bucket, key, SCYLLA_SSTABLE_SIZE) + return key + + +@pytest.mark.e2e +class TestCopyPerPartMetricsSubprocess: + """Real subprocess + Prometheus: reserved bytes must sawtooth during copy.""" + + def test_memory_reserved_drops_between_internal_parts( + self, metrics_server, metrics_bucket, fast_source + ): + port, _, proc = metrics_server + client, bucket = metrics_bucket + + poller = MetricsPoller(port) + poller.start() + + result = _upload_part_copy(client, bucket, "metrics-dest.bin", fast_source) + poller.stop() + + assert proc.poll() is None, "s3proxy crashed during copy" + assert result["success"], f"copy failed: {result}" + assert_copy_memory_sawtooth(poller.samples, plaintext_size=FAST_SOURCE_SIZE) + + def test_three_concurrent_copies_all_succeed_on_192mb( + self, metrics_server, metrics_bucket, fast_source + ): + """Per-part interleaving: 3 large copies fit where whole-object hold allowed 2.""" + _, _, proc = metrics_server + client, bucket = metrics_bucket + + def one_copy(i: int) -> dict: + return _upload_part_copy(client, bucket, f"concurrent-{i}.bin", fast_source) + + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + results = list(pool.map(one_copy, range(3))) + + assert proc.poll() is None + succeeded = sum(1 for r in results if r["success"]) + slowed = sum(1 for r in results if r.get("code") == "SlowDown") + assert succeeded == 3, ( + f"expected all 3 copies to succeed with per-part reservation; " + f"succeeded={succeeded} slowed={slowed} results={results}" + ) + + +@pytest.mark.e2e +class TestScyllaBackupCopyLoad: + """Prod-shaped load: 1280MB SSTables, 192MB governor, concurrent UploadPartCopy.""" + + def test_scylla_1280mb_copy_sawtooths_across_40_internal_parts( + self, metrics_server, metrics_bucket, scylla_sstable_source + ): + port, _, proc = metrics_server + client, bucket = metrics_bucket + + poller = MetricsPoller(port, interval_s=0.05) + poller.start() + + result = _upload_part_copy(client, bucket, "scylla-backup-dest.bin", scylla_sstable_source) + poller.stop() + + assert proc.poll() is None, "s3proxy crashed during 1280MB copy" + assert result["success"], f"1280MB copy failed: {result}" + # 40 internal parts — expect many inter-part drops, not a flat ~88MB line. + assert_copy_memory_sawtooth( + poller.samples, + plaintext_size=SCYLLA_SSTABLE_SIZE, + min_drops=10, + min_samples=15, + ) + + def test_two_concurrent_scylla_1280mb_copies_succeed_without_flat_reservation( + self, metrics_server, metrics_bucket, scylla_sstable_source + ): + """Prod regression: 2×1280MB copies held ~176MB flat; per-part must interleave.""" + port, _, proc = metrics_server + client, bucket = metrics_bucket + + poller = MetricsPoller(port, interval_s=0.05) + poller.start() + + def one_copy(i: int) -> dict: + return _upload_part_copy( + client, bucket, f"scylla-concurrent-{i}.bin", scylla_sstable_source + ) + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(one_copy, range(2))) + + poller.stop() + + assert proc.poll() is None, "s3proxy crashed under Scylla backup load" + assert all(r["success"] for r in results), ( + f"expected both 1280MB copies to succeed on {PROD_GOVERNOR_MB}MB budget: {results}" + ) + assert_concurrent_copies_not_flatlined(poller.samples, concurrent_copies=2) + # Old code: min stays ~176MB (2 × 88MB whole-object holds). Per-part frees budget. + assert min(poller.samples) <= BASELINE_CEILING, ( + f"reservation never dropped during concurrent Scylla copies " + f"(min={min(poller.samples) / MB:.2f}MB, expected <= {BASELINE_CEILING / MB:.2f}MB)" + ) + assert max(poller.samples) <= CHUNK_PEAK * 2.5, ( + f"peak reserved {max(poller.samples) / MB:.2f}MB — more than brief 2-chunk overlap?" + ) diff --git a/tests/unit/test_copy_clamp_and_concurrency.py b/tests/unit/test_copy_clamp_and_concurrency.py index 4924be8..ac94d77 100644 --- a/tests/unit/test_copy_clamp_and_concurrency.py +++ b/tests/unit/test_copy_clamp_and_concurrency.py @@ -27,8 +27,9 @@ from s3proxy import concurrency, crypto from s3proxy.concurrency import MIN_RESERVATION -from s3proxy.errors import S3Error -from tests.unit.test_copy_reservation_vs_real import _measure_peak +from s3proxy.state import MultipartUploadState +from tests.unit.test_copy_per_part_memory import _TrackingMgr +from tests.unit.test_copy_reservation_vs_real import _Client, _handler, _measure_peak MB = 1024 * 1024 @@ -97,32 +98,39 @@ async def test_large_copy_admitted_while_baseline_reservations_held(): @pytest.mark.asyncio async def test_multiple_copies_run_concurrently_bounded_by_budget(): - """Copies are no longer serialized to one; several run at once, capped by the - budget (each ~90MB), and total reserved never exceeds it -- so no OOM.""" + """Per-part reservation lets copies interleave: several complete on a 192MB + budget without holding ~88MB for the whole object duration.""" original_timeout = concurrency.BACKPRESSURE_TIMEOUT - concurrency.BACKPRESSURE_TIMEOUT = 0 + concurrency.BACKPRESSURE_TIMEOUT = 30 concurrency.reset_state() concurrency.set_memory_limit(192) - per_copy = crypto.copy_pipeline_peak(SCYLLA_MANIFEST_BYTES) - assert per_copy < 192 * MB + per_chunk = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) + assert per_chunk < 192 * MB - inside = 0 - max_inside = 0 - peak_active = 0 - admitted = 0 + completed = 0 async def one_copy(): - nonlocal inside, max_inside, peak_active, admitted - try: - async with concurrency.reserve_copy_memory(per_copy): - inside += 1 - admitted += 1 - max_inside = max(max_inside, inside) - peak_active = max(peak_active, concurrency.get_active_memory()) - await asyncio.sleep(0.05) - inside -= 1 - except S3Error: - pass + nonlocal completed + client = _Client(128 * MB) + handler = _handler() + handler.multipart_manager = _TrackingMgr() + await handler._streaming_copy_part_inner( + client, + "b", + "k", + "u", + 1, + MultipartUploadState(dek=crypto.generate_dek(), bucket="b", key="k", upload_id="u"), + "b", + "src", + None, + None, + None, + {}, + {}, + 128 * MB, + ) + completed += 1 try: await asyncio.gather(*[one_copy() for _ in range(5)]) @@ -130,9 +138,8 @@ async def one_copy(): concurrency.BACKPRESSURE_TIMEOUT = original_timeout concurrency.reset_state() - assert max_inside >= 2, "at least two ~90MB copies must run concurrently in 192MB" - assert peak_active <= 192 * MB, "governor must never overrun the budget (OOM guard)" - assert admitted >= 2 + assert completed == 5 + assert concurrency.get_active_memory() == 0 @pytest.mark.asyncio diff --git a/tests/unit/test_copy_per_part_memory.py b/tests/unit/test_copy_per_part_memory.py new file mode 100644 index 0000000..e8934d4 --- /dev/null +++ b/tests/unit/test_copy_per_part_memory.py @@ -0,0 +1,246 @@ +"""Prove copy-path memory is reserved per internal part, not for the whole object. + +These tests drive the real _streaming_copy_part_inner / _pump_copy_chunks path and +observe concurrency.get_active_memory(). They would fail if the copy held one +~88MB reservation for the entire multi-part pipeline (the prod regression where +active_mb stuck at 176.38 for two concurrent 1280MB copies). +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from s3proxy import concurrency, crypto +from s3proxy.state import MultipartUploadState +from tests.unit.test_copy_reservation_vs_real import _Client, _handler + +MB = 1024 * 1024 + + +class _SlowUploadClient(_Client): + """Blocks every upload_part until gate is set.""" + + def __init__(self, total: int, gate: asyncio.Event): + super().__init__(total) + self.gate = gate + self.upload_calls = 0 + + async def upload_part(self, bucket, key, upload_id, part_number, body): + self.upload_calls += 1 + await self.gate.wait() + return await super().upload_part(bucket, key, upload_id, part_number, body) + + +class _TrackingMgr: + def __init__(self): + self.allocated = 0 + + async def allocate_internal_parts(self, bucket, key, upload_id, count, client_part_number): + start = self.allocated + 1 + self.allocated += count + return start + + async def add_part(self, *a, **k): + return None + + +def _streaming_copy(handler, client, plaintext_size: int): + state = MultipartUploadState(dek=crypto.generate_dek(), bucket="b", key="k", upload_id="u") + return handler._streaming_copy_part_inner( + client, + "b", + "k", + "u", + 1, + state, + "b", + "src", + None, + None, + None, + {}, + {}, + plaintext_size, + ) + + +@pytest.mark.asyncio +async def test_governor_returns_to_baseline_between_internal_parts(): + """Smoking gun: active_bytes must hit 0 between internal parts of one copy.""" + concurrency.reset_state() + concurrency.set_memory_limit(192) + + # 96MB = 3 internal parts at 32MB each + plaintext_size = 96 * MB + gate = asyncio.Event() # unset: block uploads until test releases + client = _SlowUploadClient(plaintext_size, gate) + handler = _handler() + handler.multipart_manager = _TrackingMgr() + + copy_task = asyncio.create_task(_streaming_copy(handler, client, plaintext_size)) + + # Wait until part 1 encrypt finished and upload_part is blocked on gate + for _ in range(200): + if client.upload_calls >= 1: + break + await asyncio.sleep(0.01) + assert client.upload_calls >= 1 + mid_upload_active = concurrency.get_active_memory() + gate.set() + await copy_task + + assert client.upload_calls == 3 + assert concurrency.get_active_memory() == 0 + assert mid_upload_active == 0, ( + f"reservation still held during S3 upload wait: {mid_upload_active / MB:.2f}MB" + ) + + +@pytest.mark.asyncio +async def test_second_copy_admitted_while_first_waits_on_s3(): + """Between parts, a second copy must acquire memory while the first uploads.""" + concurrency.reset_state() + concurrency.set_memory_limit(192) + + plaintext_size = 64 * MB # 2 internal parts + gate = asyncio.Event() + client1 = _SlowUploadClient(plaintext_size, gate) + handler1 = _handler() + handler1.multipart_manager = _TrackingMgr() + + acquired_during_upload = asyncio.Event() + + async def second_copy_acquires(): + await asyncio.sleep(0.05) # let copy1 finish part1 encrypt and block on upload + assert client1.upload_calls >= 1 + assert concurrency.get_active_memory() == 0 + reserved = await concurrency.try_acquire_copy_memory(crypto.copy_chunk_peak(32 * MB)) + assert reserved > 0 + acquired_during_upload.set() + await concurrency.release_memory(reserved) + + copy1 = asyncio.create_task(_streaming_copy(handler1, client1, plaintext_size)) + copy2 = asyncio.create_task(second_copy_acquires()) + + for _ in range(100): + if client1.upload_calls >= 1: + break + await asyncio.sleep(0.01) + gate.set() + + await asyncio.wait_for(acquired_during_upload.wait(), timeout=2) + await asyncio.gather(copy1, copy2) + assert concurrency.get_active_memory() == 0 + + +@pytest.mark.asyncio +async def test_five_large_copies_all_complete_with_per_part_reservation(): + """Five 128MB copies interleave on a 192MB budget (impossible with whole-object hold).""" + original_timeout = concurrency.BACKPRESSURE_TIMEOUT + concurrency.BACKPRESSURE_TIMEOUT = 30 + concurrency.reset_state() + concurrency.set_memory_limit(192) + + plaintext_size = 128 * MB # 4 internal parts each + completed = 0 + + async def one_copy(idx: int): + nonlocal completed + client = _Client(plaintext_size) + handler = _handler() + handler.multipart_manager = _TrackingMgr() + await _streaming_copy(handler, client, plaintext_size) + completed += 1 + + try: + await asyncio.gather(*[one_copy(i) for i in range(5)]) + finally: + concurrency.BACKPRESSURE_TIMEOUT = original_timeout + concurrency.reset_state() + + assert completed == 5 + peak = max( + crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE), + crypto.copy_pipeline_peak(plaintext_size), + ) + assert peak < 192 * MB + + +@pytest.mark.asyncio +async def test_per_part_reserve_uses_chunk_peak_not_whole_object(): + """Each internal part reserves copy_chunk_peak(chunk_size), not object size.""" + concurrency.reset_state() + concurrency.set_memory_limit(192) + + acquired: list[int] = [] + original_acquire = concurrency._default.try_acquire + + async def tracking_acquire(bytes_needed: int, *, copy: bool = False): + reserved = await original_acquire(bytes_needed, copy=copy) + if copy: + acquired.append(reserved) + return reserved + + concurrency._default.try_acquire = tracking_acquire # type: ignore[method-assign] + + plaintext_size = 128 * MB + try: + handler = _handler() + handler.multipart_manager = _TrackingMgr() + await _streaming_copy(handler, _Client(plaintext_size), plaintext_size) + finally: + concurrency._default.try_acquire = original_acquire # type: ignore[method-assign] + concurrency.reset_state() + + expected = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) + assert len(acquired) == 4, f"expected 4 per-part acquires, got {len(acquired)}" + assert all(r == expected for r in acquired), ( + f"per-part reserve drifted: {{r / MB for r in acquired}}MB, expected {expected / MB}MB" + ) + assert expected == crypto.copy_pipeline_peak(plaintext_size) + + +def test_copy_chunk_peak_32mb_uses_framed_formula_not_small_buffer(): + """32MB internal parts must not hit the small-buffer 3x formula (96MB bug).""" + peak = crypto.copy_chunk_peak(32 * MB) + assert peak == 88 * MB, f"32MB chunk peak should be 88MB, got {peak / MB}MB" + assert peak < 96 * MB + + +@pytest.mark.asyncio +async def test_whole_copy_duration_active_memory_bounded_to_one_chunk(): + """While one copy runs, governor active must never exceed one chunk peak.""" + concurrency.reset_state() + concurrency.set_memory_limit(192) + + chunk_peak = crypto.copy_chunk_peak(crypto.COPY_INTERNAL_PART_SIZE) + peak_seen = 0 + original_acquire = concurrency._default.try_acquire + original_release = concurrency._default.release + + async def tracking_acquire(bytes_needed: int, *, copy: bool = False): + nonlocal peak_seen + reserved = await original_acquire(bytes_needed, copy=copy) + peak_seen = max(peak_seen, concurrency.get_active_memory()) + return reserved + + async def tracking_release(bytes_reserved: int) -> None: + await original_release(bytes_reserved) + + concurrency._default.try_acquire = tracking_acquire # type: ignore[method-assign] + concurrency._default.release = tracking_release # type: ignore[method-assign] + + try: + handler = _handler() + handler.multipart_manager = _TrackingMgr() + await _streaming_copy(handler, _Client(256 * MB), 256 * MB) + finally: + concurrency._default.try_acquire = original_acquire # type: ignore[method-assign] + concurrency._default.release = original_release # type: ignore[method-assign] + concurrency.reset_state() + + assert peak_seen <= chunk_peak + 64 * 1024, ( + f"peak active {peak_seen / MB:.2f}MB exceeded one chunk budget {chunk_peak / MB:.2f}MB" + )