Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions s3proxy/crypto.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,16 +191,17 @@ def streaming_upload_peak(content_length: int) -> int:
def copy_pipeline_peak(plaintext_size: int) -> int:
"""Peak memory a server-side copy holds while decrypting + re-encrypting.

A copy request has no body, so the request-level memory limiter reserves
~nothing for it -- yet the copy reads the source, decrypts it and re-encrypts
it. Large copies stream in MAX_BUFFER_SIZE chunks, so their peak is the
pipeline (source chunk + plaintext buffer + dest ciphertext), independent of
object size; small copies buffer the whole object (~3x). Reserving this lets
the limiter BOUND concurrent copies instead of admitting an unbounded dedup
flood that OOMs the pod.
A copy request has no body, so the request-level limiter reserves ~nothing
for it -- yet the copy reads the source, decrypts it and re-encrypts it. The
streaming copy path now frames exactly like the framed UploadPart path
(encrypt_frame per FRAME_PLAINTEXT_SIZE, one internal part at a time, sized by
memory_bounded_part_size), so its per-part peak matches the upload path. It
adds one pipeline stage the body-fed upload path lacks: the source is read in
MAX_BUFFER_SIZE chunks into a _PlaintextReader buffer, so reserve two extra
buffers for it. Small copies buffer the whole object and re-encrypt it (~3x).
"""
if plaintext_size > STREAMING_THRESHOLD:
return 4 * MAX_BUFFER_SIZE
return streaming_upload_peak(plaintext_size) + 2 * MAX_BUFFER_SIZE
return max(MAX_BUFFER_SIZE, 3 * plaintext_size)


Expand Down
160 changes: 83 additions & 77 deletions s3proxy/handlers/multipart/copy.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
"""UploadPartCopy handler for multipart uploads."""

import asyncio
import hashlib
import math
import time
from collections.abc import AsyncIterator
from datetime import UTC, datetime

Expand All @@ -22,11 +22,10 @@
)
from ...utils import format_iso8601
from ..base import BaseHandler
from .upload_part import _PlaintextReader

logger: BoundLogger = structlog.get_logger(__name__)

MAX_PARALLEL_INTERNAL_UPLOADS = 2


class CopyPartMixin(BaseHandler):
async def handle_upload_part_copy(self, request: Request, creds: S3Credentials) -> Response:
Expand Down Expand Up @@ -60,21 +59,25 @@ async def handle_upload_part_copy(self, request: Request, creds: S3Credentials)
)

if plaintext_size <= crypto.STREAMING_THRESHOLD:
return await self._simple_copy_part(
client,
bucket,
key,
upload_id,
part_num,
state,
src_bucket,
src_key,
copy_source_range,
head_resp,
src_metadata,
src_wrapped_dek,
src_multipart_meta,
)
# Small copies buffer the whole object + re-encrypt it; gate them
# by the limiter too (they carry no body, so the request-level
# reservation was ~nothing and a small-object flood ran unbounded).
async with concurrency.reserve_memory(crypto.copy_pipeline_peak(plaintext_size)):
return await self._simple_copy_part(
client,
bucket,
key,
upload_id,
part_num,
state,
src_bucket,
src_key,
copy_source_range,
head_resp,
src_metadata,
src_wrapped_dek,
src_multipart_meta,
)
return await self._streaming_copy_part(
client,
bucket,
Expand Down Expand Up @@ -254,7 +257,10 @@ async def _streaming_copy_part_inner(
plaintext_size: int,
) -> Response:
"""Stream-decrypt the source and encrypt+upload each chunk as an internal S3 part."""
chunk_size = crypto.calculate_optimal_part_size(plaintext_size)
# Same part sizing as the framed UploadPart path: bounds the internal part
# count to MAX_INTERNAL_PARTS_PER_CLIENT (so a large copy can never overflow
# the per-client internal-part range) and keeps parts small.
chunk_size = crypto.memory_bounded_part_size(plaintext_size)
estimated_parts = max(1, math.ceil(plaintext_size / chunk_size))

internal_part_start = await self.multipart_manager.allocate_internal_parts(
Expand Down Expand Up @@ -381,69 +387,69 @@ async def _pump_copy_chunks(
chunk_size: int,
internal_part_start: int,
) -> tuple[list[InternalPartMetadata], int, int, object]:
"""Buffer plaintext from src_iter into chunk_size pieces and upload as internal parts."""
buf = bytearray()
current_internal = internal_part_start
upload_tasks: dict[int, asyncio.Task] = {}
semaphore = asyncio.Semaphore(MAX_PARALLEL_INTERNAL_UPLOADS)
"""Frame-encrypt the copy source into internal S3 parts, one at a time.

Mirrors UploadPartMixin._stream_and_upload_framed: reads FRAME_PLAINTEXT_SIZE
plaintext frames from the source, encrypts each with encrypt_frame,
accumulates one internal part's ciphertext, then uploads it before starting
the next. Peak memory is O(one internal part ciphertext + one frame) -- it
never holds the whole part's plaintext and ciphertext at once and runs one
part at a time, which is what makes copy_pipeline_peak == streaming_upload_peak.
Parts are byte-identical to the framed upload path and read back via
decrypt_framed / _iter_multipart_plaintext.
"""
reader = _PlaintextReader(src_iter)
md5 = hashlib.md5(usedforsecurity=False)
internal_parts: list[InternalPartMetadata] = []
total_plaintext = 0

async for raw in src_iter:
buf.extend(raw)
md5.update(raw)
total_plaintext += len(raw)

while len(buf) >= chunk_size:
data = bytes(buf[:chunk_size])
del buf[:chunk_size]
await semaphore.acquire()
task = asyncio.create_task(
self._upload_internal_part_with_semaphore( # type: ignore[attr-defined]
client,
bucket,
key,
upload_id,
part_num,
state,
data,
current_internal,
semaphore,
total_ciphertext = 0
internal_part_num = internal_part_start

while True:
ciphertext = bytearray()
part_plaintext = 0
frame_idx = 0
while part_plaintext < chunk_size:
frame_pt = await reader.read(
min(crypto.FRAME_PLAINTEXT_SIZE, chunk_size - part_plaintext)
)
if not frame_pt:
break
md5.update(frame_pt)
part_plaintext += len(frame_pt)
ciphertext.extend(
crypto.encrypt_frame(
frame_pt, state.dek, upload_id, internal_part_num, frame_idx
)
)
upload_tasks[current_internal] = task
current_internal += 1

if buf:
await semaphore.acquire()
data = bytes(buf)
task = asyncio.create_task(
self._upload_internal_part_with_semaphore( # type: ignore[attr-defined]
client,
bucket,
key,
upload_id,
part_num,
state,
data,
current_internal,
semaphore,
frame_idx += 1

if part_plaintext == 0:
break

upload_start = time.monotonic()
resp = await client.upload_part(bucket, key, upload_id, internal_part_num, ciphertext)
del ciphertext
ct_size = crypto.framed_ciphertext_size(part_plaintext)
internal_parts.append(
InternalPartMetadata(
internal_part_number=internal_part_num,
plaintext_size=part_plaintext,
ciphertext_size=ct_size,
etag=resp["ETag"].strip('"'),
)
)
upload_tasks[current_internal] = task

results = await asyncio.gather(*upload_tasks.values(), return_exceptions=True)
self._check_upload_results(results, bucket, key, upload_id, part_num) # type: ignore[attr-defined]

results_by_part: dict[int, InternalPartMetadata] = {
r.internal_part_number: r
for r in results # type: ignore[union-attr]
}
internal_parts = []
total_ciphertext = 0
for pn in sorted(results_by_part):
meta = results_by_part[pn]
internal_parts.append(meta)
total_ciphertext += meta.ciphertext_size
logger.info(
"INTERNAL_PART_UPLOADED",
bucket=bucket,
key=key,
client_part=part_num,
internal_part=internal_part_num,
plaintext_mb=f"{part_plaintext / 1024 / 1024:.2f}MB",
elapsed_sec=f"{time.monotonic() - upload_start:.2f}s",
)
total_plaintext += part_plaintext
total_ciphertext += ct_size
internal_part_num += 1

return internal_parts, total_plaintext, total_ciphertext, md5
23 changes: 14 additions & 9 deletions tests/unit/test_copy_memory_governing.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,17 @@
MB = 1024 * 1024


def test_copy_pipeline_peak_streamed_is_bounded():
# Large copies stream in MAX_BUFFER_SIZE chunks, so their peak is the
# pipeline (source chunk + plaintext + dest ciphertext), independent of size.
def test_copy_pipeline_peak_matches_framed_upload():
# The streaming copy path now frames exactly like the framed UploadPart path
# (one internal part at a time), so its peak is identical: the old
# size-independent 4*MAX_BUFFER_SIZE under-counted the real peak ~6x -- when
# it un-framed a whole part -- and OOMed the pod.
for size in (64 * MB, 512 * MB, 5 * 1024 * MB):
peak = crypto.copy_pipeline_peak(size)
assert peak == 4 * crypto.MAX_BUFFER_SIZE
assert peak < size # never scales with object size
# Same framed writer peak, plus two MAX_BUFFER_SIZE buffers for the copy
# source read pipeline the body-fed upload path doesn't have.
assert crypto.copy_pipeline_peak(size) == (
crypto.streaming_upload_peak(size) + 2 * crypto.MAX_BUFFER_SIZE
)


def test_copy_pipeline_peak_small_is_three_x():
Expand All @@ -48,9 +52,10 @@ async def test_reserve_memory_bounds_concurrent_copies():
original_timeout = concurrency.BACKPRESSURE_TIMEOUT
concurrency.BACKPRESSURE_TIMEOUT = 30
limiter = concurrency._default
limiter.set_memory_limit(64) # 64MB budget
limiter.active_bytes = 0
per_copy = crypto.copy_pipeline_peak(64 * MB) # 32MB -> budget fits 2
per_copy = crypto.copy_pipeline_peak(64 * MB)
limiter.set_memory_limit((per_copy * 2) // MB) # budget sized to fit ~2
budget_fits = limiter.limit_bytes // per_copy

peak_active = 0
inside = 0
Expand All @@ -71,5 +76,5 @@ async def one_copy():
concurrency.BACKPRESSURE_TIMEOUT = original_timeout

assert peak_active <= limiter.limit_bytes # never overran the budget
assert max_inside <= 2 # 64MB / 32MB == at most 2 copies at once
assert max_inside <= budget_fits # limiter admits only what the budget fits
assert limiter.active_bytes == 0 # everything released
119 changes: 119 additions & 0 deletions tests/unit/test_copy_reservation_vs_real.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
"""The copy reservation must bound the streaming UploadPartCopy path's REAL peak.

Analogue of test_upload_reservation_vs_real.py, for the copy path. This is the
test that would have caught the ScyllaDB backup OOM: copy_pipeline_peak()
reserved a size-independent 32MB (4*MAX_BUFFER_SIZE) on the false premise that
the copy "streams in MAX_BUFFER_SIZE chunks", while _pump_copy_chunks actually
buffers a full internal part and _upload_internal_part_with_semaphore holds
plaintext + ciphertext of that part at once, times MAX_PARALLEL_INTERNAL_UPLOADS.
The governor therefore admitted ~6x too many concurrent copies -> OOMKilled.

It drives the ACTUAL handler method (not a re-implementation) under tracemalloc,
so it fails if the reservation drifts below reality OR the copy path starts
allocating more. The mock client copies the uploaded body the way aiobotocore
does for the signed HTTP request, which is part of the real peak.
"""

import hashlib
import tracemalloc

import pytest

from s3proxy import crypto
from s3proxy.handlers.multipart import MultipartHandlerMixin
from s3proxy.state import MultipartUploadState

MB = 1024 * 1024


class _Mgr:
async def allocate_internal_parts(self, bucket, key, upload_id, count, client_part_number):
return 1

async def add_part(self, *a, **k):
return None


class _Body:
"""aiohttp-like streaming body: `async with body`, `body.content.read(n)`."""

def __init__(self, total, chunk):
self._total = total
self._chunk = chunk
self._sent = 0
self.content = self

async def read(self, n):
if self._sent >= self._total:
return b""
take = min(n, self._total - self._sent)
self._sent += take
return b"x" * take

async def __aenter__(self):
return self

async def __aexit__(self, *a):
return False


class _Client:
def __init__(self, total):
self._total = total

async def get_object(self, bucket, key, range_header=None):
return {"Body": _Body(self._total, crypto.MAX_BUFFER_SIZE)}

async def upload_part(self, bucket, key, upload_id, part_number, body):
# aiobotocore copies the body to sign and send it; mirror that so the
# measured peak reflects what the real transport holds.
sent = bytes(body)
return {"ETag": hashlib.md5(sent).hexdigest()}


def _handler():
h = MultipartHandlerMixin.__new__(MultipartHandlerMixin) # bypass BaseHandler.__init__
h.multipart_manager = _Mgr()
return h


async def _measure_peak(plaintext_size):
h = _handler()
state = MultipartUploadState(dek=crypto.generate_dek(), bucket="b", key="k", upload_id="u")
tracemalloc.start()
base = tracemalloc.get_traced_memory()[0]
# Plaintext (unencrypted) source: src_wrapped_dek=None, src_multipart_meta=None.
await h._streaming_copy_part_inner(
_Client(plaintext_size),
"b",
"k",
"u",
1,
state,
"b",
"src",
None,
None,
None,
{},
{},
plaintext_size,
)
peak = tracemalloc.get_traced_memory()[1]
tracemalloc.stop()
return peak - base


@pytest.mark.asyncio
@pytest.mark.parametrize("mb", [64, 128, 512])
async def test_reservation_bounds_real_copy_peak(mb):
plaintext_size = mb * MB
real_peak = await _measure_peak(plaintext_size)
reserved = crypto.copy_pipeline_peak(plaintext_size)

# The whole point: the reservation must cover the REAL peak. The old
# size-independent 4*MAX_BUFFER_SIZE reservation fails here.
assert reserved >= real_peak, (
f"{mb}MB copy: reserved {reserved / MB:.1f}MB < real peak "
f"{real_peak / MB:.1f}MB -- governor under-counts, will OOM"
)
Loading