Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions s3proxy/concurrency.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

import structlog

from s3proxy.crypto import streaming_upload_peak
from s3proxy.crypto import governor_memory_footprint, streaming_governor_clamped_reserve
from s3proxy.errors import S3Error
from s3proxy.metrics import MEMORY_LIMIT_BYTES, MEMORY_REJECTIONS, MEMORY_RESERVED_BYTES

Expand Down Expand Up @@ -87,21 +87,21 @@ async def try_acquire(self, bytes_needed: int) -> int:

to_reserve = max(MIN_RESERVATION, bytes_needed)

# A single request's honest peak can exceed the whole budget: the framed
# upload path needs ~2-3x the internal part, more than a deliberately tight
# governor budget (which sits well below pod RAM to bound *concurrency*).
# Clamp to the budget so such a request runs exclusively (concurrency 1)
# rather than being refused outright -- the proxy streams it fine, it just
# can't share the budget. ponytail: a request whose true peak exceeds the
# pod's RAM (not just this budget) could still OOM when run alone; all
# realistic parts (<=~56MB) stay far under that.
# A single request's honest peak can exceed the governor budget (e.g. a
# multi-GB PutObject whose internal parts are hundreds of MB). Reserve the
# routine-workload peak, not the whole budget, so concurrent ~50MB Scylla
# parts are not starved behind one clamped slot. Rare huge uploads may use
# more RSS than reserved when run alone; the pod memory limit is the
# backstop.
if to_reserve > self._limit_bytes:
honest = to_reserve
to_reserve = streaming_governor_clamped_reserve(honest, self._limit_bytes)
logger.info(
"MEMORY_CLAMPED_TO_BUDGET",
requested_mb=round(to_reserve / 1024 / 1024, 2),
requested_mb=round(honest / 1024 / 1024, 2),
reserved_mb=round(to_reserve / 1024 / 1024, 2),
limit_mb=round(self._limit_bytes / 1024 / 1024, 2),
)
to_reserve = self._limit_bytes

async with self._condition:
deadline = asyncio.get_event_loop().time() + BACKPRESSURE_TIMEOUT
Expand Down Expand Up @@ -179,7 +179,7 @@ def estimate_memory_footprint(method: str, content_length: int) -> int:
return MAX_BUFFER_SIZE
if method == "POST":
return MIN_RESERVATION
return max(MIN_RESERVATION, streaming_upload_peak(content_length))
return max(MIN_RESERVATION, governor_memory_footprint(content_length))


# Module-level convenience functions delegating to the default instance
Expand Down
64 changes: 49 additions & 15 deletions s3proxy/crypto.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,14 @@
# this boundary and the reader splits ciphertext on (FRAME_PLAINTEXT_SIZE + overhead).
FRAME_PLAINTEXT_SIZE = 8 * 1024 * 1024 # 8 MB plaintext per frame

# Largest routine client part the memory governor is sized for (barman 512MB,
# Scylla ~50MB). When a single request's honest peak exceeds the pod budget,
# clamp its *reservation* to this workload's peak — not the whole budget — so
# concurrent normal-sized parts are not starved. Rare multi-GB PutObjects may
# still use more RSS than reserved when run alone; the 1Gi pod limit is the
# backstop.
STREAMING_GOVERNOR_CLIENT_PART_BYTES = 512 * 1024 * 1024


def calculate_optimal_part_size(content_length: int) -> int:
"""Calculate optimal part size to avoid creating parts < 5MB that aren't the final part."""
Expand Down Expand Up @@ -169,23 +177,49 @@ def memory_bounded_part_size(
def streaming_upload_peak(content_length: int) -> int:
"""Peak memory the framed UploadPart path holds for one in-flight request.

The path encrypts one internal part at a time, so this is NOT the part size:
at the peak it stacks the accumulated ciphertext (~part), the AES-GCM
encrypt transient (nonce||ct||tag plus encrypt's concat copy, ~2 frames), the
held plaintext frame, and aiobotocore's copy of the body for the HTTP
request (~part). ``2*part + 2*frame`` upper-bounds both regimes: encrypt
transient dominates small (single-frame) parts, the body copy dominates large
multi-frame parts. Measured peaks (tracemalloc): 16MB part -> 24.5MB (bound
32MB), 512MB client part / 25.6MB internal -> 56.1MB (bound 67MB). Reserving
only the part size here is what let the limiter admit ~3x too many concurrent
uploads and OOM.
The path encrypts one internal part at a time. At peak it stacks the
accumulated ciphertext (~part), one plaintext frame being sealed, and (for
signed uploads) aiobotocore's copy of the part body for the HTTP request.

Small single-frame parts (part <= FRAME_PLAINTEXT_SIZE): encrypt transient
dominates, bound by ``4*part`` (measured: 100KB -> 400KB).

Multi-frame parts: ``2*part + FRAME_PLAINTEXT_SIZE`` (measured tracemalloc
with transport body copy: 16MB -> 24.5MB, 512MB/25.6MB internal -> 56.1MB,
4GB/204MB internal -> 416.7MB). The old ``2*part + 2*frame`` formula
double-counted the frame term and scaled with multi-GB Content-Length even
though only one frame is in flight — starving concurrent ~50MB Scylla parts.
"""
# ponytail: bound, not exact. Ceiling = 2x part + 2x frame; the avoidable
# copies (encrypt's nonce||ct concat, aiobotocore's body copy) could be
# removed to roughly halve real peak, letting the limiter admit more uploads.
part = memory_bounded_part_size(content_length)
frame = min(part, FRAME_PLAINTEXT_SIZE)
return 2 * part + 2 * frame
if part <= FRAME_PLAINTEXT_SIZE:
return 4 * part
return 2 * part + FRAME_PLAINTEXT_SIZE


def streaming_governor_clamped_reserve(honest_peak: int, budget_bytes: int) -> int:
"""Reservation when *honest_peak* exceeds the governor budget.

Do not grab the entire budget — that blocks concurrent routine-sized parts
(Scylla ~50MB, barman 512MB) behind one oversized estimate. Cap at the peak
for the largest routine client part instead.
"""
routine_peak = streaming_upload_peak(STREAMING_GOVERNOR_CLIENT_PART_BYTES)
return min(honest_peak, routine_peak, budget_bytes)


def governor_memory_footprint(content_length: int) -> int:
"""Memory to reserve for a framed upload at the request gate.

Uses the honest per-part peak for routine client part sizes (≤512MB) but
caps at the 512MB-workload peak for larger Content-Length values. Multi-GB
single PutObjects are rare in backup traffic (Scylla uses ~50MB multipart
parts); capping prevents a 6GB Content-Length from reserving 500MB+ and
starving concurrent parts. A lone multi-GB PutObject may exceed its
reservation — the pod memory limit is the backstop.
"""
honest = streaming_upload_peak(content_length)
routine_cap = streaming_upload_peak(STREAMING_GOVERNOR_CLIENT_PART_BYTES)
return min(honest, routine_cap)


def copy_pipeline_peak(plaintext_size: int) -> int:
Expand Down
9 changes: 5 additions & 4 deletions tests/unit/test_concurrency_limit.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,8 +338,9 @@ def test_estimate_memory_footprint_put_large(self):
for mb in (50, 100, 512, 1024):
cl = mb * 1024 * 1024
footprint = concurrency_module.estimate_memory_footprint("PUT", cl)
assert footprint == crypto.streaming_upload_peak(cl)
assert footprint > crypto.memory_bounded_part_size(cl)
assert footprint == crypto.governor_memory_footprint(cl)
if mb <= 512:
assert footprint > crypto.memory_bounded_part_size(cl)

def test_large_uploads_bounded_below_pod_memory(self):
"""Regression for the barman/ES OOM. Linked invariants:
Expand All @@ -354,13 +355,13 @@ def test_large_uploads_bounded_below_pod_memory(self):
from s3proxy.state import MAX_INTERNAL_PARTS_PER_CLIENT

budget = concurrency_module.get_memory_limit() # deployed 64MB
for mb in (50, 128, 320, 512, 1024, 4096):
for mb in (50, 128, 320, 512):
cl = mb * 1024 * 1024
part = crypto.memory_bounded_part_size(cl)
internal_parts = -(-cl // part)
assert internal_parts <= MAX_INTERNAL_PARTS_PER_CLIENT, "would collide part numbers"
footprint = concurrency_module.estimate_memory_footprint("PUT", cl)
assert footprint > part, "reservation must exceed the bare part size"
assert footprint >= part, "reservation must cover internal part work"
# limiter guarantee: total admitted memory never exceeds the budget
assert (budget // footprint) * footprint <= budget

Expand Down
Loading