Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
fail-fast: false
matrix:
shard: [memory_usage, memory_leak, memory_copy, core, multipart, copy_range, misc]
timeout-minutes: ${{ matrix.shard == 'memory_copy' && 25 || (contains(fromJSON('["memory_usage", "memory_leak"]'), matrix.shard) && 15 || 10) }}
timeout-minutes: ${{ matrix.shard == 'memory_copy' && 35 || (contains(fromJSON('["memory_usage", "memory_leak"]'), matrix.shard) && 15 || 10) }}
steps:
- uses: actions/checkout@v7

Expand Down
8 changes: 6 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: test test-all test-unit test-integration test-integration-shard test-run test-oom verify-copy-memory e2e cluster lint
.PHONY: test test-all test-unit test-integration test-integration-shard test-run test-oom verify-copy-memory verify-passthrough e2e cluster lint

# Lint: ruff check + format check
lint:
Expand All @@ -16,10 +16,14 @@ test-unit:
verify-copy-memory:
uv run pytest tests/integration/test_copy_per_part_metrics.py -m e2e -v -n0

# Exhaustive UploadPartCopy passthrough proof (real subprocess + MinIO).
verify-passthrough:
uv run pytest tests/integration/test_upload_part_copy_passthrough_e2e.py -m e2e -v -n0

# Integration shards for parallel CI (make test-integration-shard SHARD=memory_usage)
INTEGRATION_memory_usage_TESTS = tests/integration/test_memory_usage.py
INTEGRATION_memory_leak_TESTS = tests/integration/test_memory_leak.py
INTEGRATION_memory_copy_TESTS = tests/integration/test_copy_memory_governor.py tests/integration/test_copy_per_part_metrics.py
INTEGRATION_memory_copy_TESTS = tests/integration/test_copy_memory_governor.py tests/integration/test_copy_per_part_metrics.py tests/integration/test_upload_part_copy_passthrough_e2e.py
INTEGRATION_memory_copy_PYTEST_OPTS = -n0
INTEGRATION_core_TESTS = \
tests/integration/test_integration.py \
Expand Down
17 changes: 10 additions & 7 deletions s3proxy/concurrency.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ async def try_acquire(self, bytes_needed: int, *, copy: bool = False) -> int:
if exclusive:
self._pending_exclusive += 1
try:
logged_backpressure = False
while not self._can_admit(to_reserve, exclusive):
remaining = deadline - asyncio.get_event_loop().time()
if remaining <= 0:
Expand All @@ -158,13 +159,15 @@ async def try_acquire(self, bytes_needed: int, *, copy: bool = False) -> int:
f"Memory limit: {active_mb:.0f}MB + "
f"{request_mb:.0f}MB > {limit_mb:.0f}MB"
)
logger.info(
"MEMORY_BACKPRESSURE",
active_mb=round(self._active_bytes / 1024 / 1024, 2),
requested_mb=round(to_reserve / 1024 / 1024, 2),
limit_mb=round(self._limit_bytes / 1024 / 1024, 2),
remaining_sec=round(remaining, 1),
)
if not logged_backpressure:
logger.info(
"MEMORY_BACKPRESSURE",
active_mb=round(self._active_bytes / 1024 / 1024, 2),
requested_mb=round(to_reserve / 1024 / 1024, 2),
limit_mb=round(self._limit_bytes / 1024 / 1024, 2),
remaining_sec=round(remaining, 1),
)
logged_backpressure = True
with contextlib.suppress(TimeoutError):
await asyncio.wait_for(self._condition.wait(), timeout=remaining)

Expand Down
17 changes: 13 additions & 4 deletions s3proxy/crypto.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,10 +264,9 @@ def copy_internal_part_size(plaintext_size: int) -> int:
def copy_chunk_peak(chunk_plaintext_bytes: int) -> int:
"""Peak memory while encrypting one internal copy chunk (streaming path).

The streaming copy path acquires/releases this amount per internal part so
memory is free between chunks and during S3 upload I/O. Reservation covers
read-buffer slack plus framed encrypt peak; ciphertext upload runs after
release (smaller RSS than encrypt peak).
The streaming copy path acquires/releases this amount per internal part.
Reservation covers read-buffer slack plus framed encrypt peak and the
ciphertext buffer through S3 upload (released after del ciphertext).
"""
framed = (
4 * chunk_plaintext_bytes
Expand All @@ -280,6 +279,16 @@ def copy_chunk_peak(chunk_plaintext_bytes: int) -> int:
return peak


# UploadPartCopy passthrough moves bytes server-side; in-process peak is tiny.
COPY_PASSTHROUGH_SEGMENT_PEAK = 64 * 1024


def copy_passthrough_segment_peak(plaintext_size: int) -> int:
"""Governor reservation for one native copy segment (no decrypt/re-encrypt)."""
_ = plaintext_size
return COPY_PASSTHROUGH_SEGMENT_PEAK


def copy_small_buffered_peak(plaintext_size: int) -> int:
"""Peak for a small copy that buffers the whole object in RAM."""
return max(MAX_BUFFER_SIZE, 3 * plaintext_size)
Expand Down
Loading