diff --git a/s3proxy/client/s3.py b/s3proxy/client/s3.py index aa6992a..a7518f7 100644 --- a/s3proxy/client/s3.py +++ b/s3proxy/client/s3.py @@ -346,6 +346,17 @@ async def list_parts( _add_optional_kwargs(kwargs, PartNumberMarker=part_number_marker) return await self._cached_client.list_parts(**kwargs) + async def list_all_parts(self, bucket: str, key: str, upload_id: str) -> list[dict[str, Any]]: + """List ALL parts of a multipart upload, following pagination.""" + parts: list[dict[str, Any]] = [] + marker: int | None = None + while True: + resp = await self.list_parts(bucket, key, upload_id, part_number_marker=marker) + parts.extend(resp.get("Parts", [])) + if not resp.get("IsTruncated"): + return parts + marker = resp.get("NextPartNumberMarker") + async def list_buckets(self) -> dict[str, Any]: """List all buckets owned by the authenticated user.""" return await self._cached_client.list_buckets() diff --git a/s3proxy/client/verifier.py b/s3proxy/client/verifier.py index 0f67e9f..397b926 100644 --- a/s3proxy/client/verifier.py +++ b/s3proxy/client/verifier.py @@ -48,11 +48,10 @@ def _parse_v4_credential( Returns: (credentials, date_stamp, region, service, error) """ - try: - parts = credential.split("/") - access_key, date_stamp, region, service = parts[0], parts[1], parts[2], parts[3] - except IndexError, ValueError: + parts = credential.split("/") + if len(parts) < 4: return None, "", "", "", "Invalid credential format" + access_key, date_stamp, region, service = parts[0], parts[1], parts[2], parts[3] secret_key = self.credentials_store.get(access_key) if not secret_key: diff --git a/s3proxy/handlers/multipart/lifecycle.py b/s3proxy/handlers/multipart/lifecycle.py index f385f4c..9696a73 100644 --- a/s3proxy/handlers/multipart/lifecycle.py +++ b/s3proxy/handlers/multipart/lifecycle.py @@ -254,11 +254,11 @@ def internal_to_client_part(internal_part_number: int) -> int: state = await self.multipart_manager.create_upload(bucket, key, upload_id, dek, kid) try: - parts_resp = await client.list_parts(bucket, key, upload_id) + all_parts = await client.list_all_parts(bucket, key, upload_id) # Group S3 internal parts by client part number client_parts: dict[int, list[dict]] = defaultdict(list) - for part in parts_resp.get("Parts", []): + for part in all_parts: internal_part_num = part.get("PartNumber", 0) client_part_num = internal_to_client_part(internal_part_num) client_parts[client_part_num].append(part) @@ -268,7 +268,7 @@ def internal_to_client_part(internal_part_number: int) -> int: bucket=bucket, key=key, upload_id=upload_id[:20] + "...", - s3_parts=len(parts_resp.get("Parts", [])), + s3_parts=len(all_parts), client_parts=sorted(client_parts.keys()), ) diff --git a/s3proxy/state/recovery.py b/s3proxy/state/recovery.py index 759163d..e3172ea 100644 --- a/s3proxy/state/recovery.py +++ b/s3proxy/state/recovery.py @@ -65,7 +65,7 @@ async def reconstruct_upload_state_from_s3( # Step 2: List all uploaded parts from S3 try: - parts_response = await s3_client.list_parts(bucket, key, upload_id, max_parts=10000) + s3_parts = await s3_client.list_all_parts(bucket, key, upload_id) except Exception as e: logger.error( "RECONSTRUCT_LIST_PARTS_FAILED", @@ -77,7 +77,6 @@ async def reconstruct_upload_state_from_s3( return None # Step 3: Group S3 parts by client part number - s3_parts = parts_response.get("Parts", []) logger.debug( "RECONSTRUCT_PARTS", bucket=bucket, diff --git a/tests/unit/test_multipart.py b/tests/unit/test_multipart.py index 3e0becd..0e2a468 100644 --- a/tests/unit/test_multipart.py +++ b/tests/unit/test_multipart.py @@ -686,19 +686,17 @@ async def mock_read(): mock_body.read = mock_read mock_client.get_object = AsyncMock(return_value={"Body": mock_body}) - # Mock list_parts to return some uploaded parts (using internal part numbers) + # Mock list_all_parts to return some uploaded parts (using internal part numbers) # With MAX_INTERNAL_PARTS_PER_CLIENT=20: # - Client part 1 -> internal part 1 # - Client part 2 -> internal part 21 # - Client part 3 -> internal part 41 - mock_client.list_parts = AsyncMock( - return_value={ - "Parts": [ - {"PartNumber": 1, "Size": 1028, "ETag": '"etag1"'}, # Client part 1 - {"PartNumber": 21, "Size": 2056, "ETag": '"etag2"'}, # Client part 2 - {"PartNumber": 41, "Size": 1028, "ETag": '"etag3"'}, # Client part 3 - ] - } + mock_client.list_all_parts = AsyncMock( + return_value=[ + {"PartNumber": 1, "Size": 1028, "ETag": '"etag1"'}, # Client part 1 + {"PartNumber": 21, "Size": 2056, "ETag": '"etag2"'}, # Client part 2 + {"PartNumber": 41, "Size": 1028, "ETag": '"etag3"'}, # Client part 3 + ] ) # Mock the crypto.unwrap_key to return the original DEK @@ -767,8 +765,8 @@ async def mock_read(): mock_body.read = mock_read mock_client.get_object = AsyncMock(return_value={"Body": mock_body}) - # Mock list_parts to fail - mock_client.list_parts = AsyncMock(side_effect=Exception("Upload not found")) + # Mock list_all_parts to fail + mock_client.list_all_parts = AsyncMock(side_effect=Exception("Upload not found")) with patch("s3proxy.crypto.unwrap_key", return_value=original_dek): state = await reconstruct_upload_state_from_s3( @@ -797,8 +795,8 @@ async def mock_read(): mock_body.read = mock_read mock_client.get_object = AsyncMock(return_value={"Body": mock_body}) - # Mock list_parts to return empty list - mock_client.list_parts = AsyncMock(return_value={"Parts": []}) + # Mock list_all_parts to return empty list + mock_client.list_all_parts = AsyncMock(return_value=[]) with patch("s3proxy.crypto.unwrap_key", return_value=original_dek): state = await reconstruct_upload_state_from_s3( diff --git a/tests/unit/test_state_recovery_fix.py b/tests/unit/test_state_recovery_fix.py index 56705c1..4fb5fa5 100644 --- a/tests/unit/test_state_recovery_fix.py +++ b/tests/unit/test_state_recovery_fix.py @@ -60,31 +60,29 @@ async def mock_get_object(bucket_name, key_name): mock_s3_client.get_object = mock_get_object - # Mock list_parts to return 3 client parts (using internal part numbers) + # Mock list_all_parts to return 3 internal parts spanning 3 client parts # With MAX_INTERNAL_PARTS_PER_CLIENT=20: # - Client part 1 -> internal part 1 # - Client part 2 -> internal part 21 # - Client part 3 -> internal part 41 - mock_s3_client.list_parts = AsyncMock( - return_value={ - "Parts": [ - { - "PartNumber": 1, # Client part 1 - "Size": 5_242_880 + ENCRYPTION_OVERHEAD, - "ETag": '"etag-1"', - }, - { - "PartNumber": 21, # Client part 2 - "Size": 5_242_880 + ENCRYPTION_OVERHEAD, - "ETag": '"etag-2"', - }, - { - "PartNumber": 41, # Client part 3 - "Size": 3_000_000 + ENCRYPTION_OVERHEAD, - "ETag": '"etag-3"', - }, - ] - } + mock_s3_client.list_all_parts = AsyncMock( + return_value=[ + { + "PartNumber": 1, # Client part 1 + "Size": 5_242_880 + ENCRYPTION_OVERHEAD, + "ETag": '"etag-1"', + }, + { + "PartNumber": 21, # Client part 2 + "Size": 5_242_880 + ENCRYPTION_OVERHEAD, + "ETag": '"etag-2"', + }, + { + "PartNumber": 41, # Client part 3 + "Size": 3_000_000 + ENCRYPTION_OVERHEAD, + "ETag": '"etag-3"', + }, + ] ) # Reconstruct state @@ -141,8 +139,8 @@ async def mock_get_object(bucket_name, key_name): mock_s3_client.get_object = mock_get_object - # Mock list_parts to return empty list - mock_s3_client.list_parts = AsyncMock(return_value={"Parts": []}) + # Mock list_all_parts to return empty list + mock_s3_client.list_all_parts = AsyncMock(return_value=[]) # Reconstruct state state = await reconstruct_upload_state_from_s3( @@ -202,8 +200,8 @@ async def mock_get_object(bucket_name, key_name): mock_s3_client.get_object = mock_get_object - # Mock list_parts to fail - mock_s3_client.list_parts = AsyncMock(side_effect=Exception("List failed")) + # Mock list_all_parts to fail + mock_s3_client.list_all_parts = AsyncMock(side_effect=Exception("List failed")) # Reconstruct state should return None state = await reconstruct_upload_state_from_s3( @@ -237,31 +235,29 @@ async def mock_get_object(bucket_name, key_name): mock_s3_client.get_object = mock_get_object - # Mock list_parts with out-of-order client parts (using internal part numbers) + # Mock list_all_parts with out-of-order client parts (using internal part numbers) # With MAX_INTERNAL_PARTS_PER_CLIENT=20: # - Client part 1 -> internal part 1 # - Client part 3 -> internal part 41 # - Client part 5 -> internal part 81 - mock_s3_client.list_parts = AsyncMock( - return_value={ - "Parts": [ - { - "PartNumber": 1, # Client part 1 - "Size": 5_242_880 + ENCRYPTION_OVERHEAD, - "ETag": '"etag-1"', - }, - { - "PartNumber": 41, # Client part 3 - "Size": 5_242_880 + ENCRYPTION_OVERHEAD, - "ETag": '"etag-3"', - }, - { - "PartNumber": 81, # Client part 5 - "Size": 5_242_880 + ENCRYPTION_OVERHEAD, - "ETag": '"etag-5"', - }, - ] - } + mock_s3_client.list_all_parts = AsyncMock( + return_value=[ + { + "PartNumber": 1, # Client part 1 + "Size": 5_242_880 + ENCRYPTION_OVERHEAD, + "ETag": '"etag-1"', + }, + { + "PartNumber": 41, # Client part 3 + "Size": 5_242_880 + ENCRYPTION_OVERHEAD, + "ETag": '"etag-3"', + }, + { + "PartNumber": 81, # Client part 5 + "Size": 5_242_880 + ENCRYPTION_OVERHEAD, + "ETag": '"etag-5"', + }, + ] ) # Reconstruct state @@ -334,3 +330,97 @@ async def test_store_and_retrieve_reconstructed_state(self): assert 2 in retrieved.parts assert retrieved.next_internal_part_number == 3 assert retrieved.total_plaintext_size == 8_242_880 + + +class TestListAllPartsMethod: + """Test S3Client.list_all_parts follows IsTruncated pagination.""" + + @pytest.mark.asyncio + async def test_follows_pagination(self): + """list_all_parts must issue successive calls until IsTruncated is False.""" + from unittest.mock import patch + + from s3proxy.client.s3 import S3Client + + client = S3Client.__new__(S3Client) + + page1_parts = [ + {"PartNumber": i, "Size": 100, "ETag": f'"etag-{i}"'} for i in range(1, 1001) + ] + page2_parts = [{"PartNumber": 1001, "Size": 50, "ETag": '"etag-1001"'}] + calls: list[int | None] = [] + + async def fake_list_parts(bucket, key, upload_id, part_number_marker=None, max_parts=1000): + calls.append(part_number_marker) + if part_number_marker is None: + return {"Parts": page1_parts, "IsTruncated": True, "NextPartNumberMarker": 1000} + return {"Parts": page2_parts, "IsTruncated": False} + + with patch.object(client, "list_parts", fake_list_parts): + result = await client.list_all_parts("bucket", "key", "upload-id") + + assert len(result) == 1001 + assert result[-1]["PartNumber"] == 1001 + assert calls == [None, 1000], f"unexpected call sequence: {calls}" + + @pytest.mark.asyncio + async def test_single_page_no_extra_call(self): + """list_all_parts stops after the first non-truncated page.""" + from unittest.mock import patch + + from s3proxy.client.s3 import S3Client + + client = S3Client.__new__(S3Client) + calls: list = [] + + async def fake_list_parts(bucket, key, upload_id, part_number_marker=None, max_parts=1000): + calls.append(part_number_marker) + return {"Parts": [{"PartNumber": 1}], "IsTruncated": False} + + with patch.object(client, "list_parts", fake_list_parts): + result = await client.list_all_parts("bucket", "key", "upload-id") + + assert result == [{"PartNumber": 1}] + assert len(calls) == 1 + + +class TestRecoveryWithPaginatedParts: + """Recovery must reconstruct state from >1000 internal parts (two ListParts pages).""" + + @pytest.mark.asyncio + async def test_reconstruct_over_1000_parts(self): + """reconstruct_upload_state_from_s3 must include parts from all pages.""" + import base64 + + from s3proxy.state import json_dumps + + kek = crypto.generate_dek() + dek = crypto.generate_dek() + wrapped_dek = crypto.wrap_key(dek, kek) + state_data = json_dumps( + {"dek": base64.b64encode(wrapped_dek).decode(), "kid": RECOVERY_KID} + ) + + mock_s3_client = AsyncMock() + + async def mock_get_object(bucket_name, key_name): + return {"Body": AsyncMock(read=AsyncMock(return_value=state_data))} + + mock_s3_client.get_object = mock_get_object + + # list_all_parts returns 1001 parts across two pages (simulating paginated fetch) + # Parts 1–1000 → client parts 1–50; part 1001 → client part 51 + all_parts = [ + {"PartNumber": i, "Size": 100 + ENCRYPTION_OVERHEAD, "ETag": f'"etag-{i}"'} + for i in range(1, 1002) + ] + mock_s3_client.list_all_parts = AsyncMock(return_value=all_parts) + + state = await reconstruct_upload_state_from_s3( + mock_s3_client, "bucket", "key", "upload-id", _single_key_ring(kek) + ) + + assert state is not None + assert 51 in state.parts, "client part 51 (internal 1001) must be present" + assert state.next_internal_part_number == 1002 + assert len(state.parts) == 51