From bb391965be091872f76c412de4488317a0bacb68 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Tue, 30 Jun 2026 09:18:41 +0200 Subject: [PATCH 1/2] fix: don't URL-encode V2 continuation tokens under encoding-type=url MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ListObjectsV2 continuation tokens are opaque cursors, not keys. Per the S3 spec, encoding-type=url only encodes Key/Prefix/Delimiter/StartAfter, and botocore never URL-decodes the continuation token. The V2 serializer ran NextContinuationToken / ContinuationToken through _encode_key(), so under encoding-type=url a key-shaped backend token like 'production-v3/production/base/.../data_0007.tar' became '...%2F...data_0007.tar'. That mangled token can't round-trip: the backend never advances, the same page (and token) repeats, and botocore's paginator aborts with 'The same next token was received twice' — which wedged CNPG barman-cloud base backups and retention on large catalogs. Emit V2 continuation tokens XML-escaped only, regardless of encoding_type. Keys remain URL-encoded under encoding-type=url. The handler already documented this intent ('opaque and must be passed back unchanged'); the serializer violated it. --- s3proxy/xml_responses.py | 10 ++- .../test_list_objects_continuation_token.py | 68 +++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_list_objects_continuation_token.py diff --git a/s3proxy/xml_responses.py b/s3proxy/xml_responses.py index 78c82bf..1f23b87 100644 --- a/s3proxy/xml_responses.py +++ b/s3proxy/xml_responses.py @@ -70,14 +70,20 @@ def list_objects( {obj.get("storage_class", "STANDARD")}{owner_xml} """ + # V2 continuation tokens are opaque cursors, not keys. Per the S3 spec only + # Key/Prefix/Delimiter/StartAfter are URL-encoded under encoding-type=url, and + # clients (botocore) never URL-decode the continuation token. URL-encoding it + # corrupts the round-trip (e.g. '/' -> '%2F'), the backend can't advance, and + # the same token repeats -> botocore aborts with "same next token received + # twice". XML-escape only, regardless of encoding_type. next_token_xml = ( - f"{_encode_key(next_token, encoding_type)}" + f"{escape(next_token)}" if next_token else "" ) continuation_token_xml = ( - f"{_encode_key(continuation_token, encoding_type)}" + f"{escape(continuation_token)}" if continuation_token is not None else "" ) diff --git a/tests/unit/test_list_objects_continuation_token.py b/tests/unit/test_list_objects_continuation_token.py new file mode 100644 index 0000000..1f189e0 --- /dev/null +++ b/tests/unit/test_list_objects_continuation_token.py @@ -0,0 +1,68 @@ +"""Self-check: V2 continuation tokens must not be URL-encoded. + +barman-cloud / botocore send encoding-type=url on ListObjectsV2. Continuation +tokens are opaque cursors, not keys: per the S3 spec only Key/Prefix/Delimiter/ +StartAfter are URL-encoded, and botocore never URL-decodes the token. When the +proxy URL-encoded a key-shaped token ('a/b/c.tar' -> 'a%2Fb%2Fc.tar') the token +could not round-trip, the backend kept returning the first page, and botocore +aborted the paginator with "The same next token was received twice". + +This proves the V2 serializer emits the token verbatim (XML-escaped only) even +under encoding-type=url, so it survives the round-trip and pagination advances. +""" + +from xml.etree.ElementTree import fromstring + +from s3proxy.xml_responses import list_objects + +TOKEN = "production-v3/production/base/20260619T223000/data_0007.tar" +_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def _parse(xml: str) -> dict: + root = fromstring(xml) + return {child.tag.replace(_NS, ""): child.text for child in root} + + +def test_v2_continuation_token_not_url_encoded(): + xml = list_objects( + bucket="oceanio-dc2-postgresql-backups", + prefix="production-v3/production/base/", + max_keys=1000, + is_truncated=True, + next_token=TOKEN, + objects=[], + continuation_token=TOKEN, + encoding_type="url", # what barman/botocore sends + ) + fields = _parse(xml) + + # Verbatim token -> '/' preserved, not '%2F'. This is the actual bug guard. + assert fields["NextContinuationToken"] == TOKEN + assert fields["ContinuationToken"] == TOKEN + assert "%2F" not in xml + + # Keys are still URL-encoded under encoding-type=url (regression guard). + key_xml = list_objects( + bucket="b", + prefix="", + max_keys=1000, + is_truncated=False, + next_token=None, + objects=[ + { + "key": "a/b/c.tar", + "last_modified": "2026-06-24T09:00:00", + "etag": "x", + "size": 1, + "storage_class": "STANDARD", + } + ], + encoding_type="url", + ) + assert "a%2Fb%2Fc.tar" in key_xml + + +if __name__ == "__main__": + test_v2_continuation_token_not_url_encoded() + print("ok") From bb29dd86907dc33f8bb5459365502d7352ca1de1 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Tue, 30 Jun 2026 09:27:05 +0200 Subject: [PATCH 2/2] style: ruff format (collapse one-line ternary) --- s3proxy/xml_responses.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/s3proxy/xml_responses.py b/s3proxy/xml_responses.py index 1f23b87..0af5383 100644 --- a/s3proxy/xml_responses.py +++ b/s3proxy/xml_responses.py @@ -77,9 +77,7 @@ def list_objects( # the same token repeats -> botocore aborts with "same next token received # twice". XML-escape only, regardless of encoding_type. next_token_xml = ( - f"{escape(next_token)}" - if next_token - else "" + f"{escape(next_token)}" if next_token else "" ) continuation_token_xml = (