diff --git a/s3proxy/handlers/buckets.py b/s3proxy/handlers/buckets.py index fef9b07..4b66da3 100644 --- a/s3proxy/handlers/buckets.py +++ b/s3proxy/handlers/buckets.py @@ -123,8 +123,17 @@ async def handle_list_objects_v1(self, request: Request, creds: S3Credentials) - max_keys = int(query.get("max-keys", ["1000"])[0]) encoding_type = query.get("encoding-type", [""])[0] or None + # Serve the client's legacy V1 ListObjects via the backend's V2 API. + # Hetzner (and other modern S3 backends) only implement ListObjectsV2 and + # reject V1 with HTTP 400, which breaks any V1 client (scylla-manager's + # rclone 1.51.0, barman-cloud-backup-delete). V1 marker pagination is + # stateless (marker == last key returned), which maps exactly onto V2 + # StartAfter, so the translation is lossless for the recursive (no + # delimiter) listings these clients use. try: - resp = await client.list_objects_v1(bucket, prefix, marker, delimiter, max_keys) + resp = await client.list_objects_v2( + bucket, prefix, None, max_keys, delimiter, marker + ) except ClientError as e: self._raise_bucket_error(e, bucket) @@ -140,14 +149,20 @@ async def handle_list_objects_v1(self, request: Request, creds: S3Credentials) - if stripped is not None: common_prefixes.append(stripped) - # V1 uses NextMarker (or last key if truncated and no delimiter) - next_marker = _strip_minio_cache_suffix(resp.get("NextMarker")) - if resp.get("IsTruncated") and not next_marker: - # Fallback: use last object key or last common prefix - if objects: - next_marker = objects[-1]["key"] - elif common_prefixes: - next_marker = common_prefixes[-1] + # V1 resumes via marker == last key returned. Synthesize NextMarker from + # the largest RAW backend key/prefix on this page (not the filtered list: + # a page may be all-internal keys yet still truncated) so the next + # marker -> StartAfter resumes after everything returned. + # ponytail: O(page) max over the page. A single delimiter-level prefix + # with >max_keys *distinct* sub-prefixes can re-emit one boundary prefix + # across pages (harmless, idempotent for these read-only catalog walks); + # upgrade path is a redis-backed marker->continuation-token map. + next_marker = None + if resp.get("IsTruncated"): + raw = [o["Key"] for o in resp.get("Contents", [])] + raw += [cp["Prefix"] for cp in resp.get("CommonPrefixes", [])] + if raw: + next_marker = _strip_minio_cache_suffix(max(raw)) return Response( content=xml_responses.list_objects_v1( diff --git a/tests/unit/test_list_objects_v1_via_v2.py b/tests/unit/test_list_objects_v1_via_v2.py new file mode 100644 index 0000000..778dec2 --- /dev/null +++ b/tests/unit/test_list_objects_v1_via_v2.py @@ -0,0 +1,143 @@ +"""Self-check: client V1 ListObjects is served via the backend's V2 API. + +Hetzner Object Storage (and other modern S3 backends) only implement +ListObjectsV2 and reject legacy V1 ListObjects with HTTP 400 — which broke +every V1 client (scylla-manager's rclone 1.51.0, barman-cloud-backup-delete). + +This proves handle_list_objects_v1: +- calls the backend's list_objects_v2 (never list_objects_v1), +- maps the client's V1 marker onto V2 StartAfter, +- filters internal keys, and +- synthesizes a V1 NextMarker from the largest backend key when truncated. +""" + +import asyncio +import contextlib +import datetime as dt +from xml.etree.ElementTree import fromstring + +from s3proxy.handlers.buckets import BucketHandlerMixin + +INTERNAL_PREFIX = "s3proxy-internal/" +_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +class _URL: + def __init__(self, path, query): + self.path = path + self.query = query + + +class _Request: + def __init__(self, path, query): + self.url = _URL(path, query) + + +class _FakeClient: + def __init__(self, resp): + self.resp = resp + self.v2_calls = [] + + async def list_objects_v1(self, *a, **k): # must never be hit + raise AssertionError("backend V1 ListObjects must not be called") + + async def list_objects_v2( + self, + bucket, + prefix=None, + continuation_token=None, + max_keys=1000, + delimiter=None, + start_after=None, + ): + self.v2_calls.append({"start_after": start_after, "delimiter": delimiter, "prefix": prefix}) + return self.resp + + async def head_object(self, bucket, key): + return {"Metadata": {}} + + +class _Handler(BucketHandlerMixin): + handle_list_objects_v1 = BucketHandlerMixin.handle_list_objects_v1 + _process_list_objects = BucketHandlerMixin._process_list_objects + + def __init__(self, client): + self._fake = client + + def _parse_bucket(self, path): + return path.lstrip("/").split("/")[0] + + def _is_internal_key(self, key): + return key.startswith(INTERNAL_PREFIX) + + def _get_plaintext_size(self, meta, fallback): + return fallback + + def _get_effective_etag(self, meta, fallback): + return fallback.strip('"') + + def _client(self, creds): + client = self._fake + + @contextlib.asynccontextmanager + async def cm(): + yield client + + return cm() + + +def _obj(key): + return { + "Key": key, + "Size": 9, + "ETag": '"e"', + "LastModified": dt.datetime(2026, 6, 30, 9, 0, 0), + "StorageClass": "STANDARD", + } + + +def _run(resp, query, marker_expected_start_after): + handler = _Handler(_FakeClient(resp)) + req = _Request("/postgres-backups", query) + out = asyncio.run(handler.handle_list_objects_v1(req, creds=None)) + root = fromstring(out.body) + return ( + handler, + {c.tag.replace(_NS, ""): c.text for c in root}, + [e.find(f"{_NS}Key").text for e in root.findall(f"{_NS}Contents")], + ) + + +def test_v1_served_via_v2_marker_maps_to_start_after(): + resp = { + "Contents": [_obj("a.txt"), _obj(f"{INTERNAL_PREFIX}meta"), _obj("b.txt")], + "CommonPrefixes": [], + "IsTruncated": True, + } + handler, fields, keys = _run(resp, "prefix=&marker=last-seen.txt", "last-seen.txt") + + # Backend was hit via V2 with the client marker mapped to StartAfter. + assert handler._fake.v2_calls == [ + {"start_after": "last-seen.txt", "delimiter": None, "prefix": ""} + ] + # Internal key filtered out; real keys preserved in order. + assert keys == ["a.txt", "b.txt"] + # Truncated -> NextMarker is the largest RAW backend key (incl. would-be internal). + assert fields["IsTruncated"] == "true" + assert fields["NextMarker"] == f"{INTERNAL_PREFIX}meta" # max("a.txt","b.txt",internal) + # Echoes the request marker. + assert fields["Marker"] == "last-seen.txt" + + +def test_v1_not_truncated_has_no_next_marker(): + resp = {"Contents": [_obj("only.txt")], "CommonPrefixes": [], "IsTruncated": False} + _, fields, keys = _run(resp, "prefix=", None) + assert keys == ["only.txt"] + assert fields["IsTruncated"] == "false" + assert fields.get("NextMarker") is None + + +if __name__ == "__main__": + test_v1_served_via_v2_marker_maps_to_start_after() + test_v1_not_truncated_has_no_next_marker() + print("ok")