From a1c66130dce8ccaf822c87e1de283e683ffbf35c Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Mon, 26 Oct 2026 18:48:14 +0100 Subject: [PATCH 01/12] Add with_api_key to backend_v2 methods, and a dedicated API key for serverless-init Each sender uses its own API key, so with_api_key tells where a request comes from. Co-Authored-By: Claude Opus 5.5 --- tests/test_the_test/test_mock_backend_v2.py | 43 +++++++++++++ utils/_context/containers.py | 4 +- utils/interfaces/_backend_v2.py | 69 ++++++++++++++------- 3 files changed, 93 insertions(+), 23 deletions(-) diff --git a/tests/test_the_test/test_mock_backend_v2.py b/tests/test_the_test/test_mock_backend_v2.py index d8fb76e10fd..6c9c06c3bcb 100644 --- a/tests/test_the_test/test_mock_backend_v2.py +++ b/tests/test_the_test/test_mock_backend_v2.py @@ -13,6 +13,7 @@ import zstandard from utils import scenarios +from utils.interfaces._backend_v2 import _BackendV2InterfaceValidator from utils.mocked_backend import MockedBackendClient, MockedBackendServer from utils.mocked_backend.ffe import UFC_FIXTURE_RELATIVE_PATH @@ -231,3 +232,45 @@ def test_health_endpoint(server: MockedBackendServer, tmp_path: Path): assert response.json() == {"status": "ok"} # a control endpoint: not recorded assert list(tmp_path.glob("*.json")) == [] + + +def _backend_v2_with_requests(tmp_path: Path, requests_: list[tuple[str, str | None]]) -> _BackendV2InterfaceValidator: + """A backend_v2 interface that received a series request per (filename, api_key)""" + interface = _BackendV2InterfaceValidator() + interface.configure(str(tmp_path), replay=False) + for filename, api_key in requests_: + headers = [["Content-Type", "application/json"]] + ([["DD-API-KEY", api_key]] if api_key else []) + file = Path(interface.log_folder) / filename + data = { + "log_filename": str(file), + "path": "/api/v2/series", + "request": {"headers": headers, "content": {"series": [{"metric": filename}]}}, + "response": {}, + } + file.write_text(json.dumps(data)) + interface.ingest_file(str(file)) + return interface + + +@scenarios.test_the_test +def test_backend_v2_with_api_key_only_returns_requests_sent_with_this_key(tmp_path: Path): + interface = _backend_v2_with_requests( + tmp_path, [("000__agent.json", "agent-key"), ("001__sidecar.json", "sidecar-key"), ("002__nokey.json", None)] + ) + + assert [d["log_filename"].split("/")[-1] for d in interface.get_data(with_api_key="sidecar-key")] == [ + "001__sidecar.json" + ] + assert [point["metric"] for _, point in interface.get_metrics(with_api_key="agent-key")] == ["000__agent.json"] + assert list(interface.get_data(with_api_key="unknown-key")) == [] + # without it, every request + assert len(list(interface.get_data())) == 3 + + +@scenarios.test_the_test +def test_backend_v2_with_api_key_ignores_the_header_name_case(tmp_path: Path): + interface = _backend_v2_with_requests(tmp_path, [("000__agent.json", "agent-key")]) + data = next(interface.get_data()) + data["request"]["headers"] = [["dd-api-key", "agent-key"]] + + assert len(list(interface.get_data(with_api_key="agent-key"))) == 1 diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 1a4f5e88c18..f8eaf3d1958 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -53,6 +53,8 @@ # fake key of length 32 FAKE_DD_API_KEY = "0123456789abcdef0123456789abcdef" +# its own key tells, in the mocked backend, what serverless-init sent (see interfaces.backend_v2 with_api_key) +SERVERLESS_INIT_FAKE_DD_API_KEY = "fedcba9876543210fedcba9876543210" _DEFAULT_NETWORK_NAME = "system-tests_default" _NETWORK_NAME = "bridge" if "GITLAB_CI" in os.environ else _DEFAULT_NETWORK_NAME @@ -956,7 +958,7 @@ def __init__(self) -> None: name="ffe-serverless-init", image_name="datadog/serverless-init:1.10.4", environment={ - "DD_API_KEY": FAKE_DD_API_KEY, + "DD_API_KEY": SERVERLESS_INIT_FAKE_DD_API_KEY, "DD_SITE": "mock-intake.invalid", "DD_SERVICE": "ffe-system-tests-serverless-init", "DD_ENV": "system-tests", diff --git a/utils/interfaces/_backend_v2.py b/utils/interfaces/_backend_v2.py index 1cb8193d627..89214fadd9a 100644 --- a/utils/interfaces/_backend_v2.py +++ b/utils/interfaces/_backend_v2.py @@ -33,7 +33,17 @@ def ingest_all_files(self) -> None: # same path as given by the watchdog, so files are not ingested twice self.ingest_file(str(file)) - def get_traces(self, request: HttpResponse | None = None) -> Generator[tuple[dict, DataDogAgentTrace]]: + def get_data(self, path_filters: Iterable[str] | str | None = None, *, with_api_key: str | None = None): + """Same as other interfaces. with_api_key: only requests sent with this API key. Each sender (agent, library, + serverless-init...) uses its own key, it tells where a request comes from. + """ + for data in super().get_data(path_filters=path_filters): + if with_api_key is None or _get_api_key(data) == with_api_key: + yield data + + def get_traces( + self, request: HttpResponse | None = None, *, with_api_key: str | None = None + ) -> Generator[tuple[dict, DataDogAgentTrace]]: """Attempts to fetch the traces the agent submitted to the mocked backend. When a valid request is given, then we filter the spans to the ones sampled @@ -46,7 +56,7 @@ def get_traces(self, request: HttpResponse | None = None) -> Generator[tuple[dic if rid: logger.debug(f"Will try to find backend_v2 spans related to request {rid}") - for data in self.get_data(path_filters="/api/v0.2/traces"): + for data in self.get_data(path_filters="/api/v0.2/traces", with_api_key=with_api_key): logger.debug(f"Looking at backend_v2 data {data['log_filename']}") if "content" not in data["request"]: @@ -78,7 +88,9 @@ def get_traces(self, request: HttpResponse | None = None) -> Generator[tuple[dic yield data, trace break - def get_spans(self, request: HttpResponse | None = None) -> Generator[tuple[dict, DataDogAgentSpan], None, None]: + def get_spans( + self, request: HttpResponse | None = None, *, with_api_key: str | None = None + ) -> Generator[tuple[dict, DataDogAgentSpan], None, None]: """Attempts to fetch the spans the agent submitted to the mocked backend. When a valid request is given, then we filter the spans to the ones sampled @@ -91,18 +103,20 @@ def get_spans(self, request: HttpResponse | None = None) -> Generator[tuple[dict if rid: logger.debug(f"Will try to find backend_v2 spans related to request {rid}") - for data, trace in self.get_traces(request=request): + for data, trace in self.get_traces(request=request, with_api_key=with_api_key): for span in trace.spans: if rid is None or span.get_rid() == rid: yield data, span - def get_spans_list(self, request: HttpResponse | None = None) -> list[DataDogAgentSpan]: - return [span for _, span in self.get_spans(request)] + def get_spans_list( + self, request: HttpResponse | None = None, *, with_api_key: str | None = None + ) -> list[DataDogAgentSpan]: + return [span for _, span in self.get_spans(request, with_api_key=with_api_key)] def get_appsec_data( - self, request: HttpResponse + self, request: HttpResponse, *, with_api_key: str | None = None ) -> Generator[tuple[dict[str, Any], DataDogAgentSpan, dict[str, Any]], Any, None]: - for data, span in self.get_spans(request): + for data, span in self.get_spans(request, with_api_key=with_api_key): json_data = span.meta.get("_dd.appsec.json") if json_data is not None: yield data, span, json_data @@ -111,11 +125,11 @@ def get_appsec_data( elif (v1_metastruct_data := span.meta.get("appsec")) is not None: yield data, span, v1_metastruct_data - def get_profiling_data(self): - yield from self.get_data(path_filters="/api/v2/profile") + def get_profiling_data(self, *, with_api_key: str | None = None): + yield from self.get_data(path_filters="/api/v2/profile", with_api_key=with_api_key) - def get_telemetry_data(self, *, flatten_message_batches: bool = True): - all_data = self.get_data(path_filters="/api/v2/apmtelemetry") + def get_telemetry_data(self, *, flatten_message_batches: bool = True, with_api_key: str | None = None): + all_data = self.get_data(path_filters="/api/v2/apmtelemetry", with_api_key=with_api_key) if flatten_message_batches: yield from all_data else: @@ -131,8 +145,8 @@ def get_telemetry_data(self, *, flatten_message_batches: bool = True): else: yield data - def assert_trace_exists(self, request: HttpResponse): - for _, _ in self.get_traces(request=request): + def assert_trace_exists(self, request: HttpResponse, *, with_api_key: str | None = None): + for _, _ in self.get_traces(request=request, with_api_key=with_api_key): return raise ValueError(f"No trace has been found for request {request.get_rid()}") @@ -160,10 +174,12 @@ def get_span_metrics(span: DataDogAgentSpan) -> dict[str, str]: raise ValueError(f"Unknown span format: {span.trace.format}") - def get_metrics(self): + def get_metrics(self, *, with_api_key: str | None = None): """Attempts to fetch the metrics the agent submitted to the mocked backend.""" - for data in self.get_data(path_filters=("/api/v2/series", "/api/intake/metrics/v3/series")): + for data in self.get_data( + path_filters=("/api/v2/series", "/api/intake/metrics/v3/series"), with_api_key=with_api_key + ): content = data["request"]["content"] assert isinstance(content, dict), f"content is not a dict in {data['log_filename']}" @@ -178,10 +194,10 @@ def get_metrics(self): for point in series: yield data, point - def get_sketches(self): + def get_sketches(self, *, with_api_key: str | None = None): """Attempts to fetch the sketches the agent submitted to the mocked backend.""" - all_data = self.get_data(path_filters="/api/beta/sketches") + all_data = self.get_data(path_filters="/api/beta/sketches", with_api_key=with_api_key) for data in all_data: if "sketches" in data["request"]["content"]: @@ -190,10 +206,10 @@ def get_sketches(self): for point in content: yield data, point - def get_dsm_data(self): - return self.get_data(path_filters="/api/v0.1/pipeline_stats") + def get_dsm_data(self, *, with_api_key: str | None = None): + return self.get_data(path_filters="/api/v0.1/pipeline_stats", with_api_key=with_api_key) - def get_stats(self, resource: str | list[str] = ""): + def get_stats(self, resource: str | list[str] = "", *, with_api_key: str | None = None): """Attempts to fetch the stats the agent submitted to the mocked backend. When a valid request is given, then we filter the stats to the ones sampled @@ -201,7 +217,7 @@ def get_stats(self, resource: str | list[str] = ""): """ resources = [resource] if isinstance(resource, str) else resource - for data in self.get_data(path_filters="/api/v0.2/stats"): + for data in self.get_data(path_filters="/api/v0.2/stats", with_api_key=with_api_key): client_stats_payloads = data["request"]["content"]["Stats"] for client_stats_payload in client_stats_payloads: @@ -209,3 +225,12 @@ def get_stats(self, resource: str | list[str] = ""): for client_grouped_stat in client_stats_buckets["Stats"]: if not resources or "" in resources or client_grouped_stat["Resource"] in resources: yield client_grouped_stat + + +def _get_api_key(data: dict) -> str | None: + """The API key a request was sent with""" + for name, value in data["request"]["headers"]: + if name.lower() == "dd-api-key": + return value + + return None From e8d4d839ed19b6d2c25961207bf9eeb14f46b1cb Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Mon, 26 Oct 2026 18:56:18 +0100 Subject: [PATCH 02/12] Serve the mock intake hostnames over HTTPS in the mocked backend container Agentless senders only speak HTTPS to the hostnames derived from DD_SITE. The mocked backend container gets network aliases for them, and serves them over TLS with a certificate signed by the mitmproxy CA, sharing its routes and records with the HTTP port. Co-Authored-By: Claude Opus 5.5 --- tests/test_the_test/test_mock_backend_v2.py | 53 ++++++++++ utils/_context/containers.py | 37 ++++++- utils/mocked_backend/__main__.py | 13 ++- utils/mocked_backend/_backend.py | 19 +++- utils/mocked_backend/_tls.py | 107 ++++++++++++++++++++ 5 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 utils/mocked_backend/_tls.py diff --git a/tests/test_the_test/test_mock_backend_v2.py b/tests/test_the_test/test_mock_backend_v2.py index 6c9c06c3bcb..7d88fffa24c 100644 --- a/tests/test_the_test/test_mock_backend_v2.py +++ b/tests/test_the_test/test_mock_backend_v2.py @@ -2,6 +2,8 @@ import gzip import signal +import socket +import ssl import json from collections.abc import Generator from pathlib import Path @@ -274,3 +276,54 @@ def test_backend_v2_with_api_key_ignores_the_header_name_case(tmp_path: Path): data["request"]["headers"] = [["dd-api-key", "agent-key"]] assert len(list(interface.get_data(with_api_key="agent-key"))) == 1 + + +_MITMPROXY_CA = "utils/proxy/.mitmproxy/mitmproxy-ca.pem" +_INTAKE_HOSTNAME = "event-platform-intake.mock-intake.invalid" + + +def _https_post(port: int, hostname: str, path: str, body: bytes) -> str: + """POST over HTTPS to 127.0.0.1, verifying the certificate is valid for hostname: the status line""" + context = ssl.create_default_context(cafile=_MITMPROXY_CA) + with ( + socket.create_connection(("127.0.0.1", port), timeout=5) as raw, + context.wrap_socket(raw, server_hostname=hostname) as tls, + ): + request = f"POST {path} HTTP/1.1\r\nHost: {hostname}\r\nContent-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n" + tls.sendall(request.encode() + body) + return tls.recv(4096).decode().splitlines()[0] + + +@scenarios.test_the_test +def test_https_requests_are_served_and_recorded_with_the_http_ones(tmp_path: Path): + server = MockedBackendServer( + str(tmp_path), port=0, tls_port=0, tls_ca_file=_MITMPROXY_CA, tls_hostnames=[_INTAKE_HOSTNAME] + ) + try: + assert server.tls_listener is not None + requests.post(f"{server.base_url}/api/v2/series", json={"over": "http"}, timeout=5) + status_line = _https_post(server.tls_listener.port, _INTAKE_HOSTNAME, "/api/v2/exposures", b'{"over": "https"}') + finally: + server.close() + + assert status_line == "HTTP/1.0 200 OK" + # one message count for both: no file overwrites another + assert sorted(file.name for file in tmp_path.glob("*.json")) == [ + "000__api_v2_series.json", + "001__api_v2_exposures.json", + ] + recorded = json.loads((tmp_path / "001__api_v2_exposures.json").read_text()) + assert recorded["request"]["content"] == {"over": "https"} + + +@scenarios.test_the_test +def test_https_certificate_is_only_valid_for_the_served_hostnames(tmp_path: Path): + server = MockedBackendServer( + str(tmp_path), port=0, tls_port=0, tls_ca_file=_MITMPROXY_CA, tls_hostnames=[_INTAKE_HOSTNAME] + ) + try: + assert server.tls_listener is not None + with pytest.raises(ssl.SSLCertVerificationError): + _https_post(server.tls_listener.port, "other.mock-intake.invalid", "/api/v2/exposures", b"{}") + finally: + server.close() diff --git a/utils/_context/containers.py b/utils/_context/containers.py index f8eaf3d1958..4799c64fd4e 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -53,6 +53,8 @@ # fake key of length 32 FAKE_DD_API_KEY = "0123456789abcdef0123456789abcdef" +# DD_SITE of agentless senders: the reserved .invalid domain fails closed if a request is not sent to the mocked backend +MOCK_INTAKE_SITE = "mock-intake.invalid" # its own key tells, in the mocked backend, what serverless-init sent (see interfaces.backend_v2 with_api_key) SERVERLESS_INIT_FAKE_DD_API_KEY = "fedcba9876543210fedcba9876543210" @@ -105,6 +107,7 @@ def __init__( pid_mode: str | None = None, cgroupns: str | None = None, mounts: list[docker.types.Mount] | None = None, + network_aliases: list[str] | None = None, ) -> None: assert name assert image_name @@ -143,6 +146,8 @@ def __init__( # Unlike volumes, a bind mount listed here fails container creation when its source is # missing, instead of being created as an empty directory by the docker daemon. self.mounts = mounts + # other names other containers of the network can reach it with + self.network_aliases = network_aliases or [] def _get_image_name(self, binary_file_name: str | None, default_name: str) -> str: # if the container provide binary_file_name, then a file named binaries/{binary_file_name} @@ -256,6 +261,11 @@ def start(self, network: Network) -> Container: # auto_remove=True, detach=True, network=network.name, + networking_config=( + {network.name: get_docker_client().api.create_endpoint_config(aliases=self.network_aliases)} + if self.network_aliases + else None + ), volumes=self.volumes, ports=self.ports, working_dir=self.working_dir, @@ -782,6 +792,14 @@ class MockedBackendContainer(TestedContainer): """ port = 4900 + # HTTPS for senders that only reach the intake hostnames derived from DD_SITE (e.g. an SDK sending exposures + # directly), with a certificate signed by the mitmproxy CA: these senders are given the CA bundle trusting it + tls_port = 443 + tls_hostnames = ( + f"event-platform-intake.{MOCK_INTAKE_SITE}", + f"instrumentation-telemetry-intake.{MOCK_INTAKE_SITE}", + ) + tls_ca_file = "utils/proxy/.mitmproxy/mitmproxy-ca.pem" def __init__(self) -> None: super().__init__( @@ -789,6 +807,7 @@ def __init__(self) -> None: image_name="datadog/system-tests:proxy-v1", # a valid hostname: some HTTP clients reject an underscore name="mocked-backend", + network_aliases=list(self.tls_hostnames), working_dir="/app", volumes={ # without utils/__init__.py: utils is a namespace package, and the rest of utils is not needed @@ -811,7 +830,21 @@ def configure(self, *, host_log_folder: str, replay: bool): # Same relative path in the container and on the host, so recorded log filenames are the same log_folder = f"{host_log_folder}/interfaces/backend_v2" self.volumes[f"./{log_folder}"] = {"bind": f"/app/{log_folder}", "mode": "rw"} - self.command = ["python", "-m", "utils.mocked_backend", "--port", str(self.port), "--log-folder", log_folder] + self.command = [ + "python", + "-m", + "utils.mocked_backend", + "--port", + str(self.port), + "--log-folder", + log_folder, + "--tls-port", + str(self.tls_port), + "--tls-ca-file", + self.tls_ca_file, + ] + for hostname in self.tls_hostnames: + self.command += ["--tls-hostname", hostname] @property def container_url(self) -> str: @@ -959,7 +992,7 @@ def __init__(self) -> None: image_name="datadog/serverless-init:1.10.4", environment={ "DD_API_KEY": SERVERLESS_INIT_FAKE_DD_API_KEY, - "DD_SITE": "mock-intake.invalid", + "DD_SITE": MOCK_INTAKE_SITE, "DD_SERVICE": "ffe-system-tests-serverless-init", "DD_ENV": "system-tests", "DD_APM_ENABLED": "true", diff --git a/utils/mocked_backend/__main__.py b/utils/mocked_backend/__main__.py index 83b81b95502..582d794692d 100644 --- a/utils/mocked_backend/__main__.py +++ b/utils/mocked_backend/__main__.py @@ -20,6 +20,11 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="python -m utils.mocked_backend", description=__doc__.splitlines()[0]) parser.add_argument("--port", type=int, default=4900, help="port to listen on, 0 picks a free one (default: 4900)") parser.add_argument("--log-folder", help="folder where each received request is recorded (default: not recorded)") + parser.add_argument("--tls-port", type=int, help="also serve HTTPS on this port (default: no HTTPS)") + parser.add_argument("--tls-ca-file", help="PEM file with the key and certificate of the CA signing the HTTPS one") + parser.add_argument( + "--tls-hostname", action="append", default=[], help="hostname served over HTTPS, can be repeated" + ) args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s") @@ -35,7 +40,13 @@ def stop(_signum: int, _frame: FrameType | None) -> None: signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) - server = MockedBackendServer(args.log_folder, port=args.port) + server = MockedBackendServer( + args.log_folder, + port=args.port, + tls_port=args.tls_port, + tls_ca_file=args.tls_ca_file, + tls_hostnames=args.tls_hostname, + ) # read by whoever started the process when the port is picked by the OS print(f"Mocked backend listening on port {server.port}", flush=True) # noqa: T201 try: diff --git a/utils/mocked_backend/_backend.py b/utils/mocked_backend/_backend.py index e90b8ab5c50..43e0a1c7012 100644 --- a/utils/mocked_backend/_backend.py +++ b/utils/mocked_backend/_backend.py @@ -13,12 +13,13 @@ from typing import TYPE_CHECKING from ._request_handler import MockedBackendServerCore, MockBackendV2RequestHandler, HttpMethod, RouteHandler +from ._tls import TlsListener, create_server_ssl_context from ._backend_ffe import MockedBackendServerFfeMixin from ._client import MockedBackendClient from ._utils import CONTROL_URL_PREFIX, HOST_DOCKER_INTERNAL, logger if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Mapping, Sequence class MockedBackendServer(MockedBackendServerFfeMixin, MockedBackendServerCore): @@ -42,7 +43,13 @@ def __init__( on_message: Callable[[dict], None] | None = None, *, port: int = _DEFAULT_PORT, + tls_port: int | None = None, + tls_ca_file: str | None = None, + tls_hostnames: Sequence[str] = (), ) -> None: + """With tls_port, it also serves HTTPS on this port, for tls_hostnames, with a certificate signed by the CA in + tls_ca_file: for clients that only speak HTTPS to hostnames derived from DD_SITE. + """ super().__init__(("0.0.0.0", port), MockBackendV2RequestHandler) # noqa: S104 self.log_folder = log_folder self.on_message = on_message if on_message else lambda _: None @@ -56,6 +63,14 @@ def __init__( self._thread = threading.Thread(target=self.serve_forever, name=self.thread_name, daemon=True) self._thread.start() + + self.tls_listener: TlsListener | None = None + if tls_port is not None: + assert tls_ca_file, "TLS requires a CA file" + assert tls_hostnames, "TLS requires hostnames" + ssl_context = create_server_ssl_context(tls_ca_file, tls_hostnames) + self.tls_listener = TlsListener(self, tls_port, ssl_context) + logger.debug(f"{self.thread_name} server serves {', '.join(tls_hostnames)} on {self.tls_listener.base_url}") logging_into = f"logging into {log_folder}" if log_folder else "not logging to disk" logger.debug(f"{self.thread_name} server started on {self.base_url}, {logging_into}") @@ -108,6 +123,8 @@ def _default_handler(self, request: MockBackendV2RequestHandler, _: bytes | dict def close(self) -> None: logger.debug(f"Stopping {self.thread_name} server on {self.base_url}") + if self.tls_listener: + self.tls_listener.close() self.shutdown() self.server_close() self._thread.join(timeout=5) diff --git a/utils/mocked_backend/_tls.py b/utils/mocked_backend/_tls.py new file mode 100644 index 00000000000..861bb7c4f96 --- /dev/null +++ b/utils/mocked_backend/_tls.py @@ -0,0 +1,107 @@ +"""TLS listener of the mocked backend: for clients that only speak HTTPS to hostnames derived from DD_SITE (e.g. an +SDK sending exposures directly to the intake). It serves the same routes, and records into the same folder, as the +plain HTTP server it is attached to. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +import re +import ssl +import tempfile +import threading +from pathlib import Path +from typing import TYPE_CHECKING + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID + +from ._request_handler import MockedBackendServerCore, MockBackendV2RequestHandler + +if TYPE_CHECKING: + from collections.abc import Sequence + import socket + + from ._request_handler import HttpMethod, RouteHandler + + +def create_server_ssl_context(ca_file: str, hostnames: Sequence[str]) -> ssl.SSLContext: + """A server context with a certificate for ``hostnames``, signed by the CA (key and certificate) in ``ca_file``. + Clients trusting this CA accept it. + """ + pem = Path(ca_file).read_text(encoding="utf-8") + ca_key = serialization.load_pem_private_key(_pem_block(pem, "PRIVATE KEY").encode(), password=None) + ca_cert = x509.load_pem_x509_certificate(_pem_block(pem, "CERTIFICATE").encode()) + assert isinstance(ca_key, rsa.RSAPrivateKey), "the CA key must be an RSA key" + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + now = datetime.now(UTC) + cert = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, hostnames[0])])) + .issuer_name(ca_cert.subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(days=1)) + .not_valid_after(now + timedelta(days=30)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName(name) for name in hostnames]), critical=False) + .sign(ca_key, hashes.SHA256()) + ) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # load_cert_chain only reads files + with tempfile.TemporaryDirectory() as folder: + cert_file, key_file = Path(folder) / "cert.pem", Path(folder) / "key.pem" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ) + ) + context.load_cert_chain(cert_file, key_file) + + return context + + +def _pem_block(pem: str, label: str) -> str: + """The first block of this kind (e.g. "CERTIFICATE") in a PEM file holding several ones""" + match = re.search(rf"-----BEGIN ([A-Z ]*{label})-----.+?-----END \1-----", pem, re.DOTALL) + assert match, f"no {label} in the PEM file" + return match.group(0) + + +class TlsListener(MockedBackendServerCore): + """Accepts HTTPS requests, and hands them to ``backend``: same routes, same records, same message count""" + + daemon_threads = True + + def __init__(self, backend: MockedBackendServerCore, port: int, ssl_context: ssl.SSLContext) -> None: + super().__init__(("0.0.0.0", port), MockBackendV2RequestHandler) # noqa: S104 + self.backend = backend + self.on_message = backend.on_message + self.port = self.server_port + self._ssl_context = ssl_context + self._thread = threading.Thread(target=self.serve_forever, name="mock-backend-v2-tls", daemon=True) + self._thread.start() + + def get_request(self) -> tuple[socket.socket, tuple[str, int]]: + sock, address = super().get_request() + # the handshake runs in the request thread, on first read: a slow client never blocks accepting others + return self._ssl_context.wrap_socket(sock, server_side=True, do_handshake_on_connect=False), address + + def get_handler(self, method: HttpMethod, path: str) -> tuple[RouteHandler, str | None, str | None]: + return self.backend.get_handler(method, path) + + def add_handler(self, method: HttpMethod, path: str, handler: RouteHandler) -> None: + self.backend.add_handler(method, path, handler) + + @property + def base_url(self) -> str: + return f"https://127.0.0.1:{self.port}" + + def close(self) -> None: + self.shutdown() + self.server_close() + self._thread.join(timeout=5) From d5bdde564d0f8bf82d5cff7c1871d24c6170d04e Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Mon, 26 Oct 2026 19:08:07 +0100 Subject: [PATCH 03/12] Rename MOCK_INTAKE_SITE to MOCK_INVALID_INTAKE_SITE Co-Authored-By: Claude Opus 5.5 --- utils/_context/containers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 4799c64fd4e..00cdc44b3b9 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -54,7 +54,7 @@ # fake key of length 32 FAKE_DD_API_KEY = "0123456789abcdef0123456789abcdef" # DD_SITE of agentless senders: the reserved .invalid domain fails closed if a request is not sent to the mocked backend -MOCK_INTAKE_SITE = "mock-intake.invalid" +MOCK_INVALID_INTAKE_SITE = "mock-intake.invalid" # its own key tells, in the mocked backend, what serverless-init sent (see interfaces.backend_v2 with_api_key) SERVERLESS_INIT_FAKE_DD_API_KEY = "fedcba9876543210fedcba9876543210" @@ -796,8 +796,8 @@ class MockedBackendContainer(TestedContainer): # directly), with a certificate signed by the mitmproxy CA: these senders are given the CA bundle trusting it tls_port = 443 tls_hostnames = ( - f"event-platform-intake.{MOCK_INTAKE_SITE}", - f"instrumentation-telemetry-intake.{MOCK_INTAKE_SITE}", + f"event-platform-intake.{MOCK_INVALID_INTAKE_SITE}", + f"instrumentation-telemetry-intake.{MOCK_INVALID_INTAKE_SITE}", ) tls_ca_file = "utils/proxy/.mitmproxy/mitmproxy-ca.pem" @@ -992,7 +992,7 @@ def __init__(self) -> None: image_name="datadog/serverless-init:1.10.4", environment={ "DD_API_KEY": SERVERLESS_INIT_FAKE_DD_API_KEY, - "DD_SITE": MOCK_INTAKE_SITE, + "DD_SITE": MOCK_INVALID_INTAKE_SITE, "DD_SERVICE": "ffe-system-tests-serverless-init", "DD_ENV": "system-tests", "DD_APM_ENABLED": "true", From b5d08f89c2b273fae7985d0b8a0f197158bf196f Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Mon, 26 Oct 2026 19:13:47 +0100 Subject: [PATCH 04/12] Use a committed certificate for the mocked backend HTTPS listener Instead of generating one at startup. A test checks it covers the container hostnames, and is not about to expire. Co-Authored-By: Claude Opus 5.5 --- tests/test_the_test/test_mock_backend_v2.py | 27 +++++-- utils/_context/containers.py | 9 ++- utils/mocked_backend/__main__.py | 8 +-- utils/mocked_backend/_backend.py | 19 +++-- utils/mocked_backend/_tls.py | 79 ++++----------------- utils/mocked_backend/tls/mock-intake.pem | 48 +++++++++++++ 6 files changed, 97 insertions(+), 93 deletions(-) create mode 100644 utils/mocked_backend/tls/mock-intake.pem diff --git a/tests/test_the_test/test_mock_backend_v2.py b/tests/test_the_test/test_mock_backend_v2.py index 7d88fffa24c..2b617375a58 100644 --- a/tests/test_the_test/test_mock_backend_v2.py +++ b/tests/test_the_test/test_mock_backend_v2.py @@ -1,5 +1,6 @@ """Unit coverage for MockedBackendServer (utils.mocked_backend).""" +from datetime import UTC, datetime, timedelta import gzip import signal import socket @@ -13,9 +14,11 @@ from utils import pytest import requests import zstandard +from cryptography import x509 from utils import scenarios from utils.interfaces._backend_v2 import _BackendV2InterfaceValidator +from utils._context.containers import MockedBackendContainer from utils.mocked_backend import MockedBackendClient, MockedBackendServer from utils.mocked_backend.ffe import UFC_FIXTURE_RELATIVE_PATH @@ -278,7 +281,8 @@ def test_backend_v2_with_api_key_ignores_the_header_name_case(tmp_path: Path): assert len(list(interface.get_data(with_api_key="agent-key"))) == 1 -_MITMPROXY_CA = "utils/proxy/.mitmproxy/mitmproxy-ca.pem" +_MITMPROXY_CA = "utils/proxy/.mitmproxy/mitmproxy-ca-cert.pem" +_TLS_CERT_FILE = MockedBackendContainer.tls_cert_file _INTAKE_HOSTNAME = "event-platform-intake.mock-intake.invalid" @@ -296,9 +300,7 @@ def _https_post(port: int, hostname: str, path: str, body: bytes) -> str: @scenarios.test_the_test def test_https_requests_are_served_and_recorded_with_the_http_ones(tmp_path: Path): - server = MockedBackendServer( - str(tmp_path), port=0, tls_port=0, tls_ca_file=_MITMPROXY_CA, tls_hostnames=[_INTAKE_HOSTNAME] - ) + server = MockedBackendServer(str(tmp_path), port=0, tls_port=0, tls_cert_file=_TLS_CERT_FILE) try: assert server.tls_listener is not None requests.post(f"{server.base_url}/api/v2/series", json={"over": "http"}, timeout=5) @@ -318,12 +320,23 @@ def test_https_requests_are_served_and_recorded_with_the_http_ones(tmp_path: Pat @scenarios.test_the_test def test_https_certificate_is_only_valid_for_the_served_hostnames(tmp_path: Path): - server = MockedBackendServer( - str(tmp_path), port=0, tls_port=0, tls_ca_file=_MITMPROXY_CA, tls_hostnames=[_INTAKE_HOSTNAME] - ) + server = MockedBackendServer(str(tmp_path), port=0, tls_port=0, tls_cert_file=_TLS_CERT_FILE) try: assert server.tls_listener is not None with pytest.raises(ssl.SSLCertVerificationError): _https_post(server.tls_listener.port, "other.mock-intake.invalid", "/api/v2/exposures", b"{}") finally: server.close() + + +@scenarios.test_the_test +def test_https_certificate_covers_the_container_hostnames_for_a_while(): + """The committed certificate must be regenerated when a hostname is added, or before it expires""" + cert = x509.load_pem_x509_certificate(Path(_TLS_CERT_FILE).read_bytes()) + hostnames = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value.get_values_for_type( + x509.DNSName + ) + + assert sorted(hostnames) == sorted(MockedBackendContainer.tls_hostnames) + # naive, in UTC + assert cert.not_valid_after.replace(tzinfo=UTC) - datetime.now(UTC) > timedelta(days=180) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 00cdc44b3b9..ad6dd138799 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -799,7 +799,8 @@ class MockedBackendContainer(TestedContainer): f"event-platform-intake.{MOCK_INVALID_INTAKE_SITE}", f"instrumentation-telemetry-intake.{MOCK_INVALID_INTAKE_SITE}", ) - tls_ca_file = "utils/proxy/.mitmproxy/mitmproxy-ca.pem" + # covers tls_hostnames, see utils/mocked_backend/_tls.py + tls_cert_file = "utils/mocked_backend/tls/mock-intake.pem" def __init__(self) -> None: super().__init__( @@ -840,11 +841,9 @@ def configure(self, *, host_log_folder: str, replay: bool): log_folder, "--tls-port", str(self.tls_port), - "--tls-ca-file", - self.tls_ca_file, + "--tls-cert-file", + self.tls_cert_file, ] - for hostname in self.tls_hostnames: - self.command += ["--tls-hostname", hostname] @property def container_url(self) -> str: diff --git a/utils/mocked_backend/__main__.py b/utils/mocked_backend/__main__.py index 582d794692d..31d1a01457f 100644 --- a/utils/mocked_backend/__main__.py +++ b/utils/mocked_backend/__main__.py @@ -21,10 +21,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--port", type=int, default=4900, help="port to listen on, 0 picks a free one (default: 4900)") parser.add_argument("--log-folder", help="folder where each received request is recorded (default: not recorded)") parser.add_argument("--tls-port", type=int, help="also serve HTTPS on this port (default: no HTTPS)") - parser.add_argument("--tls-ca-file", help="PEM file with the key and certificate of the CA signing the HTTPS one") - parser.add_argument( - "--tls-hostname", action="append", default=[], help="hostname served over HTTPS, can be repeated" - ) + parser.add_argument("--tls-cert-file", help="PEM file with the HTTPS certificate and its key") args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s") @@ -44,8 +41,7 @@ def stop(_signum: int, _frame: FrameType | None) -> None: args.log_folder, port=args.port, tls_port=args.tls_port, - tls_ca_file=args.tls_ca_file, - tls_hostnames=args.tls_hostname, + tls_cert_file=args.tls_cert_file, ) # read by whoever started the process when the port is picked by the OS print(f"Mocked backend listening on port {server.port}", flush=True) # noqa: T201 diff --git a/utils/mocked_backend/_backend.py b/utils/mocked_backend/_backend.py index 43e0a1c7012..9d1df15e249 100644 --- a/utils/mocked_backend/_backend.py +++ b/utils/mocked_backend/_backend.py @@ -13,13 +13,13 @@ from typing import TYPE_CHECKING from ._request_handler import MockedBackendServerCore, MockBackendV2RequestHandler, HttpMethod, RouteHandler -from ._tls import TlsListener, create_server_ssl_context +from ._tls import TlsListener from ._backend_ffe import MockedBackendServerFfeMixin from ._client import MockedBackendClient from ._utils import CONTROL_URL_PREFIX, HOST_DOCKER_INTERNAL, logger if TYPE_CHECKING: - from collections.abc import Callable, Mapping, Sequence + from collections.abc import Callable, Mapping class MockedBackendServer(MockedBackendServerFfeMixin, MockedBackendServerCore): @@ -44,11 +44,10 @@ def __init__( *, port: int = _DEFAULT_PORT, tls_port: int | None = None, - tls_ca_file: str | None = None, - tls_hostnames: Sequence[str] = (), + tls_cert_file: str | None = None, ) -> None: - """With tls_port, it also serves HTTPS on this port, for tls_hostnames, with a certificate signed by the CA in - tls_ca_file: for clients that only speak HTTPS to hostnames derived from DD_SITE. + """With tls_port, it also serves HTTPS on this port, with the certificate and key in tls_cert_file: for clients + that only speak HTTPS to hostnames derived from DD_SITE. """ super().__init__(("0.0.0.0", port), MockBackendV2RequestHandler) # noqa: S104 self.log_folder = log_folder @@ -66,11 +65,9 @@ def __init__( self.tls_listener: TlsListener | None = None if tls_port is not None: - assert tls_ca_file, "TLS requires a CA file" - assert tls_hostnames, "TLS requires hostnames" - ssl_context = create_server_ssl_context(tls_ca_file, tls_hostnames) - self.tls_listener = TlsListener(self, tls_port, ssl_context) - logger.debug(f"{self.thread_name} server serves {', '.join(tls_hostnames)} on {self.tls_listener.base_url}") + assert tls_cert_file, "TLS requires a certificate file" + self.tls_listener = TlsListener(self, tls_port, tls_cert_file) + logger.debug(f"{self.thread_name} server also serves HTTPS on {self.tls_listener.base_url}") logging_into = f"logging into {log_folder}" if log_folder else "not logging to disk" logger.debug(f"{self.thread_name} server started on {self.base_url}, {logging_into}") diff --git a/utils/mocked_backend/_tls.py b/utils/mocked_backend/_tls.py index 861bb7c4f96..02b3bb989f6 100644 --- a/utils/mocked_backend/_tls.py +++ b/utils/mocked_backend/_tls.py @@ -1,96 +1,47 @@ """TLS listener of the mocked backend: for clients that only speak HTTPS to hostnames derived from DD_SITE (e.g. an SDK sending exposures directly to the intake). It serves the same routes, and records into the same folder, as the plain HTTP server it is attached to. + +tls/mock-intake.pem holds a certificate and its key for these hostnames, signed by the mitmproxy CA, which these +clients trust. To add a hostname, regenerate it: + + openssl req -new -newkey rsa:2048 -nodes -keyout key.pem -out req.csr \ + -subj "/CN=event-platform-intake.mock-intake.invalid" + echo "subjectAltName=DNS:event-platform-intake.mock-intake.invalid,DNS:..." > ext + openssl x509 -req -in req.csr -CA utils/proxy/.mitmproxy/mitmproxy-ca.pem \ + -CAkey utils/proxy/.mitmproxy/mitmproxy-ca.pem -set_serial 0x$(openssl rand -hex 8) -days 3650 -sha256 \ + -extfile ext -out cert.pem + cat cert.pem key.pem > utils/mocked_backend/tls/mock-intake.pem """ from __future__ import annotations -from datetime import UTC, datetime, timedelta -import re import ssl -import tempfile import threading -from pathlib import Path from typing import TYPE_CHECKING -from cryptography import x509 -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import NameOID - from ._request_handler import MockedBackendServerCore, MockBackendV2RequestHandler if TYPE_CHECKING: - from collections.abc import Sequence - import socket - from ._request_handler import HttpMethod, RouteHandler -def create_server_ssl_context(ca_file: str, hostnames: Sequence[str]) -> ssl.SSLContext: - """A server context with a certificate for ``hostnames``, signed by the CA (key and certificate) in ``ca_file``. - Clients trusting this CA accept it. - """ - pem = Path(ca_file).read_text(encoding="utf-8") - ca_key = serialization.load_pem_private_key(_pem_block(pem, "PRIVATE KEY").encode(), password=None) - ca_cert = x509.load_pem_x509_certificate(_pem_block(pem, "CERTIFICATE").encode()) - assert isinstance(ca_key, rsa.RSAPrivateKey), "the CA key must be an RSA key" - - key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - now = datetime.now(UTC) - cert = ( - x509.CertificateBuilder() - .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, hostnames[0])])) - .issuer_name(ca_cert.subject) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(now - timedelta(days=1)) - .not_valid_after(now + timedelta(days=30)) - .add_extension(x509.SubjectAlternativeName([x509.DNSName(name) for name in hostnames]), critical=False) - .sign(ca_key, hashes.SHA256()) - ) - - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - # load_cert_chain only reads files - with tempfile.TemporaryDirectory() as folder: - cert_file, key_file = Path(folder) / "cert.pem", Path(folder) / "key.pem" - cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) - key_file.write_bytes( - key.private_bytes( - serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() - ) - ) - context.load_cert_chain(cert_file, key_file) - - return context - - -def _pem_block(pem: str, label: str) -> str: - """The first block of this kind (e.g. "CERTIFICATE") in a PEM file holding several ones""" - match = re.search(rf"-----BEGIN ([A-Z ]*{label})-----.+?-----END \1-----", pem, re.DOTALL) - assert match, f"no {label} in the PEM file" - return match.group(0) - - class TlsListener(MockedBackendServerCore): """Accepts HTTPS requests, and hands them to ``backend``: same routes, same records, same message count""" daemon_threads = True - def __init__(self, backend: MockedBackendServerCore, port: int, ssl_context: ssl.SSLContext) -> None: + def __init__(self, backend: MockedBackendServerCore, port: int, cert_file: str) -> None: super().__init__(("0.0.0.0", port), MockBackendV2RequestHandler) # noqa: S104 + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert_file) + self.socket = context.wrap_socket(self.socket, server_side=True) self.backend = backend self.on_message = backend.on_message self.port = self.server_port - self._ssl_context = ssl_context self._thread = threading.Thread(target=self.serve_forever, name="mock-backend-v2-tls", daemon=True) self._thread.start() - def get_request(self) -> tuple[socket.socket, tuple[str, int]]: - sock, address = super().get_request() - # the handshake runs in the request thread, on first read: a slow client never blocks accepting others - return self._ssl_context.wrap_socket(sock, server_side=True, do_handshake_on_connect=False), address - def get_handler(self, method: HttpMethod, path: str) -> tuple[RouteHandler, str | None, str | None]: return self.backend.get_handler(method, path) diff --git a/utils/mocked_backend/tls/mock-intake.pem b/utils/mocked_backend/tls/mock-intake.pem new file mode 100644 index 00000000000..a716cb53e28 --- /dev/null +++ b/utils/mocked_backend/tls/mock-intake.pem @@ -0,0 +1,48 @@ +-----BEGIN CERTIFICATE----- +MIIDTDCCAjSgAwIBAgIIMd91Y0jP1LAwDQYJKoZIhvcNAQELBQAwKDESMBAGA1UE +AwwJbWl0bXByb3h5MRIwEAYDVQQKDAltaXRtcHJveHkwHhcNMjYxMDI2MTgwOTUx +WhcNMzYxMDIzMTgwOTUxWjA0MTIwMAYDVQQDDClldmVudC1wbGF0Zm9ybS1pbnRh +a2UubW9jay1pbnRha2UuaW52YWxpZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBANtqu1rkdGU8uUspjlzGBmMmxCpFuSiOFjmXncCcKGRSw3Pcj7XMg+31 +TBBrNKYadBoIimBXt67aD5obTbfzxoGyy9GY39IHFQaUFz3mSlXqyytQlknXOSJn +dQQXj1WTUbqi2HTJ//7IAVZBXeLfC/+dgBPE+Hzjesl46/HNw6YtqmvUJoqvC6xQ +ZLh42P0wXBLC0E+R8HLK6XcbkYAjgckliwR6YgV/2uhDkSGpTp7QDGHwtT/ySO66 +SFnpbsOFekJmNmX4vTr4LDjlxe3NRWrMew3i/ok1ajJX16RcITHhUFYlVrDETant +oqPIJTajgXV3oFhdkp3fl1sxuPtD9p8CAwEAAaNuMGwwagYDVR0RBGMwYYIpZXZl +bnQtcGxhdGZvcm0taW50YWtlLm1vY2staW50YWtlLmludmFsaWSCNGluc3RydW1l +bnRhdGlvbi10ZWxlbWV0cnktaW50YWtlLm1vY2staW50YWtlLmludmFsaWQwDQYJ +KoZIhvcNAQELBQADggEBADeJRRWIsQhxyE9SIANhHcUs9+CjrX+uUBjDFZmML0Ra +rQVBYAFcPljs5piKYpptUocy5yCLls2AF+7/tIz68uNZtl1ME2etupUJV1JQfugH +Q/bPoK8D+2AzCIyzVUwHX0KSNzm8ihy8WCZuovP4bd51reRZgy1bA7LwY8TbR03D ++TjgTTCFPbvNWoAUSpOR1FzY9ALeKIOmzlHdsyike/mSLUZCJG3C7sgoNV9bgWxs +tov1ZGH0bxpyLlvPrJjpqWUpINhHpX1WV7BYGkoQoj+eGbXzWXzC3eNLWO6CfxsT +/k20yB6tkFQw1JTN/b2ipL0DGry6zIH6zgL4Yq6wyQE= +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDbarta5HRlPLlL +KY5cxgZjJsQqRbkojhY5l53AnChkUsNz3I+1zIPt9UwQazSmGnQaCIpgV7eu2g+a +G02388aBssvRmN/SBxUGlBc95kpV6ssrUJZJ1zkiZ3UEF49Vk1G6oth0yf/+yAFW +QV3i3wv/nYATxPh843rJeOvxzcOmLapr1CaKrwusUGS4eNj9MFwSwtBPkfByyul3 +G5GAI4HJJYsEemIFf9roQ5EhqU6e0Axh8LU/8kjuukhZ6W7DhXpCZjZl+L06+Cw4 +5cXtzUVqzHsN4v6JNWoyV9ekXCEx4VBWJVawxE2p7aKjyCU2o4F1d6BYXZKd35db +Mbj7Q/afAgMBAAECggEAAMcrLMLzpgOyLOonvPfayQv55rW2TensKpi7jCdk5+Q6 +0lxz8+2ocNt1OHMzWMNK567LlkL4QkhJhUEo+mM1RkIuWcwUYqzrIe0C6ienINvC +B6F1+BBwGALTWTibeayVZaOR+WWOYP3BmCWiJ2rsFWIMyroecuO+f1WOzzy3yG5D +RSPCh4rlazriGUpI918mSqsrtXqtxT7xlICccQsti+zjNaos0khhfCJENOuoEKSJ +4vMXE203aKI3UVnhV1q3zbfOwW+wQv+Eg/dDawY0e/SCFAzHdSMeidXkTq6d/cuz +cn7gbdqGSVomlMfOKq9y92ZTYUpWmUFzZ0DVWl5BOQKBgQD37Hwm/p1m5vdEbWM+ +ERSs/xHxiwPX8qMS2+TOB/cxoWu++hqNaXEXXyY6mIrfe8fJaaKbmebAMkJihl2b +gQVcRH7fzz1b9fcSyg2nRYudeBR7g4A70U1FmqZAVxJDWCNYjkPOPMo+QZ6qyKJp +tAIXL25AFF4Mzw0l/vc15AYehQKBgQDikITsJOGEfoKM7/wPixLS3y0XpC4hF8aq +zwvv4K3PJhHk4kqNtrrz5Z99MPKma9+tZ+arVRzi0YwL/LJhRAfdFYrizJC13yyP +ueyDnbF2xHVBNu59mMRftqK7etfYTiv6mm69oRTW09OBVXXUubfepRoubj88jw/d +a3d/mgxD0wKBgD8Q0hgdRq7pa9blOIyOTcMf81GNOkIRf7gJZa+fSRs/46el+WEl +hFYcpx3ateDZnN24WFvaj18uqcJkSaWH8XQiHGAUWJhu/hWyfjyhrT02UYeF0PXm +bRCaoVXWkFe/MOaS8d6hihTCHnl1JhNbbfi14Qa955q8hKy4qev0gDX9AoGARE7M +NKI9+ANoevOT7F/C0h8Fal5Oph2UAUWhVLZJ0O2qcXakgat5HDW76zPJxNE7Q/3S +dw+XQO9qqOaMJXUsZcSr9gs1ChB6T+7/VwNBUQyUSZA1QVrAwha/Y4T69LVE3y6W +esfJD6gX2aOZfgRsYk0m4p7MpAaCAwD2q1sXnY8CgYBtBY02xh0/pbygsfdwB0vq +wUaLUqjRCCFHLcN6tQ1IZwx2/kcIaPhEqW7XmUp8V7nztBHHKkaFrfmW/DWFpM++ +6m9Uim7o4lHseItVSnhqFX8uE89lrvotEGxctKfL+2oZSIlSCO/sPoADP4Xu2udl +uRC6vumSGBOyLn2H6+QGbg== +-----END PRIVATE KEY----- From 40fbaf119bf083f5299b08fa811158e701c0f62e Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Mon, 26 Oct 2026 19:31:20 +0100 Subject: [PATCH 05/12] The mocked backend serves either HTTP or HTTPS, not both No scenario needs both: the container gets a tls option (off by default), and scenarios will turn it on when they move their HTTPS senders to the mocked backend. The TLS listener is gone; the certificate also covers the container name and localhost. Co-Authored-By: Claude Opus 5.5 --- tests/test_the_test/test_mock_backend_v2.py | 56 +++++++++++++------- utils/_context/containers.py | 37 +++++++------ utils/mocked_backend/__main__.py | 4 +- utils/mocked_backend/_backend.py | 25 ++++----- utils/mocked_backend/_client.py | 13 +++-- utils/mocked_backend/_tls.py | 58 --------------------- utils/mocked_backend/tls/README.md | 13 +++++ utils/mocked_backend/tls/mock-intake.pem | 48 ----------------- utils/mocked_backend/tls/mocked-backend.pem | 48 +++++++++++++++++ 9 files changed, 141 insertions(+), 161 deletions(-) delete mode 100644 utils/mocked_backend/_tls.py create mode 100644 utils/mocked_backend/tls/README.md delete mode 100644 utils/mocked_backend/tls/mock-intake.pem create mode 100644 utils/mocked_backend/tls/mocked-backend.pem diff --git a/tests/test_the_test/test_mock_backend_v2.py b/tests/test_the_test/test_mock_backend_v2.py index 2b617375a58..202942cf23e 100644 --- a/tests/test_the_test/test_mock_backend_v2.py +++ b/tests/test_the_test/test_mock_backend_v2.py @@ -299,32 +299,37 @@ def _https_post(port: int, hostname: str, path: str, body: bytes) -> str: @scenarios.test_the_test -def test_https_requests_are_served_and_recorded_with_the_http_ones(tmp_path: Path): - server = MockedBackendServer(str(tmp_path), port=0, tls_port=0, tls_cert_file=_TLS_CERT_FILE) +def test_https_mode_serves_and_records_https_requests(tmp_path: Path): + server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE) try: - assert server.tls_listener is not None - requests.post(f"{server.base_url}/api/v2/series", json={"over": "http"}, timeout=5) - status_line = _https_post(server.tls_listener.port, _INTAKE_HOSTNAME, "/api/v2/exposures", b'{"over": "https"}') + status_line = _https_post(server.port, _INTAKE_HOSTNAME, "/api/v2/exposures", b'{"over": "https"}') + # the control endpoints, as used from the host + health = requests.get(f"{server.base_url}/system-tests/health", timeout=5, verify=_MITMPROXY_CA) finally: server.close() assert status_line == "HTTP/1.0 200 OK" - # one message count for both: no file overwrites another - assert sorted(file.name for file in tmp_path.glob("*.json")) == [ - "000__api_v2_series.json", - "001__api_v2_exposures.json", - ] - recorded = json.loads((tmp_path / "001__api_v2_exposures.json").read_text()) + assert health.json() == {"status": "ok"} + recorded = json.loads((tmp_path / "000__api_v2_exposures.json").read_text()) assert recorded["request"]["content"] == {"over": "https"} +@scenarios.test_the_test +def test_https_mode_does_not_serve_http(tmp_path: Path): + server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE) + try: + with pytest.raises(requests.exceptions.ConnectionError): + requests.post(f"http://127.0.0.1:{server.port}/api/v2/series", json={}, timeout=5) + finally: + server.close() + + @scenarios.test_the_test def test_https_certificate_is_only_valid_for_the_served_hostnames(tmp_path: Path): - server = MockedBackendServer(str(tmp_path), port=0, tls_port=0, tls_cert_file=_TLS_CERT_FILE) + server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE) try: - assert server.tls_listener is not None with pytest.raises(ssl.SSLCertVerificationError): - _https_post(server.tls_listener.port, "other.mock-intake.invalid", "/api/v2/exposures", b"{}") + _https_post(server.port, "other.mock-intake.invalid", "/api/v2/exposures", b"{}") finally: server.close() @@ -333,10 +338,25 @@ def test_https_certificate_is_only_valid_for_the_served_hostnames(tmp_path: Path def test_https_certificate_covers_the_container_hostnames_for_a_while(): """The committed certificate must be regenerated when a hostname is added, or before it expires""" cert = x509.load_pem_x509_certificate(Path(_TLS_CERT_FILE).read_bytes()) - hostnames = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value.get_values_for_type( - x509.DNSName - ) + names = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + container = MockedBackendContainer(tls=True) - assert sorted(hostnames) == sorted(MockedBackendContainer.tls_hostnames) + # reached by its name, its intake hostnames, and by localhost from the healthcheck and the host + assert sorted(names.get_values_for_type(x509.DNSName)) == sorted( + [container.name, *container.network_aliases, "localhost"] + ) + assert [str(ip) for ip in names.get_values_for_type(x509.IPAddress)] == ["127.0.0.1"] # naive, in UTC assert cert.not_valid_after.replace(tzinfo=UTC) - datetime.now(UTC) > timedelta(days=180) + + +@scenarios.test_the_test +def test_mocked_backend_container_serves_either_http_or_https(): + http = MockedBackendContainer() + assert http.container_url == "http://mocked-backend:4900" + assert http.network_aliases == [] + + https = MockedBackendContainer(tls=True) + # senders build the intake URLs from DD_SITE: the default HTTPS port, and the intake hostnames + assert https.container_url == "https://mocked-backend:443" + assert https.network_aliases == list(MockedBackendContainer.tls_hostnames) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index ad6dd138799..328efcc8eb0 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -791,24 +791,29 @@ class MockedBackendContainer(TestedContainer): What it receives is recorded in the logs folder, and read by interfaces.backend_v2. """ - port = 4900 - # HTTPS for senders that only reach the intake hostnames derived from DD_SITE (e.g. an SDK sending exposures - # directly), with a certificate signed by the mitmproxy CA: these senders are given the CA bundle trusting it - tls_port = 443 + # With tls, it serves HTTPS instead of HTTP: for senders that only speak HTTPS to the intake hostnames derived + # from DD_SITE (e.g. an SDK sending exposures directly). Its certificate is signed by the mitmproxy CA: these + # senders must trust it. See utils/mocked_backend/tls/README.md tls_hostnames = ( f"event-platform-intake.{MOCK_INVALID_INTAKE_SITE}", f"instrumentation-telemetry-intake.{MOCK_INVALID_INTAKE_SITE}", ) - # covers tls_hostnames, see utils/mocked_backend/_tls.py - tls_cert_file = "utils/mocked_backend/tls/mock-intake.pem" - - def __init__(self) -> None: + tls_cert_file = "utils/mocked_backend/tls/mocked-backend.pem" + tls_ca_file = "utils/proxy/.mitmproxy/mitmproxy-ca-cert.pem" + + def __init__(self, *, tls: bool = False) -> None: + self.tls = tls + self.scheme = "https" if tls else "http" + # the default HTTPS port: senders build the intake URLs from DD_SITE, without port + self.port = 443 if tls else 4900 + # in the healthcheck, the certificate is checked against localhost + health_context = f"ssl.create_default_context(cafile='/app/{self.tls_ca_file}')" if tls else "None" super().__init__( # it has every dependency of utils/mocked_backend image_name="datadog/system-tests:proxy-v1", # a valid hostname: some HTTP clients reject an underscore name="mocked-backend", - network_aliases=list(self.tls_hostnames), + network_aliases=list(self.tls_hostnames) if tls else None, working_dir="/app", volumes={ # without utils/__init__.py: utils is a namespace package, and the rest of utils is not needed @@ -820,7 +825,7 @@ def __init__(self) -> None: # for the control endpoints, used from the host. Docker picks a free host port ports={f"{self.port}/tcp": ("127.0.0.1", None)}, healthcheck={ - "test": f"python -c \"import urllib.request; print(urllib.request.urlopen('http://localhost:{self.port}/system-tests/health', timeout=2).read().decode())\"", # noqa: E501 + "test": f"python -c \"import ssl, urllib.request; print(urllib.request.urlopen('{self.scheme}://localhost:{self.port}/system-tests/health', timeout=2, context={health_context}).read().decode())\"", # noqa: E501 "retries": 30, }, ) @@ -839,22 +844,22 @@ def configure(self, *, host_log_folder: str, replay: bool): str(self.port), "--log-folder", log_folder, - "--tls-port", - str(self.tls_port), - "--tls-cert-file", - self.tls_cert_file, ] + if self.tls: + self.command += ["--tls-cert-file", self.tls_cert_file] @property def container_url(self) -> str: """The URL other containers use to reach the mocked backend""" - return f"http://{self.name}:{self.port}" + return f"{self.scheme}://{self.name}:{self.port}" @property def client(self) -> MockedBackendClient: """Client of the control endpoints, from the host""" host_port = self.runtime_container.attrs["NetworkSettings"]["Ports"][f"{self.port}/tcp"][0]["HostPort"] - return MockedBackendClient(f"http://127.0.0.1:{host_port}") + return MockedBackendClient( + f"{self.scheme}://127.0.0.1:{host_port}", verify=self.tls_ca_file if self.tls else True + ) class LambdaProxyContainer(TestedContainer): diff --git a/utils/mocked_backend/__main__.py b/utils/mocked_backend/__main__.py index 31d1a01457f..973ada3c908 100644 --- a/utils/mocked_backend/__main__.py +++ b/utils/mocked_backend/__main__.py @@ -20,8 +20,7 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="python -m utils.mocked_backend", description=__doc__.splitlines()[0]) parser.add_argument("--port", type=int, default=4900, help="port to listen on, 0 picks a free one (default: 4900)") parser.add_argument("--log-folder", help="folder where each received request is recorded (default: not recorded)") - parser.add_argument("--tls-port", type=int, help="also serve HTTPS on this port (default: no HTTPS)") - parser.add_argument("--tls-cert-file", help="PEM file with the HTTPS certificate and its key") + parser.add_argument("--tls-cert-file", help="serve HTTPS with this certificate and its key (default: HTTP)") args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s") @@ -40,7 +39,6 @@ def stop(_signum: int, _frame: FrameType | None) -> None: server = MockedBackendServer( args.log_folder, port=args.port, - tls_port=args.tls_port, tls_cert_file=args.tls_cert_file, ) # read by whoever started the process when the port is picked by the OS diff --git a/utils/mocked_backend/_backend.py b/utils/mocked_backend/_backend.py index 9d1df15e249..84bf228fe52 100644 --- a/utils/mocked_backend/_backend.py +++ b/utils/mocked_backend/_backend.py @@ -9,11 +9,11 @@ from __future__ import annotations from http import HTTPStatus +import ssl import threading from typing import TYPE_CHECKING from ._request_handler import MockedBackendServerCore, MockBackendV2RequestHandler, HttpMethod, RouteHandler -from ._tls import TlsListener from ._backend_ffe import MockedBackendServerFfeMixin from ._client import MockedBackendClient from ._utils import CONTROL_URL_PREFIX, HOST_DOCKER_INTERNAL, logger @@ -43,13 +43,17 @@ def __init__( on_message: Callable[[dict], None] | None = None, *, port: int = _DEFAULT_PORT, - tls_port: int | None = None, tls_cert_file: str | None = None, ) -> None: - """With tls_port, it also serves HTTPS on this port, with the certificate and key in tls_cert_file: for clients - that only speak HTTPS to hostnames derived from DD_SITE. + """With tls_cert_file (a certificate and its key), it serves HTTPS instead of HTTP: for senders that only speak + HTTPS to the intake hostnames derived from DD_SITE. """ super().__init__(("0.0.0.0", port), MockBackendV2RequestHandler) # noqa: S104 + self.scheme = "https" if tls_cert_file else "http" + if tls_cert_file: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(tls_cert_file) + self.socket = context.wrap_socket(self.socket, server_side=True) self.log_folder = log_folder self.on_message = on_message if on_message else lambda _: None self.message_count = 0 @@ -62,23 +66,18 @@ def __init__( self._thread = threading.Thread(target=self.serve_forever, name=self.thread_name, daemon=True) self._thread.start() - - self.tls_listener: TlsListener | None = None - if tls_port is not None: - assert tls_cert_file, "TLS requires a certificate file" - self.tls_listener = TlsListener(self, tls_port, tls_cert_file) - logger.debug(f"{self.thread_name} server also serves HTTPS on {self.tls_listener.base_url}") logging_into = f"logging into {log_folder}" if log_folder else "not logging to disk" logger.debug(f"{self.thread_name} server started on {self.base_url}, {logging_into}") @property def base_url(self) -> str: - return f"http://127.0.0.1:{self.port}" + return f"{self.scheme}://127.0.0.1:{self.port}" @property def client(self) -> MockedBackendClient: """Client of the control endpoints of this server""" - return MockedBackendClient(self.base_url) + # over HTTPS, it only talks to itself on the loopback: nothing to verify + return MockedBackendClient(self.base_url, verify=self.scheme == "http") @property def container_url(self) -> str: @@ -120,8 +119,6 @@ def _default_handler(self, request: MockBackendV2RequestHandler, _: bytes | dict def close(self) -> None: logger.debug(f"Stopping {self.thread_name} server on {self.base_url}") - if self.tls_listener: - self.tls_listener.close() self.shutdown() self.server_close() self._thread.join(timeout=5) diff --git a/utils/mocked_backend/_client.py b/utils/mocked_backend/_client.py index 48b7db268fe..72ec4469776 100644 --- a/utils/mocked_backend/_client.py +++ b/utils/mocked_backend/_client.py @@ -15,11 +15,13 @@ class MockedBackendClient: - def __init__(self, base_url: str) -> None: + def __init__(self, base_url: str, *, verify: bool | str = True) -> None: + """verify: as in requests, e.g. the CA file trusting the certificate of an HTTPS server""" self.base_url = base_url.rstrip("/") + self.verify = verify def ffe_reset(self) -> None: - response = requests.post(f"{self.base_url}{_FFE_CONTROL_PATH}/reset", timeout=5) + response = requests.post(f"{self.base_url}{_FFE_CONTROL_PATH}/reset", timeout=5, verify=self.verify) response.raise_for_status() def ffe_set_response(self, response_id: str) -> None: @@ -28,11 +30,14 @@ def ffe_set_response(self, response_id: str) -> None: def ffe_set_responses(self, responses: object) -> None: validated_responses = validate_responses(responses) response = requests.post( - f"{self.base_url}{_FFE_CONTROL_PATH}/responses", json={"responses": validated_responses}, timeout=5 + f"{self.base_url}{_FFE_CONTROL_PATH}/responses", + json={"responses": validated_responses}, + timeout=5, + verify=self.verify, ) response.raise_for_status() def ffe_get_status(self) -> MockedBackendFFEStatus: - response = requests.get(f"{self.base_url}{_FFE_CONTROL_PATH}/status", timeout=5) + response = requests.get(f"{self.base_url}{_FFE_CONTROL_PATH}/status", timeout=5, verify=self.verify) response.raise_for_status() return cast("MockedBackendFFEStatus", response.json()) diff --git a/utils/mocked_backend/_tls.py b/utils/mocked_backend/_tls.py deleted file mode 100644 index 02b3bb989f6..00000000000 --- a/utils/mocked_backend/_tls.py +++ /dev/null @@ -1,58 +0,0 @@ -"""TLS listener of the mocked backend: for clients that only speak HTTPS to hostnames derived from DD_SITE (e.g. an -SDK sending exposures directly to the intake). It serves the same routes, and records into the same folder, as the -plain HTTP server it is attached to. - -tls/mock-intake.pem holds a certificate and its key for these hostnames, signed by the mitmproxy CA, which these -clients trust. To add a hostname, regenerate it: - - openssl req -new -newkey rsa:2048 -nodes -keyout key.pem -out req.csr \ - -subj "/CN=event-platform-intake.mock-intake.invalid" - echo "subjectAltName=DNS:event-platform-intake.mock-intake.invalid,DNS:..." > ext - openssl x509 -req -in req.csr -CA utils/proxy/.mitmproxy/mitmproxy-ca.pem \ - -CAkey utils/proxy/.mitmproxy/mitmproxy-ca.pem -set_serial 0x$(openssl rand -hex 8) -days 3650 -sha256 \ - -extfile ext -out cert.pem - cat cert.pem key.pem > utils/mocked_backend/tls/mock-intake.pem -""" - -from __future__ import annotations - -import ssl -import threading -from typing import TYPE_CHECKING - -from ._request_handler import MockedBackendServerCore, MockBackendV2RequestHandler - -if TYPE_CHECKING: - from ._request_handler import HttpMethod, RouteHandler - - -class TlsListener(MockedBackendServerCore): - """Accepts HTTPS requests, and hands them to ``backend``: same routes, same records, same message count""" - - daemon_threads = True - - def __init__(self, backend: MockedBackendServerCore, port: int, cert_file: str) -> None: - super().__init__(("0.0.0.0", port), MockBackendV2RequestHandler) # noqa: S104 - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - context.load_cert_chain(cert_file) - self.socket = context.wrap_socket(self.socket, server_side=True) - self.backend = backend - self.on_message = backend.on_message - self.port = self.server_port - self._thread = threading.Thread(target=self.serve_forever, name="mock-backend-v2-tls", daemon=True) - self._thread.start() - - def get_handler(self, method: HttpMethod, path: str) -> tuple[RouteHandler, str | None, str | None]: - return self.backend.get_handler(method, path) - - def add_handler(self, method: HttpMethod, path: str, handler: RouteHandler) -> None: - self.backend.add_handler(method, path, handler) - - @property - def base_url(self) -> str: - return f"https://127.0.0.1:{self.port}" - - def close(self) -> None: - self.shutdown() - self.server_close() - self._thread.join(timeout=5) diff --git a/utils/mocked_backend/tls/README.md b/utils/mocked_backend/tls/README.md new file mode 100644 index 00000000000..0ec22a45255 --- /dev/null +++ b/utils/mocked_backend/tls/README.md @@ -0,0 +1,13 @@ +`mocked-backend.pem` holds the certificate, and its key, the mocked backend uses when it serves HTTPS +(`--tls-cert-file`). It is signed by the mitmproxy CA (`utils/proxy/.mitmproxy`), trusted by the senders of HTTPS +scenarios. It covers the container name, the intake hostnames derived from the mock `DD_SITE`, and `localhost`. + +To add a hostname, or before it expires (in 2036), regenerate it: + +```bash +openssl req -new -newkey rsa:2048 -nodes -keyout key.pem -subj "/CN=mocked-backend" -out req.csr +echo "subjectAltName=DNS:mocked-backend,DNS:event-platform-intake.mock-intake.invalid,DNS:instrumentation-telemetry-intake.mock-intake.invalid,DNS:localhost,IP:127.0.0.1" > ext +openssl x509 -req -in req.csr -CA utils/proxy/.mitmproxy/mitmproxy-ca.pem -CAkey utils/proxy/.mitmproxy/mitmproxy-ca.pem \ + -set_serial 0x$(openssl rand -hex 8) -days 3650 -sha256 -extfile ext -out cert.pem +cat cert.pem key.pem > utils/mocked_backend/tls/mocked-backend.pem +``` diff --git a/utils/mocked_backend/tls/mock-intake.pem b/utils/mocked_backend/tls/mock-intake.pem deleted file mode 100644 index a716cb53e28..00000000000 --- a/utils/mocked_backend/tls/mock-intake.pem +++ /dev/null @@ -1,48 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDTDCCAjSgAwIBAgIIMd91Y0jP1LAwDQYJKoZIhvcNAQELBQAwKDESMBAGA1UE -AwwJbWl0bXByb3h5MRIwEAYDVQQKDAltaXRtcHJveHkwHhcNMjYxMDI2MTgwOTUx -WhcNMzYxMDIzMTgwOTUxWjA0MTIwMAYDVQQDDClldmVudC1wbGF0Zm9ybS1pbnRh -a2UubW9jay1pbnRha2UuaW52YWxpZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC -AQoCggEBANtqu1rkdGU8uUspjlzGBmMmxCpFuSiOFjmXncCcKGRSw3Pcj7XMg+31 -TBBrNKYadBoIimBXt67aD5obTbfzxoGyy9GY39IHFQaUFz3mSlXqyytQlknXOSJn -dQQXj1WTUbqi2HTJ//7IAVZBXeLfC/+dgBPE+Hzjesl46/HNw6YtqmvUJoqvC6xQ -ZLh42P0wXBLC0E+R8HLK6XcbkYAjgckliwR6YgV/2uhDkSGpTp7QDGHwtT/ySO66 -SFnpbsOFekJmNmX4vTr4LDjlxe3NRWrMew3i/ok1ajJX16RcITHhUFYlVrDETant -oqPIJTajgXV3oFhdkp3fl1sxuPtD9p8CAwEAAaNuMGwwagYDVR0RBGMwYYIpZXZl -bnQtcGxhdGZvcm0taW50YWtlLm1vY2staW50YWtlLmludmFsaWSCNGluc3RydW1l -bnRhdGlvbi10ZWxlbWV0cnktaW50YWtlLm1vY2staW50YWtlLmludmFsaWQwDQYJ -KoZIhvcNAQELBQADggEBADeJRRWIsQhxyE9SIANhHcUs9+CjrX+uUBjDFZmML0Ra -rQVBYAFcPljs5piKYpptUocy5yCLls2AF+7/tIz68uNZtl1ME2etupUJV1JQfugH -Q/bPoK8D+2AzCIyzVUwHX0KSNzm8ihy8WCZuovP4bd51reRZgy1bA7LwY8TbR03D -+TjgTTCFPbvNWoAUSpOR1FzY9ALeKIOmzlHdsyike/mSLUZCJG3C7sgoNV9bgWxs -tov1ZGH0bxpyLlvPrJjpqWUpINhHpX1WV7BYGkoQoj+eGbXzWXzC3eNLWO6CfxsT -/k20yB6tkFQw1JTN/b2ipL0DGry6zIH6zgL4Yq6wyQE= ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDbarta5HRlPLlL -KY5cxgZjJsQqRbkojhY5l53AnChkUsNz3I+1zIPt9UwQazSmGnQaCIpgV7eu2g+a -G02388aBssvRmN/SBxUGlBc95kpV6ssrUJZJ1zkiZ3UEF49Vk1G6oth0yf/+yAFW -QV3i3wv/nYATxPh843rJeOvxzcOmLapr1CaKrwusUGS4eNj9MFwSwtBPkfByyul3 -G5GAI4HJJYsEemIFf9roQ5EhqU6e0Axh8LU/8kjuukhZ6W7DhXpCZjZl+L06+Cw4 -5cXtzUVqzHsN4v6JNWoyV9ekXCEx4VBWJVawxE2p7aKjyCU2o4F1d6BYXZKd35db -Mbj7Q/afAgMBAAECggEAAMcrLMLzpgOyLOonvPfayQv55rW2TensKpi7jCdk5+Q6 -0lxz8+2ocNt1OHMzWMNK567LlkL4QkhJhUEo+mM1RkIuWcwUYqzrIe0C6ienINvC -B6F1+BBwGALTWTibeayVZaOR+WWOYP3BmCWiJ2rsFWIMyroecuO+f1WOzzy3yG5D -RSPCh4rlazriGUpI918mSqsrtXqtxT7xlICccQsti+zjNaos0khhfCJENOuoEKSJ -4vMXE203aKI3UVnhV1q3zbfOwW+wQv+Eg/dDawY0e/SCFAzHdSMeidXkTq6d/cuz -cn7gbdqGSVomlMfOKq9y92ZTYUpWmUFzZ0DVWl5BOQKBgQD37Hwm/p1m5vdEbWM+ -ERSs/xHxiwPX8qMS2+TOB/cxoWu++hqNaXEXXyY6mIrfe8fJaaKbmebAMkJihl2b -gQVcRH7fzz1b9fcSyg2nRYudeBR7g4A70U1FmqZAVxJDWCNYjkPOPMo+QZ6qyKJp -tAIXL25AFF4Mzw0l/vc15AYehQKBgQDikITsJOGEfoKM7/wPixLS3y0XpC4hF8aq -zwvv4K3PJhHk4kqNtrrz5Z99MPKma9+tZ+arVRzi0YwL/LJhRAfdFYrizJC13yyP -ueyDnbF2xHVBNu59mMRftqK7etfYTiv6mm69oRTW09OBVXXUubfepRoubj88jw/d -a3d/mgxD0wKBgD8Q0hgdRq7pa9blOIyOTcMf81GNOkIRf7gJZa+fSRs/46el+WEl -hFYcpx3ateDZnN24WFvaj18uqcJkSaWH8XQiHGAUWJhu/hWyfjyhrT02UYeF0PXm -bRCaoVXWkFe/MOaS8d6hihTCHnl1JhNbbfi14Qa955q8hKy4qev0gDX9AoGARE7M -NKI9+ANoevOT7F/C0h8Fal5Oph2UAUWhVLZJ0O2qcXakgat5HDW76zPJxNE7Q/3S -dw+XQO9qqOaMJXUsZcSr9gs1ChB6T+7/VwNBUQyUSZA1QVrAwha/Y4T69LVE3y6W -esfJD6gX2aOZfgRsYk0m4p7MpAaCAwD2q1sXnY8CgYBtBY02xh0/pbygsfdwB0vq -wUaLUqjRCCFHLcN6tQ1IZwx2/kcIaPhEqW7XmUp8V7nztBHHKkaFrfmW/DWFpM++ -6m9Uim7o4lHseItVSnhqFX8uE89lrvotEGxctKfL+2oZSIlSCO/sPoADP4Xu2udl -uRC6vumSGBOyLn2H6+QGbg== ------END PRIVATE KEY----- diff --git a/utils/mocked_backend/tls/mocked-backend.pem b/utils/mocked_backend/tls/mocked-backend.pem new file mode 100644 index 00000000000..bcfa6d06eb8 --- /dev/null +++ b/utils/mocked_backend/tls/mocked-backend.pem @@ -0,0 +1,48 @@ +-----BEGIN CERTIFICATE----- +MIIDVzCCAj+gAwIBAgIISpCwLXrcV+EwDQYJKoZIhvcNAQELBQAwKDESMBAGA1UE +AwwJbWl0bXByb3h5MRIwEAYDVQQKDAltaXRtcHJveHkwHhcNMjYxMDI2MTgyNTM2 +WhcNMzYxMDIzMTgyNTM2WjAZMRcwFQYDVQQDDA5tb2NrZWQtYmFja2VuZDCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALX9Yo6kd1HjWiMHS3Ym3AJkXzBw +BzmpR+1fdXbRsl+SonAnH9Iy4dAkMqxfbRqEk+hhDfbapRGMxmpl0JGo0WG2YnGg +jXhMqPIubA82mwIItQPKy2uICJar4t0eciLXoA2mEjgrbCsLfu6zhJLIOnbyVhHx +lnJwxxUPrEaT47rq0aGV3QpMRzq+yEj00xlHG386KsQIrzq9no4kTu5vPSrWfBR9 +XpiQWzMMv1MBJmqZymdFWn3oyy9iDqN2YoK6rf3XdfhHTS5dR5nPKkkzpUiXl2zy +0DC28pST1ikUTLRSLAt3Wb7SDVXBm292LjC/BD2+s3VXciD2eb/wDMAss48CAwEA +AaOBkzCBkDCBjQYDVR0RBIGFMIGCgg5tb2NrZWQtYmFja2VuZIIpZXZlbnQtcGxh +dGZvcm0taW50YWtlLm1vY2staW50YWtlLmludmFsaWSCNGluc3RydW1lbnRhdGlv +bi10ZWxlbWV0cnktaW50YWtlLm1vY2staW50YWtlLmludmFsaWSCCWxvY2FsaG9z +dIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAFHlXGhYckM6+v1Fmk8exFO0ebNWl +UURZ7dXBBOY0HB3vyPbD8Lp90iKLq9uj4YATsCoCReIMtYe1KvmH0pp1OHNnjYn7 +DQruvaZgN8et6qQKT0NGl687RuvuVSWk6jf4Uo5f5D+M5IRymcFoVTxkcCpL0VV9 +lS1mM3FwwveivzM4GtHfpsAAESBhppLCGU1Y9mn4m1mZVNR6c2iru281zR174bdG +aNTsMZ8+iH1vM32plUU3zAxPela60vcLEaRfQw+THqsLSS7KVJk8CPurAPufEGV0 +s1ToBDPOc88Z1a6DfLs1mHc+FJCHanpn4nQ6rmPucX1cZGeLDY5dl0mBXA== +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC1/WKOpHdR41oj +B0t2JtwCZF8wcAc5qUftX3V20bJfkqJwJx/SMuHQJDKsX20ahJPoYQ322qURjMZq +ZdCRqNFhtmJxoI14TKjyLmwPNpsCCLUDystriAiWq+LdHnIi16ANphI4K2wrC37u +s4SSyDp28lYR8ZZycMcVD6xGk+O66tGhld0KTEc6vshI9NMZRxt/OirECK86vZ6O +JE7ubz0q1nwUfV6YkFszDL9TASZqmcpnRVp96MsvYg6jdmKCuq3913X4R00uXUeZ +zypJM6VIl5ds8tAwtvKUk9YpFEy0UiwLd1m+0g1VwZtvdi4wvwQ9vrN1V3Ig9nm/ +8AzALLOPAgMBAAECggEAPxdY05qwwvLOjNY6kvtlym6BkYlE9/h3txMg5dBMxlVP +exIVNgntvvrNlnR+nnLX0/fRn4KtNyhWoQwrAlREQlu4RSwj31C0QUFWtO95TXs3 +RdlILD88uv9b5sxnRSz7j7/DETX8ie+yUY5qp3hCzHm23QGj5oiye9qzHsSmbmXj +xUNkmkSokVlQpLslDGcd2qt5sDBx3Zbs9yDUJkY4PH8FqIretr25ikj98gzWBEKk +MqnzLF6POIdl0XZc2qGw5mBTqUm0ddJ5OfnrImmPLj1gKM/GsyFnR+cb0tFl2CqB +25dkkoLgIyp9V1WhmGMMKn+HjcDIb4YgIs0UEEqCAQKBgQDnEVTU4FzkVfwhQFik +bZRQhYsNHqxJcBL5rCP+9S2u+rdFgQ1ClMWCD+tpDILIVLEIlEVgj9+q3iBGnqfM +kYfMPox4Bi+czKTYGtpddhAeieDxsYSn4uNT3qpjuaEma+/E2KrPXXOtS8BljezS +PgvbpMk/dHdlDn9+P3twiwG9XQKBgQDJoGYPCnY0flbSUsDZfiFeyUNwQ4i5SMPD +u6BGhNuIY2ug211HQ6b4DjBjT78tk9QcFefAtZaESzurnV5KCEqMkDnAiqP5OInA +18y4UGG0oQF7ef0gVr/aGUPoDCVZvE0yfMpgFTSzQtqf3beXSHYsDO2tizzTjw4+ +Gv0Bjj452wKBgH/jztEX8TIMCUWaKLWxVMKQd1rmnDL1TftUdIPbzdIoQCG/AMjW +MM8A4qm82/HnNijqb8+3MfOPh8uuam9GKSQ5hmO0AMxHVAGsMypBe0B7zb3YI/UM +FghImIQjrPboDMglp7yoG6tR3wYWXHotD0EdXqm5W4/aS2Rj3H144pidAoGAKTtN +46GGDrIPWy9lFJUAifuqeNi2qv1AW1SRMBqXMkryhZCWgK5LBMsSBh0WwWFLrPql +y/jWcPNZnnyt/YG4jQopZUB0y9BOC0uWJ9L4SBHs1a1aoDRdQ2KLSnbBZJpnRv3R +/NgZUNdJEgwrMRq2X8e9CxJAPBszFGw8dQb7nqUCgYByr+C1nsIos59/ZsQorO8k +0qddrTqdrEZwYA9zvmci2aHPBXYnD5eGl3xPVyE5Wu8fxXlZRFGmpmIZcWYHH+TT +IclLv8UpOYd81NANb4Dtt+4Y9gIkR0e9b7TT50RTEykoZv4rCvslkaFaNklEe+dU +DcM+GP3FlyA8uxVuIg/4Ng== +-----END PRIVATE KEY----- From 34da82ae68d34c83685678dd34408bee0f4a3ea7 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Tue, 27 Oct 2026 01:17:34 +0100 Subject: [PATCH 06/12] Rename MOCK_INVALID_INTAKE_SITE back to MOCK_INTAKE_SITE Co-Authored-By: Claude Opus 5.5 --- utils/_context/containers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 328efcc8eb0..05f969c81ce 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -54,7 +54,7 @@ # fake key of length 32 FAKE_DD_API_KEY = "0123456789abcdef0123456789abcdef" # DD_SITE of agentless senders: the reserved .invalid domain fails closed if a request is not sent to the mocked backend -MOCK_INVALID_INTAKE_SITE = "mock-intake.invalid" +MOCK_INTAKE_SITE = "mock-intake.invalid" # its own key tells, in the mocked backend, what serverless-init sent (see interfaces.backend_v2 with_api_key) SERVERLESS_INIT_FAKE_DD_API_KEY = "fedcba9876543210fedcba9876543210" @@ -795,8 +795,8 @@ class MockedBackendContainer(TestedContainer): # from DD_SITE (e.g. an SDK sending exposures directly). Its certificate is signed by the mitmproxy CA: these # senders must trust it. See utils/mocked_backend/tls/README.md tls_hostnames = ( - f"event-platform-intake.{MOCK_INVALID_INTAKE_SITE}", - f"instrumentation-telemetry-intake.{MOCK_INVALID_INTAKE_SITE}", + f"event-platform-intake.{MOCK_INTAKE_SITE}", + f"instrumentation-telemetry-intake.{MOCK_INTAKE_SITE}", ) tls_cert_file = "utils/mocked_backend/tls/mocked-backend.pem" tls_ca_file = "utils/proxy/.mitmproxy/mitmproxy-ca-cert.pem" @@ -996,7 +996,7 @@ def __init__(self) -> None: image_name="datadog/serverless-init:1.10.4", environment={ "DD_API_KEY": SERVERLESS_INIT_FAKE_DD_API_KEY, - "DD_SITE": MOCK_INVALID_INTAKE_SITE, + "DD_SITE": MOCK_INTAKE_SITE, "DD_SERVICE": "ffe-system-tests-serverless-init", "DD_ENV": "system-tests", "DD_APM_ENABLED": "true", From 4c3833b0da20921f96b33d51697c11208bd576b6 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Tue, 27 Oct 2026 01:27:02 +0100 Subject: [PATCH 07/12] Release the mocked backend port if its certificate cannot be loaded Co-Authored-By: Claude Opus 5.5 --- tests/test_the_test/test_mock_backend_v2.py | 14 ++++++++++++++ utils/mocked_backend/_backend.py | 11 ++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/test_the_test/test_mock_backend_v2.py b/tests/test_the_test/test_mock_backend_v2.py index 202942cf23e..e9a03eb19f6 100644 --- a/tests/test_the_test/test_mock_backend_v2.py +++ b/tests/test_the_test/test_mock_backend_v2.py @@ -324,6 +324,20 @@ def test_https_mode_does_not_serve_http(tmp_path: Path): server.close() +@scenarios.test_the_test +def test_https_mode_releases_the_port_if_the_certificate_cannot_be_loaded(tmp_path: Path): + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + + with pytest.raises(FileNotFoundError) as error: + MockedBackendServer(str(tmp_path), port=port, tls_cert_file=str(tmp_path / "missing.pem")) + + # the port is free again, even while the error (holding the half-built server) is still referenced + assert error.traceback + MockedBackendServer(str(tmp_path), port=port).close() + + @scenarios.test_the_test def test_https_certificate_is_only_valid_for_the_served_hostnames(tmp_path: Path): server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE) diff --git a/utils/mocked_backend/_backend.py b/utils/mocked_backend/_backend.py index 84bf228fe52..89e0c5ccd1d 100644 --- a/utils/mocked_backend/_backend.py +++ b/utils/mocked_backend/_backend.py @@ -51,9 +51,14 @@ def __init__( super().__init__(("0.0.0.0", port), MockBackendV2RequestHandler) # noqa: S104 self.scheme = "https" if tls_cert_file else "http" if tls_cert_file: - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - context.load_cert_chain(tls_cert_file) - self.socket = context.wrap_socket(self.socket, server_side=True) + try: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(tls_cert_file) + self.socket = context.wrap_socket(self.socket, server_side=True) + except Exception: + # the caller never gets a server to close: release the port now + self.server_close() + raise self.log_folder = log_folder self.on_message = on_message if on_message else lambda _: None self.message_count = 0 From bbd2a086720599d1a54ad93d17f5506c27864910 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Tue, 27 Oct 2026 10:32:04 +0100 Subject: [PATCH 08/12] Note that network aliases are not supported on GitLab's bridge network Co-Authored-By: Claude Opus 5.5 --- utils/_context/containers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 05f969c81ce..c679eac58e9 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -813,6 +813,8 @@ def __init__(self, *, tls: bool = False) -> None: image_name="datadog/system-tests:proxy-v1", # a valid hostname: some HTTP clients reject an underscore name="mocked-backend", + # Docker rejects aliases on the default bridge network, used on GitLab (see _NETWORK_NAME). If a tls + # scenario runs there, map tls_hostnames to this container with extra_hosts on the senders instead network_aliases=list(self.tls_hostnames) if tls else None, working_dir="/app", volumes={ From 12bb1b3f4b6801c46aaa5a2b9885746f25854fb1 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Tue, 27 Oct 2026 10:45:40 +0100 Subject: [PATCH 09/12] Run the mocked backend TLS handshake in the request thread An idle connection no longer blocks the other requests, nor close(). Co-Authored-By: Claude Opus 5.5 --- tests/test_the_test/test_mock_backend_v2.py | 23 +++++++++++++++++++++ utils/mocked_backend/_backend.py | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/test_the_test/test_mock_backend_v2.py b/tests/test_the_test/test_mock_backend_v2.py index e9a03eb19f6..a9a6d1e020a 100644 --- a/tests/test_the_test/test_mock_backend_v2.py +++ b/tests/test_the_test/test_mock_backend_v2.py @@ -4,6 +4,7 @@ import gzip import signal import socket +import threading import ssl import json from collections.abc import Generator @@ -338,6 +339,28 @@ def test_https_mode_releases_the_port_if_the_certificate_cannot_be_loaded(tmp_pa MockedBackendServer(str(tmp_path), port=port).close() +@scenarios.test_the_test +def test_https_mode_is_not_blocked_by_an_idle_connection(tmp_path: Path): + server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE) + closed = threading.Event() + + def close() -> None: + server.close() + closed.set() + + try: + # connects, and never starts the TLS handshake + with socket.create_connection(("127.0.0.1", server.port), timeout=5): + status_line = _https_post(server.port, _INTAKE_HOSTNAME, "/api/v2/exposures", b"{}") + threading.Thread(target=close, daemon=True).start() + assert closed.wait(timeout=10), "close() hangs while a connection is idle" + finally: + if not closed.is_set(): + server.close() + + assert status_line == "HTTP/1.0 200 OK" + + @scenarios.test_the_test def test_https_certificate_is_only_valid_for_the_served_hostnames(tmp_path: Path): server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE) diff --git a/utils/mocked_backend/_backend.py b/utils/mocked_backend/_backend.py index 89e0c5ccd1d..f23eea6b4d5 100644 --- a/utils/mocked_backend/_backend.py +++ b/utils/mocked_backend/_backend.py @@ -54,7 +54,8 @@ def __init__( try: context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) context.load_cert_chain(tls_cert_file) - self.socket = context.wrap_socket(self.socket, server_side=True) + # the handshake runs in each request thread, not in the accepting one: an idle client blocks nothing + self.socket = context.wrap_socket(self.socket, server_side=True, do_handshake_on_connect=False) except Exception: # the caller never gets a server to close: release the port now self.server_close() From 5984109ddce93d66e35fd71f6eee3e8aeb79aaff Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Tue, 27 Oct 2026 10:47:54 +0100 Subject: [PATCH 10/12] Advertise the mocked backend URLs with its scheme Co-Authored-By: Claude Opus 5.5 --- tests/test_the_test/test_mock_backend_v2.py | 10 ++++++++++ utils/mocked_backend/_backend.py | 2 +- utils/mocked_backend/_backend_ffe.py | 2 +- utils/mocked_backend/_request_handler.py | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/test_the_test/test_mock_backend_v2.py b/tests/test_the_test/test_mock_backend_v2.py index a9a6d1e020a..8410dc854e7 100644 --- a/tests/test_the_test/test_mock_backend_v2.py +++ b/tests/test_the_test/test_mock_backend_v2.py @@ -361,6 +361,16 @@ def close() -> None: assert status_line == "HTTP/1.0 200 OK" +@scenarios.test_the_test +def test_https_mode_advertises_https_urls(tmp_path: Path): + server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE) + try: + assert server.container_url.startswith("https://") + assert server.ffe_library_config_url.startswith("https://") + finally: + server.close() + + @scenarios.test_the_test def test_https_certificate_is_only_valid_for_the_served_hostnames(tmp_path: Path): server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE) diff --git a/utils/mocked_backend/_backend.py b/utils/mocked_backend/_backend.py index f23eea6b4d5..4cbf1aa83c1 100644 --- a/utils/mocked_backend/_backend.py +++ b/utils/mocked_backend/_backend.py @@ -88,7 +88,7 @@ def client(self) -> MockedBackendClient: @property def container_url(self) -> str: """The URL a container can use to reach the mock server on the docker host.""" - return f"http://{HOST_DOCKER_INTERNAL}:{self.port}" + return f"{self.scheme}://{HOST_DOCKER_INTERNAL}:{self.port}" def add_handler(self, method: HttpMethod, path: str, handler: RouteHandler) -> None: """Register a handler for an exact (method, path) pair. ``path`` is matched without its diff --git a/utils/mocked_backend/_backend_ffe.py b/utils/mocked_backend/_backend_ffe.py index 41eb2fab922..1231004dff5 100644 --- a/utils/mocked_backend/_backend_ffe.py +++ b/utils/mocked_backend/_backend_ffe.py @@ -225,7 +225,7 @@ def _ffe_library_base_url(self) -> str: host = os.environ.get("SYSTEM_TESTS_MOCK_FFE_AGENTLESS_BACKEND_HOST") or os.environ.get( "SYSTEM_TESTS_MOCK_AGENTLESS_BACKEND_HOST", "host.docker.internal" ) - return f"http://{host}:{self.port}" + return f"{self.scheme}://{host}:{self.port}" @property def ffe_library_config_url(self) -> str: diff --git a/utils/mocked_backend/_request_handler.py b/utils/mocked_backend/_request_handler.py index afe95beb03c..0961f2ccbe3 100644 --- a/utils/mocked_backend/_request_handler.py +++ b/utils/mocked_backend/_request_handler.py @@ -57,6 +57,7 @@ def _decode_content(raw_body: bytes, content_encoding: str) -> bytes: class MockedBackendServerCore(ABC, ThreadingHTTPServer): on_message: Callable[[dict], None] port: int + scheme: str @abstractmethod def get_handler(self, method: HttpMethod, path: str) -> tuple[RouteHandler, str | None, str | None]: ... From 22a01218c9b22173f60dc82203309fa4f9029663 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Tue, 27 Oct 2026 11:06:07 +0100 Subject: [PATCH 11/12] Document that HTTPS is meant for the mocked backend container Co-Authored-By: Claude Opus 5.5 --- utils/mocked_backend/_backend.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/mocked_backend/_backend.py b/utils/mocked_backend/_backend.py index 4cbf1aa83c1..56e3e34d7fc 100644 --- a/utils/mocked_backend/_backend.py +++ b/utils/mocked_backend/_backend.py @@ -47,6 +47,9 @@ def __init__( ) -> None: """With tls_cert_file (a certificate and its key), it serves HTTPS instead of HTTP: for senders that only speak HTTPS to the intake hostnames derived from DD_SITE. + + HTTPS is meant for the mocked backend container (MockedBackendContainer(tls=True)): the certificate covers its + hostnames, not host.docker.internal, so the URLs this server advertises to containers can't be verified. """ super().__init__(("0.0.0.0", port), MockBackendV2RequestHandler) # noqa: S104 self.scheme = "https" if tls_cert_file else "http" From 0a12ffbb1e4050b6ad214e68c3f636679f594b7f Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Tue, 27 Oct 2026 19:05:09 +0100 Subject: [PATCH 12/12] remove dead code --- utils/_context/containers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index c679eac58e9..54705b58487 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -1851,7 +1851,6 @@ def __init__(self) -> None: command="uvicorn app:app --host 0.0.0.0 --port 8089", volumes={ "./utils/build/docker/internal_server/app.py": {"bind": "/app/app.py", "mode": "ro"}, - "./utils/build/docker/internal_server/app.sh": {"bind": "/app/app.sh", "mode": "ro"}, }, )