Prepare the mocked backend to receive agentless FFE traffic

tests/test_the_test/test_mock_backend_v2.py

@@ -1,7 +1,11 @@
 """Unit coverage for MockedBackendServer (utils.mocked_backend)."""
 
+from datetime import UTC, datetime, timedelta
 import gzip
 import signal
+import socket
+import threading
+import ssl
 import json
 from collections.abc import Generator
 from pathlib import Path
@@ -11,8 +15,11 @@
 from utils import pytest
 import requests
 import zstandard
+from cryptography import x509
 
 from utils import scenarios
+from utils.interfaces._backend_v2 import _BackendV2InterfaceValidator
+from utils._context.containers import MockedBackendContainer
 from utils.mocked_backend import MockedBackendClient, MockedBackendServer
 from utils.mocked_backend.ffe import UFC_FIXTURE_RELATIVE_PATH
 
@@ -231,3 +238,172 @@ def test_health_endpoint(server: MockedBackendServer, tmp_path: Path):
     assert response.json() == {"status": "ok"}
     # a control endpoint: not recorded
     assert list(tmp_path.glob("*.json")) == []
+
+
+def _backend_v2_with_requests(tmp_path: Path, requests_: list[tuple[str, str | None]]) -> _BackendV2InterfaceValidator:
+    """A backend_v2 interface that received a series request per (filename, api_key)"""
+    interface = _BackendV2InterfaceValidator()
+    interface.configure(str(tmp_path), replay=False)
+    for filename, api_key in requests_:
+        headers = [["Content-Type", "application/json"]] + ([["DD-API-KEY", api_key]] if api_key else [])
+        file = Path(interface.log_folder) / filename
+        data = {
+            "log_filename": str(file),
+            "path": "/api/v2/series",
+            "request": {"headers": headers, "content": {"series": [{"metric": filename}]}},
+            "response": {},
+        }
+        file.write_text(json.dumps(data))
+        interface.ingest_file(str(file))
+    return interface
+
+
+@scenarios.test_the_test
+def test_backend_v2_with_api_key_only_returns_requests_sent_with_this_key(tmp_path: Path):
+    interface = _backend_v2_with_requests(
+        tmp_path, [("000__agent.json", "agent-key"), ("001__sidecar.json", "sidecar-key"), ("002__nokey.json", None)]
+    )
+
+    assert [d["log_filename"].split("/")[-1] for d in interface.get_data(with_api_key="sidecar-key")] == [
+        "001__sidecar.json"
+    ]
+    assert [point["metric"] for _, point in interface.get_metrics(with_api_key="agent-key")] == ["000__agent.json"]
+    assert list(interface.get_data(with_api_key="unknown-key")) == []
+    # without it, every request
+    assert len(list(interface.get_data())) == 3
+
+
+@scenarios.test_the_test
+def test_backend_v2_with_api_key_ignores_the_header_name_case(tmp_path: Path):
+    interface = _backend_v2_with_requests(tmp_path, [("000__agent.json", "agent-key")])
+    data = next(interface.get_data())
+    data["request"]["headers"] = [["dd-api-key", "agent-key"]]
+
+    assert len(list(interface.get_data(with_api_key="agent-key"))) == 1
+
+
+_MITMPROXY_CA = "utils/proxy/.mitmproxy/mitmproxy-ca-cert.pem"
+_TLS_CERT_FILE = MockedBackendContainer.tls_cert_file
+_INTAKE_HOSTNAME = "event-platform-intake.mock-intake.invalid"
+
+
+def _https_post(port: int, hostname: str, path: str, body: bytes) -> str:
+    """POST over HTTPS to 127.0.0.1, verifying the certificate is valid for hostname: the status line"""
+    context = ssl.create_default_context(cafile=_MITMPROXY_CA)
+    with (
+        socket.create_connection(("127.0.0.1", port), timeout=5) as raw,
+        context.wrap_socket(raw, server_hostname=hostname) as tls,
+    ):
+        request = f"POST {path} HTTP/1.1\r\nHost: {hostname}\r\nContent-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n"
+        tls.sendall(request.encode() + body)
+        return tls.recv(4096).decode().splitlines()[0]
+
+
+@scenarios.test_the_test
+def test_https_mode_serves_and_records_https_requests(tmp_path: Path):
+    server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE)
+    try:
+        status_line = _https_post(server.port, _INTAKE_HOSTNAME, "/api/v2/exposures", b'{"over": "https"}')
+        # the control endpoints, as used from the host
+        health = requests.get(f"{server.base_url}/system-tests/health", timeout=5, verify=_MITMPROXY_CA)
+    finally:
+        server.close()
+
+    assert status_line == "HTTP/1.0 200 OK"
+    assert health.json() == {"status": "ok"}
+    recorded = json.loads((tmp_path / "000__api_v2_exposures.json").read_text())
+    assert recorded["request"]["content"] == {"over": "https"}
+
+
+@scenarios.test_the_test
+def test_https_mode_does_not_serve_http(tmp_path: Path):
+    server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE)
+    try:
+        with pytest.raises(requests.exceptions.ConnectionError):
+            requests.post(f"http://127.0.0.1:{server.port}/api/v2/series", json={}, timeout=5)
+    finally:
+        server.close()
+
+
+@scenarios.test_the_test
+def test_https_mode_releases_the_port_if_the_certificate_cannot_be_loaded(tmp_path: Path):
+    with socket.socket() as probe:
+        probe.bind(("127.0.0.1", 0))
+        port = probe.getsockname()[1]
+
+    with pytest.raises(FileNotFoundError) as error:
+        MockedBackendServer(str(tmp_path), port=port, tls_cert_file=str(tmp_path / "missing.pem"))
+
+    # the port is free again, even while the error (holding the half-built server) is still referenced
+    assert error.traceback
+    MockedBackendServer(str(tmp_path), port=port).close()
+
+
+@scenarios.test_the_test
+def test_https_mode_is_not_blocked_by_an_idle_connection(tmp_path: Path):
+    server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE)
+    closed = threading.Event()
+
+    def close() -> None:
+        server.close()
+        closed.set()
+
+    try:
+        # connects, and never starts the TLS handshake
+        with socket.create_connection(("127.0.0.1", server.port), timeout=5):
+            status_line = _https_post(server.port, _INTAKE_HOSTNAME, "/api/v2/exposures", b"{}")
+            threading.Thread(target=close, daemon=True).start()
+            assert closed.wait(timeout=10), "close() hangs while a connection is idle"
+    finally:
+        if not closed.is_set():
+            server.close()
+
+    assert status_line == "HTTP/1.0 200 OK"
+
+
+@scenarios.test_the_test
+def test_https_mode_advertises_https_urls(tmp_path: Path):
+    server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE)
+    try:
+        assert server.container_url.startswith("https://")
+        assert server.ffe_library_config_url.startswith("https://")
+    finally:
+        server.close()
+
+
+@scenarios.test_the_test
+def test_https_certificate_is_only_valid_for_the_served_hostnames(tmp_path: Path):
+    server = MockedBackendServer(str(tmp_path), port=0, tls_cert_file=_TLS_CERT_FILE)
+    try:
+        with pytest.raises(ssl.SSLCertVerificationError):
+            _https_post(server.port, "other.mock-intake.invalid", "/api/v2/exposures", b"{}")
+    finally:
+        server.close()
+
+
+@scenarios.test_the_test
+def test_https_certificate_covers_the_container_hostnames_for_a_while():
+    """The committed certificate must be regenerated when a hostname is added, or before it expires"""
+    cert = x509.load_pem_x509_certificate(Path(_TLS_CERT_FILE).read_bytes())
+    names = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
+    container = MockedBackendContainer(tls=True)
+
+    # reached by its name, its intake hostnames, and by localhost from the healthcheck and the host
+    assert sorted(names.get_values_for_type(x509.DNSName)) == sorted(
+        [container.name, *container.network_aliases, "localhost"]
+    )
+    assert [str(ip) for ip in names.get_values_for_type(x509.IPAddress)] == ["127.0.0.1"]
+    # naive, in UTC
+    assert cert.not_valid_after.replace(tzinfo=UTC) - datetime.now(UTC) > timedelta(days=180)
+
+
+@scenarios.test_the_test
+def test_mocked_backend_container_serves_either_http_or_https():
+    http = MockedBackendContainer()
+    assert http.container_url == "http://mocked-backend:4900"
+    assert http.network_aliases == []
+
+    https = MockedBackendContainer(tls=True)
+    # senders build the intake URLs from DD_SITE: the default HTTPS port, and the intake hostnames
+    assert https.container_url == "https://mocked-backend:443"
+    assert https.network_aliases == list(MockedBackendContainer.tls_hostnames)

utils/_context/containers.py

@@ -53,6 +53,10 @@
 
 # fake key of length 32
 FAKE_DD_API_KEY = "0123456789abcdef0123456789abcdef"
+# DD_SITE of agentless senders: the reserved .invalid domain fails closed if a request is not sent to the mocked backend
+MOCK_INTAKE_SITE = "mock-intake.invalid"
+# its own key tells, in the mocked backend, what serverless-init sent (see interfaces.backend_v2 with_api_key)
+SERVERLESS_INIT_FAKE_DD_API_KEY = "fedcba9876543210fedcba9876543210"
 
 _DEFAULT_NETWORK_NAME = "system-tests_default"
 _NETWORK_NAME = "bridge" if "GITLAB_CI" in os.environ else _DEFAULT_NETWORK_NAME
@@ -103,6 +107,7 @@ def __init__(
         pid_mode: str | None = None,
         cgroupns: str | None = None,
         mounts: list[docker.types.Mount] | None = None,
+        network_aliases: list[str] | None = None,
     ) -> None:
         assert name
         assert image_name
@@ -141,6 +146,8 @@ def __init__(
         # Unlike volumes, a bind mount listed here fails container creation when its source is
         # missing, instead of being created as an empty directory by the docker daemon.
         self.mounts = mounts
+        # other names other containers of the network can reach it with
+        self.network_aliases = network_aliases or []
 
     def _get_image_name(self, binary_file_name: str | None, default_name: str) -> str:
         # if the container provide binary_file_name, then a file named binaries/{binary_file_name}
@@ -254,6 +261,11 @@ def start(self, network: Network) -> Container:
             # auto_remove=True,
             detach=True,
             network=network.name,
+            networking_config=(
+                {network.name: get_docker_client().api.create_endpoint_config(aliases=self.network_aliases)}
+                if self.network_aliases
+                else None
+            ),
             volumes=self.volumes,
             ports=self.ports,
             working_dir=self.working_dir,
@@ -779,14 +791,31 @@ class MockedBackendContainer(TestedContainer):
     What it receives is recorded in the logs folder, and read by interfaces.backend_v2.
     """
 
-    port = 4900
-
-    def __init__(self) -> None:
+    # With tls, it serves HTTPS instead of HTTP: for senders that only speak HTTPS to the intake hostnames derived
+    # from DD_SITE (e.g. an SDK sending exposures directly). Its certificate is signed by the mitmproxy CA: these
+    # senders must trust it. See utils/mocked_backend/tls/README.md
+    tls_hostnames = (
+        f"event-platform-intake.{MOCK_INTAKE_SITE}",
+        f"instrumentation-telemetry-intake.{MOCK_INTAKE_SITE}",
+    )
+    tls_cert_file = "utils/mocked_backend/tls/mocked-backend.pem"
+    tls_ca_file = "utils/proxy/.mitmproxy/mitmproxy-ca-cert.pem"
+
+    def __init__(self, *, tls: bool = False) -> None:
+        self.tls = tls
+        self.scheme = "https" if tls else "http"
+        # the default HTTPS port: senders build the intake URLs from DD_SITE, without port
+        self.port = 443 if tls else 4900
+        # in the healthcheck, the certificate is checked against localhost
+        health_context = f"ssl.create_default_context(cafile='/app/{self.tls_ca_file}')" if tls else "None"
         super().__init__(
             # it has every dependency of utils/mocked_backend
             image_name="datadog/system-tests:proxy-v1",
             # a valid hostname: some HTTP clients reject an underscore
             name="mocked-backend",
+            # Docker rejects aliases on the default bridge network, used on GitLab (see _NETWORK_NAME). If a tls
+            # scenario runs there, map tls_hostnames to this container with extra_hosts on the senders instead
+            network_aliases=list(self.tls_hostnames) if tls else None,
             working_dir="/app",
             volumes={
                 # without utils/__init__.py: utils is a namespace package, and the rest of utils is not needed
@@ -798,7 +827,7 @@ def __init__(self) -> None:
             # for the control endpoints, used from the host. Docker picks a free host port
             ports={f"{self.port}/tcp": ("127.0.0.1", None)},
             healthcheck={
-                "test": f"python -c \"import urllib.request; print(urllib.request.urlopen('http://localhost:{self.port}/system-tests/health', timeout=2).read().decode())\"",  # noqa: E501
+                "test": f"python -c \"import ssl, urllib.request; print(urllib.request.urlopen('{self.scheme}://localhost:{self.port}/system-tests/health', timeout=2, context={health_context}).read().decode())\"",  # noqa: E501
                 "retries": 30,
             },
         )
@@ -809,18 +838,30 @@ def configure(self, *, host_log_folder: str, replay: bool):
         # Same relative path in the container and on the host, so recorded log filenames are the same
         log_folder = f"{host_log_folder}/interfaces/backend_v2"
         self.volumes[f"./{log_folder}"] = {"bind": f"/app/{log_folder}", "mode": "rw"}
-        self.command = ["python", "-m", "utils.mocked_backend", "--port", str(self.port), "--log-folder", log_folder]
+        self.command = [
+            "python",
+            "-m",
+            "utils.mocked_backend",
+            "--port",
+            str(self.port),
+            "--log-folder",
+            log_folder,
+        ]
+        if self.tls:
+            self.command += ["--tls-cert-file", self.tls_cert_file]
 
     @property
     def container_url(self) -> str:
         """The URL other containers use to reach the mocked backend"""
-        return f"http://{self.name}:{self.port}"
+        return f"{self.scheme}://{self.name}:{self.port}"
 
     @property
     def client(self) -> MockedBackendClient:
         """Client of the control endpoints, from the host"""
         host_port = self.runtime_container.attrs["NetworkSettings"]["Ports"][f"{self.port}/tcp"][0]["HostPort"]
-        return MockedBackendClient(f"http://127.0.0.1:{host_port}")
+        return MockedBackendClient(
+            f"{self.scheme}://127.0.0.1:{host_port}", verify=self.tls_ca_file if self.tls else True
+        )
 
 
 class LambdaProxyContainer(TestedContainer):
@@ -956,8 +997,8 @@ def __init__(self) -> None:
             name="ffe-serverless-init",
             image_name="datadog/serverless-init:1.10.4",
             environment={
-                "DD_API_KEY": FAKE_DD_API_KEY,
-                "DD_SITE": "mock-intake.invalid",
+                "DD_API_KEY": SERVERLESS_INIT_FAKE_DD_API_KEY,
+                "DD_SITE": MOCK_INTAKE_SITE,
                 "DD_SERVICE": "ffe-system-tests-serverless-init",
                 "DD_ENV": "system-tests",
                 "DD_APM_ENABLED": "true",
@@ -1810,7 +1851,6 @@ def __init__(self) -> None:
             command="uvicorn app:app --host 0.0.0.0 --port 8089",
             volumes={
                 "./utils/build/docker/internal_server/app.py": {"bind": "/app/app.py", "mode": "ro"},
-                "./utils/build/docker/internal_server/app.sh": {"bind": "/app/app.sh", "mode": "ro"},
             },
         )