From 70c3d48200ea8c1f5a5522efe50e2b5a4d7273e2 Mon Sep 17 00:00:00 2001 From: "damian.malysza" Date: Wed, 28 Oct 2026 01:16:17 +0000 Subject: [PATCH 1/3] Hide onboarding API keys in pytest failure locals Preserve credential string values for HTTP authentication while redacting their repr in request and key-selection locals. Assisted by Codex (GPT-6). --- .../test_onboarding_backend_credentials.py | 50 +++++++++++++++++++ utils/onboarding/backend_interface.py | 17 +++++-- 2 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 tests/test_the_test/test_onboarding_backend_credentials.py diff --git a/tests/test_the_test/test_onboarding_backend_credentials.py b/tests/test_the_test/test_onboarding_backend_credentials.py new file mode 100644 index 00000000000..1dedad04cbe --- /dev/null +++ b/tests/test_the_test/test_onboarding_backend_credentials.py @@ -0,0 +1,50 @@ +import requests + +from utils import pytest +from utils.onboarding import backend_interface + + +pytestmark = pytest.mark.scenario("TEST_THE_TEST") + + +@pytest.mark.parametrize("query", ["trace", "profile"]) +def test_backend_failure_locals_hide_authentication_keys(monkeypatch: pytest.MonkeyPatch, query: str) -> None: + # Only backend frames belong in the failure rendering under test. + __tracebackhide__ = True + monkeypatch.setenv("DD_API_KEY_ONBOARDING", "synthetic-onboarding-api-key") + monkeypatch.setenv("DD_APP_KEY_ONBOARDING", "synthetic-onboarding-app-key") + monkeypatch.delenv("DD_API_KEY_ONBOARDING_2", raising=False) + monkeypatch.delenv("DD_APP_KEY_ONBOARDING_2", raising=False) + prepared_requests: list[requests.PreparedRequest] = [] + + def unavailable_response( + _session: requests.Session, request: requests.PreparedRequest, **_kwargs: object + ) -> requests.Response: + prepared_requests.append(request) + response = requests.Response() + response.status_code = 503 + return response + + monkeypatch.setattr(requests.Session, "send", unavailable_response) + times = iter([0, 400]) + monkeypatch.setattr(backend_interface.time, "perf_counter", lambda: next(times)) + + with pytest.raises(TimeoutError) as failure: + if query == "trace": + backend_interface._query_for_trace_id("synthetic-trace-id") # noqa: SLF001 + else: + backend_interface._query_for_profile("synthetic-runtime-id") # noqa: SLF001 + + request = prepared_requests[0] + assert request.headers["DD-API-KEY"] == "synthetic-onboarding-api-key" + assert request.headers["DD-APPLICATION-KEY"] == "synthetic-onboarding-app-key" + if query == "profile": + assert request.headers["Content-Type"] == "application/json" + + failure_output = str(failure.getrepr(showlocals=True)) + assert "synthetic-onboarding-api-key" not in failure_output + assert "synthetic-onboarding-app-key" not in failure_output + assert "" in failure_output + assert "Reached overall timeout" in failure_output + assert request.url is not None + assert request.url in failure_output diff --git a/utils/onboarding/backend_interface.py b/utils/onboarding/backend_interface.py index b79cc8eb980..fa2eeb041f6 100644 --- a/utils/onboarding/backend_interface.py +++ b/utils/onboarding/backend_interface.py @@ -27,13 +27,22 @@ def wait_backend_trace_id(trace_id: str, *, profile: bool = False, validator: Ca _query_for_profile(runtime_id) +class _Credential(str): + __slots__ = () + + # Pytest renders request headers and selected key pairs in failure locals. + # Keep the string value for HTTP requests while hiding it from those reprs. + def __repr__(self) -> str: + return "''" + + def _headers(): """The backend can raise a 429 error if the rate limit is reached. We can use several app keys trying to avoid the rate limit. """ # Retrieve the mandatory API and APP keys. - api_key = os.getenv("DD_API_KEY_ONBOARDING") - app_key = os.getenv("DD_APP_KEY_ONBOARDING") + api_key = _Credential(os.getenv("DD_API_KEY_ONBOARDING", "")) + app_key = _Credential(os.getenv("DD_APP_KEY_ONBOARDING", "")) if not api_key or not app_key: raise ValueError("Mandatory API or APP key is missing.") @@ -44,8 +53,8 @@ def _headers(): # Check for additional numbered pairs. index = 2 while True: - candidate_api = os.getenv(f"DD_API_KEY_ONBOARDING_{index}") - candidate_app = os.getenv(f"DD_APP_KEY_ONBOARDING_{index}") + candidate_api = _Credential(os.getenv(f"DD_API_KEY_ONBOARDING_{index}", "")) + candidate_app = _Credential(os.getenv(f"DD_APP_KEY_ONBOARDING_{index}", "")) if candidate_api and candidate_app: pairs.append((candidate_api, candidate_app)) index += 1 From ad3613bead425b76197b817adbc15e8df505caa0 Mon Sep 17 00:00:00 2001 From: "damian.malysza" Date: Wed, 28 Oct 2026 01:30:43 +0000 Subject: [PATCH 2/3] Omit credential-bearing helpers from pytest failure reports Use native traceback filtering to omit authentication locals while retaining the public caller and operation error. Restore ordinary credential strings and HTTP preparation. Assisted by Codex (GPT-6). --- .../test_onboarding_backend_credentials.py | 43 +++++++++++++------ utils/onboarding/backend_interface.py | 26 +++++------ 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/tests/test_the_test/test_onboarding_backend_credentials.py b/tests/test_the_test/test_onboarding_backend_credentials.py index 1dedad04cbe..5844236baef 100644 --- a/tests/test_the_test/test_onboarding_backend_credentials.py +++ b/tests/test_the_test/test_onboarding_backend_credentials.py @@ -1,3 +1,6 @@ +from datetime import UTC, datetime +import json + import requests from utils import pytest @@ -7,8 +10,8 @@ pytestmark = pytest.mark.scenario("TEST_THE_TEST") -@pytest.mark.parametrize("query", ["trace", "profile"]) -def test_backend_failure_locals_hide_authentication_keys(monkeypatch: pytest.MonkeyPatch, query: str) -> None: +@pytest.mark.parametrize("profile", [False, True]) +def test_backend_failure_reports_omit_authentication_headers(monkeypatch: pytest.MonkeyPatch, *, profile: bool) -> None: # Only backend frames belong in the failure rendering under test. __tracebackhide__ = True monkeypatch.setenv("DD_API_KEY_ONBOARDING", "synthetic-onboarding-api-key") @@ -17,34 +20,50 @@ def test_backend_failure_locals_hide_authentication_keys(monkeypatch: pytest.Mon monkeypatch.delenv("DD_APP_KEY_ONBOARDING_2", raising=False) prepared_requests: list[requests.PreparedRequest] = [] - def unavailable_response( + def backend_response( _session: requests.Session, request: requests.PreparedRequest, **_kwargs: object ) -> requests.Response: prepared_requests.append(request) response = requests.Response() response.status_code = 503 + if profile and len(prepared_requests) == 1: + response.status_code = 200 + response._content = json.dumps( # noqa: SLF001 + { + "trace": { + "root_id": "root", + "spans": { + "root": { + "start": datetime.now(tz=UTC).timestamp(), + "meta": {"runtime-id": "synthetic-runtime-id"}, + } + }, + } + } + ).encode() return response - monkeypatch.setattr(requests.Session, "send", unavailable_response) - times = iter([0, 400]) + monkeypatch.setattr(requests.Session, "send", backend_response) + times = iter([0, 0, 400] if profile else [0, 400]) monkeypatch.setattr(backend_interface.time, "perf_counter", lambda: next(times)) with pytest.raises(TimeoutError) as failure: - if query == "trace": - backend_interface._query_for_trace_id("synthetic-trace-id") # noqa: SLF001 - else: - backend_interface._query_for_profile("synthetic-runtime-id") # noqa: SLF001 + backend_interface.wait_backend_trace_id("synthetic-trace-id", profile=profile) - request = prepared_requests[0] + request = prepared_requests[-1] assert request.headers["DD-API-KEY"] == "synthetic-onboarding-api-key" assert request.headers["DD-APPLICATION-KEY"] == "synthetic-onboarding-app-key" - if query == "profile": + if profile: assert request.headers["Content-Type"] == "application/json" failure_output = str(failure.getrepr(showlocals=True)) assert "synthetic-onboarding-api-key" not in failure_output assert "synthetic-onboarding-app-key" not in failure_output - assert "" in failure_output + assert "DD-API-KEY" not in failure_output + assert "DD-APPLICATION-KEY" not in failure_output + assert "wait_backend_trace_id" in failure_output + assert "synthetic-trace-id" in failure_output assert "Reached overall timeout" in failure_output + assert ("post" if profile else "get") in failure_output assert request.url is not None assert request.url in failure_output diff --git a/utils/onboarding/backend_interface.py b/utils/onboarding/backend_interface.py index fa2eeb041f6..77742ea2633 100644 --- a/utils/onboarding/backend_interface.py +++ b/utils/onboarding/backend_interface.py @@ -27,22 +27,16 @@ def wait_backend_trace_id(trace_id: str, *, profile: bool = False, validator: Ca _query_for_profile(runtime_id) -class _Credential(str): - __slots__ = () - - # Pytest renders request headers and selected key pairs in failure locals. - # Keep the string value for HTTP requests while hiding it from those reprs. - def __repr__(self) -> str: - return "''" - - def _headers(): """The backend can raise a 429 error if the rate limit is reached. We can use several app keys trying to avoid the rate limit. """ + # Omit credential-selection locals from pytest failure reports. + __tracebackhide__ = True + # Retrieve the mandatory API and APP keys. - api_key = _Credential(os.getenv("DD_API_KEY_ONBOARDING", "")) - app_key = _Credential(os.getenv("DD_APP_KEY_ONBOARDING", "")) + api_key = os.getenv("DD_API_KEY_ONBOARDING") + app_key = os.getenv("DD_APP_KEY_ONBOARDING") if not api_key or not app_key: raise ValueError("Mandatory API or APP key is missing.") @@ -53,8 +47,8 @@ def _headers(): # Check for additional numbered pairs. index = 2 while True: - candidate_api = _Credential(os.getenv(f"DD_API_KEY_ONBOARDING_{index}", "")) - candidate_app = _Credential(os.getenv(f"DD_APP_KEY_ONBOARDING_{index}", "")) + candidate_api = os.getenv(f"DD_API_KEY_ONBOARDING_{index}") + candidate_app = os.getenv(f"DD_APP_KEY_ONBOARDING_{index}") if candidate_api and candidate_app: pairs.append((candidate_api, candidate_app)) index += 1 @@ -123,6 +117,9 @@ def _make_request( With the defaults, this will retry for approximately 5 minutes. """ + # Omit authentication headers from pytest failure reports. + __tracebackhide__ = True + start_time = time.perf_counter() for _attempt in range(max_retries): try: @@ -181,6 +178,9 @@ def _validate_profiler_response(json: dict): def _query_for_profile(runtime_id: str): + # This caller also retains authentication headers while the request runs. + __tracebackhide__ = True + url = f"{API_HOST}/api/unstable/profiles/list" headers = _headers() headers["Content-Type"] = "application/json" From a9a9a4b3aea763f1a09600c07fcf1431c1e42d54 Mon Sep 17 00:00:00 2001 From: "damian.malysza" Date: Wed, 28 Oct 2026 03:09:49 +0000 Subject: [PATCH 3/3] Remove new onboarding unit tests Keep the native pytest traceback omission scoped to the existing credential-retaining helpers and validate with existing repository checks. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- .../test_onboarding_backend_credentials.py | 69 ------------------- 1 file changed, 69 deletions(-) delete mode 100644 tests/test_the_test/test_onboarding_backend_credentials.py diff --git a/tests/test_the_test/test_onboarding_backend_credentials.py b/tests/test_the_test/test_onboarding_backend_credentials.py deleted file mode 100644 index 5844236baef..00000000000 --- a/tests/test_the_test/test_onboarding_backend_credentials.py +++ /dev/null @@ -1,69 +0,0 @@ -from datetime import UTC, datetime -import json - -import requests - -from utils import pytest -from utils.onboarding import backend_interface - - -pytestmark = pytest.mark.scenario("TEST_THE_TEST") - - -@pytest.mark.parametrize("profile", [False, True]) -def test_backend_failure_reports_omit_authentication_headers(monkeypatch: pytest.MonkeyPatch, *, profile: bool) -> None: - # Only backend frames belong in the failure rendering under test. - __tracebackhide__ = True - monkeypatch.setenv("DD_API_KEY_ONBOARDING", "synthetic-onboarding-api-key") - monkeypatch.setenv("DD_APP_KEY_ONBOARDING", "synthetic-onboarding-app-key") - monkeypatch.delenv("DD_API_KEY_ONBOARDING_2", raising=False) - monkeypatch.delenv("DD_APP_KEY_ONBOARDING_2", raising=False) - prepared_requests: list[requests.PreparedRequest] = [] - - def backend_response( - _session: requests.Session, request: requests.PreparedRequest, **_kwargs: object - ) -> requests.Response: - prepared_requests.append(request) - response = requests.Response() - response.status_code = 503 - if profile and len(prepared_requests) == 1: - response.status_code = 200 - response._content = json.dumps( # noqa: SLF001 - { - "trace": { - "root_id": "root", - "spans": { - "root": { - "start": datetime.now(tz=UTC).timestamp(), - "meta": {"runtime-id": "synthetic-runtime-id"}, - } - }, - } - } - ).encode() - return response - - monkeypatch.setattr(requests.Session, "send", backend_response) - times = iter([0, 0, 400] if profile else [0, 400]) - monkeypatch.setattr(backend_interface.time, "perf_counter", lambda: next(times)) - - with pytest.raises(TimeoutError) as failure: - backend_interface.wait_backend_trace_id("synthetic-trace-id", profile=profile) - - request = prepared_requests[-1] - assert request.headers["DD-API-KEY"] == "synthetic-onboarding-api-key" - assert request.headers["DD-APPLICATION-KEY"] == "synthetic-onboarding-app-key" - if profile: - assert request.headers["Content-Type"] == "application/json" - - failure_output = str(failure.getrepr(showlocals=True)) - assert "synthetic-onboarding-api-key" not in failure_output - assert "synthetic-onboarding-app-key" not in failure_output - assert "DD-API-KEY" not in failure_output - assert "DD-APPLICATION-KEY" not in failure_output - assert "wait_backend_trace_id" in failure_output - assert "synthetic-trace-id" in failure_output - assert "Reached overall timeout" in failure_output - assert ("post" if profile else "get") in failure_output - assert request.url is not None - assert request.url in failure_output