From eaefea2e107d9001a48b2d852bcffbaffd24be50 Mon Sep 17 00:00:00 2001 From: dudik <8845578+dudikeleti@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:41:57 +0200 Subject: [PATCH 1/6] [debugger] Test that updating one APM_TRACING config keeps the others applied When only one APM_TRACING config changes, the unchanged configs are still listed in client_configs and must stay applied. The new test enables dynamic instrumentation from a wildcard config, then changes a single setting of a service config, and expects probes to keep emitting. It is enabled for all libraries on purpose, to see which ones drop the unchanged config. --- .../test_debugger_inproduct_enablement.py | 97 ++++++++++++++++--- 1 file changed, 84 insertions(+), 13 deletions(-) diff --git a/tests/debugger/test_debugger_inproduct_enablement.py b/tests/debugger/test_debugger_inproduct_enablement.py index 42084d6791f..1fc41fca56a 100644 --- a/tests/debugger/test_debugger_inproduct_enablement.py +++ b/tests/debugger/test_debugger_inproduct_enablement.py @@ -3,7 +3,7 @@ # Copyright 2021 Datadog, Inc. import tests.debugger.utils as debugger -from utils import context, features, logger, scenarios, slow +from utils import context, features, logger, remote_config, scenarios, slow, weblog import json import time @@ -11,26 +11,25 @@ DEFAULT_ENVVARS = { "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", } +LOG_PROBE_TEMPLATE = """ +{ + "version": 0, + "where": { + "typeName": null, + "sourceFile": "ACTUAL_SOURCE_FILE", + "lines": ["20"] + } +} +""" @features.debugger_inproduct_enablement @scenarios.debugger_inproduct_enablement class Test_Debugger_InProduct_Enablement_Dynamic_Instrumentation(debugger.BaseDebuggerTest): ############ dynamic instrumentation ############ - _probe_template = """ - { - "version": 0, - "where": { - "typeName": null, - "sourceFile": "ACTUAL_SOURCE_FILE", - "lines": ["20"] - } - } - """ - def setup_inproduct_enablement_di(self): def _send_config(*, enabled: bool | None = None, reset: bool = True): - probe = json.loads(self._probe_template) + probe = json.loads(LOG_PROBE_TEMPLATE) probe["id"] = debugger.generate_probe_id("log") self.set_probes([probe]) @@ -263,3 +262,75 @@ def test_code_origin_enabled_by_default(self): self.assert_all_weblog_responses_ok() assert self.code_origin_enabled_by_default, "Expected code origin enabled by default" + + +@features.debugger_inproduct_enablement +@scenarios.debugger_inproduct_enablement +class Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update(debugger.BaseDebuggerTest): + """Updating one APM_TRACING config must not drop the other active configs. + + An unchanged file keeps its hash, so libraries skip it, but it is still listed + in client_configs and must stay applied. + """ + + _WILDCARD_CONFIG_PATH = "datadog/2/APM_TRACING/inproduct-multiconfig-wildcard/config" + _SERVICE_CONFIG_PATH = "datadog/2/APM_TRACING/inproduct-multiconfig-service/config" + + def _set_apm_tracing_config(self, path: str, settings: dict, *, service_name: str, env: str) -> None: + lib_config = {"library_language": "all", "library_version": "latest", "tracing_enabled": True} | settings + config = { + "schema_version": "v1.0.0", + "action": "enable", + "lib_config": lib_config, + "service_target": {"service": service_name, "env": env}, + } + if remote_config.library_supports_sdk_configuration(): + config = remote_config.to_sdk_config_payload(config) + remote_config.tracer_rc_state.set_config(path, config) + + def _apply_with_new_probe_and_check_emitting(self) -> bool: + probe = json.loads(LOG_PROBE_TEMPLATE) + probe["id"] = debugger.generate_probe_id("log") + self.set_probes([probe]) + remote_config.tracer_rc_state.set_config( + f"datadog/2/LIVE_DEBUGGING/logProbe_{probe['id']}/config", self.probe_definitions[0] + ) + + self.rc_states.append(remote_config.tracer_rc_state.apply()) + + # PHP tracer requires requests to /debugger/* to process RC and resolve probe hooks. + if context.library == "php": + weblog.get(f"/debugger/init?probes={probe['id']}") + + self.send_weblog_request("/debugger/log", reset=False) + return self.wait_for_all_probes(statuses=["EMITTING"], timeout=TIMEOUT) + + def setup_inproduct_enablement_multiconfig_partial_update(self): + self.initialize_weblog_remote_config() + self.weblog_responses = [] + self.rc_states = [] + remote_config.tracer_rc_state.reset().apply() + + # The wildcard config enables dynamic instrumentation, the service config only sets code origin. + self._set_apm_tracing_config( + self._WILDCARD_CONFIG_PATH, {"dynamic_instrumentation_enabled": True}, service_name="*", env="*" + ) + self._set_apm_tracing_config( + self._SERVICE_CONFIG_PATH, {"code_origin_enabled": True}, service_name="weblog", env="system-tests" + ) + self.di_enabled = self._apply_with_new_probe_and_check_emitting() + + # Change a single setting of the service config. The wildcard config is resent unchanged. + self._set_apm_tracing_config( + self._SERVICE_CONFIG_PATH, {"code_origin_enabled": False}, service_name="weblog", env="system-tests" + ) + self.di_kept_enabled = self._apply_with_new_probe_and_check_emitting() + + def test_inproduct_enablement_multiconfig_partial_update(self): + self.assert_rc_state_not_error() + self.assert_all_weblog_responses_ok() + + assert self.di_enabled, ( + "Expected probes to be emitting after the wildcard config enabled dynamic instrumentation" + ) + assert self.di_kept_enabled, "Expected probes to keep emitting after updating only the service config" From 28a3f79d38f198da07070986e6472403b0d86f3c Mon Sep 17 00:00:00 2001 From: dudik <8845578+dudikeleti@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:10:10 +0200 Subject: [PATCH 2/6] [debugger] Declare the multiconfig partial update test per library Follow the weblogs of the in-product DI test and gate each library on the first release with APM_TRACING multiconfig support. .NET is a known bug below 3.55.0 (DEBUG-4637). --- manifests/dotnet.yml | 3 +++ manifests/golang.yml | 1 + manifests/java.yml | 10 ++++++++++ manifests/nodejs.yml | 4 ++++ manifests/php.yml | 5 +++++ manifests/python.yml | 4 ++++ manifests/ruby.yml | 6 ++++++ 7 files changed, 33 insertions(+) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 4f5a6a997f3..16db1f6bac9 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -731,6 +731,9 @@ manifest: tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Exception_Replay: v3.29.0 ? tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Exception_Replay::test_inproduct_enablement_exception_replay_apm_multiconfig : bug (DEBUG-4637) + tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update: # DEBUG-4637: fix remote enablement (#9344) + - declaration: bug (DEBUG-4637) + component_version: '<3.55.0' tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: v2.50.0 tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction::test_pii_redaction_method_full: - declaration: missing_feature (keywords are not fully redacted) diff --git a/manifests/golang.yml b/manifests/golang.yml index ef15b04e597..75c34a19b1b 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1630,6 +1630,7 @@ manifest: tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Code_Origin_Default_On: irrelevant (Scoped to dotnet rollout in this PR) tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Dynamic_Instrumentation: missing_feature tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Exception_Replay: missing_feature + tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update: missing_feature tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: missing_feature (feature not implemented) tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction_Excluded_Identifiers: missing_feature (feature not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets: v2.2.3 diff --git a/manifests/java.yml b/manifests/java.yml index a2adfa974fa..fc932df1b66 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3197,6 +3197,16 @@ manifest: uds-spring-boot: v1.60.1 # TODO: a lower version might be supported ? tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Exception_Replay::test_inproduct_enablement_exception_replay_apm_multiconfig : bug (DEBUG-4736) + tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update: + - weblog_declaration: + "*": missing_feature + spring-boot: v1.54.0 + spring-boot-jetty: v1.54.0 + spring-boot-openliberty: v1.54.0 + spring-boot-payara: v1.54.0 + spring-boot-undertow: v1.54.0 + spring-boot-wildfly: v1.54.0 + uds-spring-boot: v1.54.0 tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: - weblog_declaration: "*": missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index e14eec6d1c1..8715495221b 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1736,6 +1736,10 @@ manifest: "*": *ref_5_83_0 nextjs: irrelevant tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Exception_Replay: missing_feature + tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update: + - weblog_declaration: + "*": *ref_5_83_0 + nextjs: irrelevant tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: - weblog_declaration: "*": *ref_5_39_0 diff --git a/manifests/php.yml b/manifests/php.yml index 049e49fb250..2b3d42fa516 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1174,6 +1174,11 @@ manifest: laravel11x: incomplete_test_app symfony7x: incomplete_test_app tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Exception_Replay: missing_feature (blocked by exception replay throwable gap) + tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update: + - weblog_declaration: + "*": v1.20.0 + laravel11x: incomplete_test_app + symfony7x: incomplete_test_app tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: - weblog_declaration: "*": v1.19.0 diff --git a/manifests/python.yml b/manifests/python.yml index 7848874ea57..8b163a5653b 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1340,6 +1340,10 @@ manifest: component_version: <4.9.0 ? tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Exception_Replay::test_inproduct_enablement_exception_replay_apm_multiconfig : missing_feature + tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update: + - weblog_declaration: + "*": missing_feature + *flask : v3.15.0 tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: - weblog_declaration: "*": missing_feature diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 2eb59bd1c7d..97771539af2 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1897,6 +1897,12 @@ manifest: rails80: v2.39.0-dev uds-rails: v2.39.0-dev tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Exception_Replay: missing_feature + tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update: + - weblog_declaration: + "*": missing_feature + rails72: v2.42.0 + rails80: v2.42.0 + uds-rails: v2.42.0 tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: - weblog_declaration: "*": irrelevant From f6501737af03d4894d1cbd47679c6585678bba6e Mon Sep 17 00:00:00 2001 From: dudik <8845578+dudikeleti@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:59:27 +0200 Subject: [PATCH 3/6] Bump .NET multiconfig partial update fix version to 3.56.0 Co-authored-by: Cursor --- manifests/dotnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index cc3b123117f..15f0b6b31f5 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -745,7 +745,7 @@ manifest: : bug (DEBUG-4637) tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update: # DEBUG-4637: fix remote enablement (#9344) - declaration: bug (DEBUG-4637) - component_version: '<3.55.0' + component_version: '<3.56.0' tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: v2.50.0 tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction::test_pii_redaction_method_full: - declaration: missing_feature (keywords are not fully redacted) From 9d3a17f1931c4061ec037c1221100e123d9b3923 Mon Sep 17 00:00:00 2001 From: dudik <8845578+dudikeleti@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:49:16 +0100 Subject: [PATCH 4/6] Gate .NET multiconfig partial update test on 3.57.0 DataDog/dd-trace-dotnet#9344 merged after 3.56.0 was released, so the first release with the fix is 3.57.0. Co-Authored-By: Claude Opus 5.5 --- manifests/dotnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 8aff1e5296c..b5b72e20f51 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -747,7 +747,7 @@ manifest: : bug (DEBUG-4637) tests/debugger/test_debugger_inproduct_enablement.py::Test_Debugger_InProduct_Enablement_Multiconfig_Partial_Update: # DEBUG-4637: fix remote enablement (#9344) - declaration: bug (DEBUG-4637) - component_version: '<3.56.0' + component_version: '<3.57.0' tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: v2.50.0 tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction::test_pii_redaction_method_full: - declaration: missing_feature (keywords are not fully redacted) From 3b4483f860564cb42ddfed8c2638f7eb6f1c89f3 Mon Sep 17 00:00:00 2001 From: dudik <8845578+dudikeleti@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:46:47 +0100 Subject: [PATCH 5/6] Keep the .NET LogProbe action from being inlined ASP.NET Core MVC calls actions through a compiled expression tree, and the JIT inlines the 12-byte LogProbe into it on the first /debugger/log request. Earlier in-product enablement tests call /debugger/log while DI is off, so after remote enablement the tracer ReJITs LogProbe but requests keep running the inlined copy, and the line probe never emits. Mark LogProbe NoInlining on its existing attribute line so no line number in the file moves. Co-Authored-By: Claude Opus 5.5 --- .../docker/dotnet/weblog/Controllers/DebuggerController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs b/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs index d159e702521..1f936d3b644 100644 --- a/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs +++ b/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs @@ -14,7 +14,7 @@ namespace weblog public class DebuggerController : Controller { [HttpGet("log")] - [Consumes("application/json", "application/xml")] + [Consumes("application/json", "application/xml"), System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public IActionResult LogProbe() { return Content($"Log probe"); From 20d3d0e21648bf25199552c9965d0f6595c65e80 Mon Sep 17 00:00:00 2001 From: dudik <8845578+dudikeleti@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:55:22 +0100 Subject: [PATCH 6/6] Share the PHP probe init call, require EMITTING in the partial update test Address review feedback on the new partial update test: - The PHP /debugger/init request now lives in BaseDebuggerTest.resolve_php_probes, used by send_rc_probes, send_rc_apm_tracing_and_probes and the new test, so the test has no library branch. - wait_for_all_probes also returns when a probe reports ERROR, so the test now collects the probe diagnostics and requires both probes to have reached EMITTING. Co-Authored-By: Claude Opus 5.5 --- .../test_debugger_inproduct_enablement.py | 16 +++++++++++----- tests/debugger/utils.py | 9 ++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/debugger/test_debugger_inproduct_enablement.py b/tests/debugger/test_debugger_inproduct_enablement.py index 1fc41fca56a..aac2657b498 100644 --- a/tests/debugger/test_debugger_inproduct_enablement.py +++ b/tests/debugger/test_debugger_inproduct_enablement.py @@ -3,7 +3,7 @@ # Copyright 2021 Datadog, Inc. import tests.debugger.utils as debugger -from utils import context, features, logger, remote_config, scenarios, slow, weblog +from utils import context, features, logger, remote_config, scenarios, slow import json import time @@ -292,23 +292,23 @@ def _apply_with_new_probe_and_check_emitting(self) -> bool: probe = json.loads(LOG_PROBE_TEMPLATE) probe["id"] = debugger.generate_probe_id("log") self.set_probes([probe]) + self.step_probe_ids.append(probe["id"]) remote_config.tracer_rc_state.set_config( f"datadog/2/LIVE_DEBUGGING/logProbe_{probe['id']}/config", self.probe_definitions[0] ) self.rc_states.append(remote_config.tracer_rc_state.apply()) - - # PHP tracer requires requests to /debugger/* to process RC and resolve probe hooks. - if context.library == "php": - weblog.get(f"/debugger/init?probes={probe['id']}") + self.resolve_php_probes() self.send_weblog_request("/debugger/log", reset=False) + # An ERROR status also ends this wait, so the test checks that each probe reached EMITTING return self.wait_for_all_probes(statuses=["EMITTING"], timeout=TIMEOUT) def setup_inproduct_enablement_multiconfig_partial_update(self): self.initialize_weblog_remote_config() self.weblog_responses = [] self.rc_states = [] + self.step_probe_ids: list[str] = [] remote_config.tracer_rc_state.reset().apply() # The wildcard config enables dynamic instrumentation, the service config only sets code origin. @@ -330,7 +330,13 @@ def test_inproduct_enablement_multiconfig_partial_update(self): self.assert_rc_state_not_error() self.assert_all_weblog_responses_ok() + self.collect() + statuses = {probe_id: diagnostics["status"] for probe_id, diagnostics in self.probe_diagnostics.items()} + first_status, second_status = (statuses.get(probe_id) for probe_id in self.step_probe_ids) + assert self.di_enabled, ( "Expected probes to be emitting after the wildcard config enabled dynamic instrumentation" ) + assert first_status == "EMITTING", f"Expected the first probe to be EMITTING, got {first_status}" assert self.di_kept_enabled, "Expected probes to keep emitting after updating only the service config" + assert second_status == "EMITTING", f"Expected the second probe to be EMITTING, got {second_status}" diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index 50db8eaf538..b73582505b0 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -391,6 +391,9 @@ def send_rc_probes(self, *, reset: bool = True) -> None: remote_config.send_debugger_command(probes=self.probe_definitions, version=BaseDebuggerTest._rc_version) ) + self.resolve_php_probes() + + def resolve_php_probes(self) -> None: # PHP tracer requires requests to /debugger/* to process RC and resolve probe hooks. # Pass probe IDs so the PHP endpoint polls until they appear in the loaded RC state. if context.library == "php": @@ -472,11 +475,7 @@ def send_rc_apm_tracing_and_probes( ) ) - # PHP tracer requires requests to /debugger/* to process RC before the test request. - # Pass probe IDs so the PHP endpoint polls until they appear in the loaded RC state. - if context.library == "php": - probe_ids = ",".join(p["id"] for p in self.probe_definitions if "id" in p) - weblog.get(f"/debugger/init?probes={probe_ids}") + self.resolve_php_probes() def send_rc_symdb(self, *, reset: bool = True) -> None: BaseDebuggerTest._rc_version += 1