From ba2dba297e61a571c3d54ed3de470feb996ebaed Mon Sep 17 00:00:00 2001 From: grant-seltzerrichman_ddog Date: Thu, 23 Jul 2026 18:04:05 +0000 Subject: [PATCH 01/13] test(debugger): add snapshot-correlation gate and Go weblog endpoints --- manifests/dotnet.yml | 4 + manifests/golang.yml | 4 + manifests/java.yml | 4 + manifests/nodejs.yml | 4 + manifests/php.yml | 4 + manifests/python.yml | 4 + manifests/ruby.yml | 4 + tests/debugger/test_debugger_probe_budgets.py | 34 ++++ .../test_debugger_snapshot_correlation.py | 145 ++++++++++++++++++ tests/debugger/utils.py | 4 + .../probe_snapshot_log_correlation.json | 38 +++++ .../probe_snapshot_log_correlation_loop.json | 15 ++ utils/_features.py | 13 ++ utils/build/docker/golang/app/chi/debugger.go | 34 +++- utils/build/docker/golang/app/chi/main.go | 5 + .../build/docker/golang/app/echo/debugger.go | 34 +++- utils/build/docker/golang/app/echo/main.go | 6 + utils/build/docker/golang/app/gin/debugger.go | 34 +++- utils/build/docker/golang/app/gin/main.go | 5 + .../app/net-http-orchestrion/debugger.go | 34 +++- .../golang/app/net-http-orchestrion/main.go | 5 + .../docker/golang/app/net-http/debugger.go | 34 +++- .../build/docker/golang/app/net-http/main.go | 5 + 23 files changed, 468 insertions(+), 5 deletions(-) create mode 100644 tests/debugger/test_debugger_snapshot_correlation.py create mode 100644 tests/debugger/utils/probes/probe_snapshot_log_correlation.json create mode 100644 tests/debugger/utils/probes/probe_snapshot_log_correlation_loop.json diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 7db6bf29e45..2b16c30cf23 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -763,6 +763,7 @@ manifest: weblog: [uds, poc] - declaration: bug (DEBUG-4746) excluded_weblog: [uds, poc] + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: irrelevant (Dotnet emits evaluation errors as span tags, not error snapshots to the debugger endpoint) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: v2.53.0 tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_default_max_length: @@ -785,6 +786,9 @@ manifest: - declaration: bug (DEBUG-5101) component_version: '<3.45.0' tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses: v2.53.0 + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: v2.53.0 tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_event_metadata: v3.44.0 tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_symdb_upload: # Modified by easy win activation script diff --git a/manifests/golang.yml b/manifests/golang.yml index d498b6a39ab..11b0a6ce6b2 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1667,6 +1667,7 @@ manifest: tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: missing_feature (feature not implemented) tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction_Excluded_Identifiers: missing_feature (feature not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets: v2.2.3 + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: missing_feature (span decoration probes not yet implemented for Go) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: v2.2.3 tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_default_max_collection_size: missing_feature (no /debugger/snapshot/limits endpoint) @@ -1703,6 +1704,9 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_metric_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_decoration_method_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_method_status: missing_feature (Not yet implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: v2.2.3 # TODO(andrei): un-skip when datadog-agent v7.79.0 is released (Go's symdb upload comes from the agent, not from dd-trace-go) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_event_metadata: # TODO: a lower version might be supported diff --git a/manifests/java.yml b/manifests/java.yml index bd3559e9466..86eadaed9e3 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3316,6 +3316,7 @@ manifest: spring-boot-undertow: v1.38.0 spring-boot-wildfly: v1.38.0 uds-spring-boot: v1.38.0 + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: bug (DEBUG-3797) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: - weblog_declaration: @@ -3396,6 +3397,9 @@ manifest: spring-boot-undertow: v1.38.0 spring-boot-wildfly: v1.38.0 uds-spring-boot: v1.38.0 + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: - weblog_declaration: "*": missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 94b6334410c..94e3aa27549 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1788,6 +1788,7 @@ manifest: fastify: *ref_5_46_0 uds-express4: *ref_5_46_0 tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_log_line_capture_expression_budgets: '>=6.16.0 || ^5.127.0' + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: missing_feature (Method probes are not supported by Node.js) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: - weblog_declaration: @@ -1846,6 +1847,9 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Line_Probe_Statuses::test_metric_line_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Line_Probe_Statuses::test_span_decoration_line_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses: missing_feature + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: missing_feature (feature not implented) tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: - weblog_declaration: diff --git a/manifests/php.yml b/manifests/php.yml index 64d0e027b3d..4322a86fd62 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1190,6 +1190,7 @@ manifest: laravel11x: incomplete_test_app symfony7x: incomplete_test_app tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_log_line_capture_expression_budgets: missing_feature + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: irrelevant tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: missing_feature tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) @@ -1219,6 +1220,9 @@ manifest: apache-mod-8.0: v1.8.3 - declaration: flaky (DEBUG-3814) component_version: '>1.8.3' + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: missing_feature tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: - weblog_declaration: diff --git a/manifests/python.yml b/manifests/python.yml index 22b9f37c023..cf5b14fc019 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2119,6 +2119,7 @@ manifest: "*": missing_feature *flask : v2.11.0 tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_log_line_capture_expression_budgets: bug (DEBUG-5743) + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented; tracer uses a time-based budget) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: irrelevant (Python emits evaluation errors as span tags, not error snapshots to the debugger endpoint) tests/debugger/test_debugger_probe_snapshot.py: - weblog_declaration: @@ -2178,6 +2179,9 @@ manifest: component_version: 2.16.0 - declaration: bug (DEBUG-3127) component_version: 2.16.1 + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: - weblog_declaration: "*": missing_feature diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 28350f8caea..d2f64dc6e95 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1998,6 +1998,7 @@ manifest: rails80: missing_feature (feature not implemented) uds-rails: missing_feature (feature not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_log_line_budgets: missing_feature (Not yet implemented) + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: - weblog_declaration: "*": irrelevant @@ -2062,6 +2063,9 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_metric_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_decoration_method_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_method_status: missing_feature (Not yet implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: - weblog_declaration: "*": irrelevant diff --git a/tests/debugger/test_debugger_probe_budgets.py b/tests/debugger/test_debugger_probe_budgets.py index e6977660247..2e01d47ba98 100644 --- a/tests/debugger/test_debugger_probe_budgets.py +++ b/tests/debugger/test_debugger_probe_budgets.py @@ -168,3 +168,37 @@ def test_span_probe_expression_budgets(self): f"{self.total_request_time:.1f}s), got {total_error_entries}. " f"Span probe expression error logging should be rate-limited." ) + + def setup_per_span_budget(self): + self.initialize_weblog_remote_config() + + language = self.get_tracer()["language"] + + probes = [] + for method in ("CorrelationLoopBody", "CorrelationLoopSibling"): + probe = debugger.read_probes("probe_snapshot_log_correlation_loop")[0] + probe["id"] = debugger.generate_probe_id("log") + self._rewrite_where_for_lines([probe], self.method_and_language_to_line_number(method, language)) + probes.append(probe) + + self.set_probes(probes) + self.send_rc_probes() + self.wait_for_all_probes(statuses=["INSTALLED"]) + + # The loop sleeps one second per iteration, so the time-based budget above would emit more + # than one snapshot while a span-scoped budget emits exactly one. + self.send_weblog_request("/debugger/correlation/loop/3") + + self.wait_for_all_probes(statuses=["EMITTING"]) + self.wait_for_all_snapshots() + + def test_per_span_budget(self): + """A probe in a loop emits one snapshot per span and does not starve a sibling probe.""" + self._assert() + + # A single request is a single span, so every probe must have emitted exactly once. + for probe_id in self.probe_ids: + snapshots = [s for s in self.probe_snapshots.get(probe_id, []) if s.get("debugger", {}).get("snapshot")] + assert len(snapshots) == 1, ( + f"probe {probe_id} emitted {len(snapshots)} snapshots for one span, expected exactly 1" + ) diff --git a/tests/debugger/test_debugger_snapshot_correlation.py b/tests/debugger/test_debugger_snapshot_correlation.py new file mode 100644 index 00000000000..a3c60bc64fb --- /dev/null +++ b/tests/debugger/test_debugger_snapshot_correlation.py @@ -0,0 +1,145 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2021 Datadog, Inc. + +import tests.debugger.utils as debugger + +from utils import scenarios, features, rfc, slow + + +@rfc( + "https://docs.google.com/document/d/17BQ1cEcJuumpMWMtBItIx9x_akG99dzjBA1NoaBFuBE/edit?tab=t.4l0zjxovcnz3#heading=h.kejjaf501thi" +) +@features.debugger_snapshot_correlation +@scenarios.debugger_probes_snapshot +@slow +class Test_Debugger_Coordinated_Sampling(debugger.BaseDebuggerTest): + """The sampling decision is made once per trace, so a trace emits its whole probe chain or none of it. + + Correlation rides on the dd.trace_id already in the snapshot envelope. The endpoint spaces its + probed call sites in time, so today's independent per-probe rate limiters land in different + windows and emit partial chains, which is what makes the assertion discriminating. + """ + + _INVOCATIONS = 30 + + def setup_coordinated_per_trace_sampling(self): + self.initialize_weblog_remote_config() + + probes = debugger.read_probes("probe_snapshot_log_correlation") + for probe in probes: + probe["id"] = debugger.generate_probe_id("log") + # This is the per-probe time limiter, not a per-trace rate: it is set only to put the + # chain under sampling pressure so that an uncoordinated implementation splits it. + # Swap it for the per-trace rate once one exists. + probe["sampling"] = {"snapshotsPerSecond": 1} + self.set_probes(probes) + + self.send_rc_probes() + if not self.wait_for_all_probes(statuses=["INSTALLED"], timeout=60): + self.setup_failures.append("Probes did not reach INSTALLED status") + return + + # Each request is one trace, so it is one sampling unit. + for i in range(self._INVOCATIONS): + self.send_weblog_request("/debugger/correlation", reset=(i == 0)) + + self.wait_for_all_probes(statuses=["EMITTING"]) + self.wait_for_all_snapshots() + + def test_coordinated_per_trace_sampling(self): + """Every trace emits either all of its probes or none of them.""" + self.collect() + + self.assert_setup_ok() + self.assert_rc_state_not_error() + self.assert_all_weblog_responses_ok() + + chain = set(self.probe_ids) + + probes_by_trace: dict[str, set[str]] = {} + for probe_id in self.probe_ids: + for snapshot in self.probe_snapshots.get(probe_id, []): + trace_id = snapshot.get("dd", {}).get("trace_id") + assert trace_id, f"snapshot for probe {probe_id} is missing dd.trace_id on the wire" + probes_by_trace.setdefault(trace_id, set()).add(probe_id) + + full = [trace_id for trace_id, probes in probes_by_trace.items() if probes == chain] + partial = [trace_id for trace_id, probes in probes_by_trace.items() if probes != chain] + + assert not partial, ( + f"partial probe chains were emitted, the sampling decision is not made once per trace: {partial}" + ) + assert full, "no trace emitted the full probe chain" + + +# Follow-on work, split out of feature 564 (which is coordinated sampling via trace_id only). There +# is no dashboard feature for these yet, and they are simply not implemented (not blocked on any +# schema negotiation), so they are not reported. +@features.not_reported +@scenarios.debugger_probes_snapshot +@slow +class Test_Debugger_Snapshot_Correlation_Fields(debugger.BaseDebuggerTest): + """Identity fields that make a snapshot correlatable: the per-process runtime_id in the envelope, + and a generation token that disambiguates a reused thread id. + """ + + def _setup(self): + self.initialize_weblog_remote_config() + + probes = debugger.read_probes("probe_snapshot_log_correlation") + for probe in probes: + probe["id"] = debugger.generate_probe_id("log") + self.set_probes(probes) + + self.send_rc_probes() + if not self.wait_for_all_probes(statuses=["INSTALLED"], timeout=60): + self.setup_failures.append("Probes did not reach INSTALLED status") + return + + self.send_weblog_request("/debugger/correlation") + self.wait_for_all_probes(statuses=["EMITTING"]) + self.wait_for_all_snapshots() + + def _assert(self): + self.collect() + + self.assert_setup_ok() + self.assert_rc_state_not_error() + self.assert_all_weblog_responses_ok() + + def setup_runtime_id_in_envelope(self): + self._setup() + + def test_runtime_id_in_envelope(self): + """Every snapshot carries the same non-empty runtime_id, since all come from one process.""" + self._assert() + + runtime_ids = set() + for probe_id in self.probe_ids: + snapshots = self.probe_snapshots.get(probe_id, []) + assert snapshots, f"no snapshot was received for probe {probe_id}" + for snapshot in snapshots: + # The envelope does not carry it today, so tolerate the root or the "dd" object. + runtime_id = snapshot.get("runtime_id") or snapshot.get("dd", {}).get("runtime_id") + assert runtime_id, f"snapshot for probe {probe_id} is missing a non-empty runtime_id" + runtime_ids.add(runtime_id) + + assert len(runtime_ids) == 1, f"expected one runtime_id across the run, got {runtime_ids}" + + def setup_generation_token(self): + self._setup() + + def test_generation_token(self): + """Every snapshot carries a generation token alongside thread_id.""" + self._assert() + + for probe_id in self.probe_ids: + snapshots = self.probe_snapshots.get(probe_id, []) + assert snapshots, f"no snapshot was received for probe {probe_id}" + for snapshot in snapshots: + context = snapshot.get("logger", {}) + assert "thread_id" in context, f"snapshot for probe {probe_id} is missing thread_id" + assert context.get("generation") not in (None, ""), ( + f"snapshot for probe {probe_id} is missing the execution-context generation token" + ) diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index e98dfa5bd35..f60cfba3294 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -213,6 +213,8 @@ def method_and_language_to_line_number(self, method: str, language: str) -> list "Nulls": {"java": [132], "dotnet": [129], "python": [138], "ruby": [192], "nodejs": [126]}, "SnapshotLimits": {"java": [155], "python": [174], "nodejs": [136], "ruby": [233], "dotnet": [152]}, "CaptureTimeout": {"java": [174], "nodejs": [157], "dotnet": [173]}, + "CorrelationLoopBody": {"golang": [144]}, + "CorrelationLoopSibling": {"golang": [147]}, } return definitions.get(method, {}).get(language, []) @@ -355,6 +357,8 @@ def __get_probe_type(probe_id: str): "20": "main.(*DebuggerController).logProbe", "71": "main.(*DebuggerController).expression", "117": "main.(*DebuggerController).budgetStep", + "144": "main.(*DebuggerController).correlationLoop", + "147": "main.(*DebuggerController).correlationLoop", } line = str(probe["where"]["lines"][0]) if line in golang_line_to_method: diff --git a/tests/debugger/utils/probes/probe_snapshot_log_correlation.json b/tests/debugger/utils/probes/probe_snapshot_log_correlation.json new file mode 100644 index 00000000000..df0803049b4 --- /dev/null +++ b/tests/debugger/utils/probes/probe_snapshot_log_correlation.json @@ -0,0 +1,38 @@ +[ + { + "language": "", + "type": "", + "id": "", + "version": 0, + "where": { + "typeName": "ACTUAL_TYPE_NAME", + "methodName": "Correlation", + "sourceFile": null + }, + "captureSnapshot": true + }, + { + "language": "", + "type": "", + "id": "", + "version": 0, + "where": { + "typeName": "ACTUAL_TYPE_NAME", + "methodName": "CorrelationMiddle", + "sourceFile": null + }, + "captureSnapshot": true + }, + { + "language": "", + "type": "", + "id": "", + "version": 0, + "where": { + "typeName": "ACTUAL_TYPE_NAME", + "methodName": "CorrelationLeaf", + "sourceFile": null + }, + "captureSnapshot": true + } +] diff --git a/tests/debugger/utils/probes/probe_snapshot_log_correlation_loop.json b/tests/debugger/utils/probes/probe_snapshot_log_correlation_loop.json new file mode 100644 index 00000000000..a547c04ed5f --- /dev/null +++ b/tests/debugger/utils/probes/probe_snapshot_log_correlation_loop.json @@ -0,0 +1,15 @@ +[ + { + "language": "", + "type": "", + "id": "", + "version": 0, + "where": { + "typeName": null, + "sourceFile": "ACTUAL_SOURCE_FILE", + "lines": [] + }, + "captureSnapshot": true, + "tags": [] + } +] diff --git a/utils/_features.py b/utils/_features.py index 4a5ce48125b..ea347ab3a9e 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2335,6 +2335,19 @@ def debugger_probe_budgets(test_object): """ return _mark_test_object(test_object, feature_id=368, owner=_Owner.debugger) + @staticmethod + def debugger_snapshot_correlation(test_object): + """Coordinated per-trace snapshot sampling (Live Debugger) + + The sampling decision is made once per trace and applies to every probe in that trace, + with snapshots correlated by the dd.trace_id already carried in the snapshot envelope (no + snapshot payload change). Runtime-id-in-envelope and the execution-context generation + token are separate follow-on work and are not part of this feature. + + https://feature-parity.us1.prod.dog/#/?feature=564 + """ + return _mark_test_object(test_object, feature_id=564, owner=_Owner.debugger) + @staticmethod def debugger_method_probe(test_object): """Method-level Probes diff --git a/utils/build/docker/golang/app/chi/debugger.go b/utils/build/docker/golang/app/chi/debugger.go index 37a36a8fb80..f02170c58cc 100644 --- a/utils/build/docker/golang/app/chi/debugger.go +++ b/utils/build/docker/golang/app/chi/debugger.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "runtime" + "time" ) // The `debugger` feature allows attachment to specific lines of code. @@ -13,7 +14,6 @@ import ( type DebuggerController struct{} -// Dummy line // Dummy line func (d *DebuggerController) logProbe(w http.ResponseWriter, r *http.Request) { // Dummy line @@ -116,3 +116,35 @@ func (d *DebuggerController) budgets(w http.ResponseWriter, r *http.Request, loo func (d *DebuggerController) budgetStep(i, loops int) int { return i + loops // loops is referenced so the capture-expression probe can read it at this line. } + +//go:noinline +func (d *DebuggerController) correlation(w http.ResponseWriter, r *http.Request) { + result := d.correlationMiddle() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + runtime.KeepAlive(result) + w.Write([]byte(fmt.Sprintf("Correlation %d", result))) +} + +//go:noinline +func (d *DebuggerController) correlationMiddle() int { + result := d.correlationLeaf() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + return result +} + +//go:noinline +func (d *DebuggerController) correlationLeaf() int { + return 3 +} + +//go:noinline +func (d *DebuggerController) correlationLoop(w http.ResponseWriter, r *http.Request, loops int) { + total := 0 + for i := 0; i < loops; i++ { + total += i // correlation loop body line probe + time.Sleep(time.Second) + } + afterLoop := total // correlation loop sibling line probe + runtime.KeepAlive(afterLoop) + w.Write([]byte(fmt.Sprintf("Loop %d", afterLoop))) +} diff --git a/utils/build/docker/golang/app/chi/main.go b/utils/build/docker/golang/app/chi/main.go index b85532021b5..e078d9a465e 100644 --- a/utils/build/docker/golang/app/chi/main.go +++ b/utils/build/docker/golang/app/chi/main.go @@ -428,6 +428,11 @@ func main() { loops, _ := strconv.Atoi(chi.RouteContext(r.Context()).URLParam("count")) d.budgets(w, r, loops) }) + mux.HandleFunc("/debugger/correlation", d.correlation) + mux.HandleFunc("/debugger/correlation/loop/{count}", func(w http.ResponseWriter, r *http.Request) { + loops, _ := strconv.Atoi(chi.RouteContext(r.Context()).URLParam("count")) + d.correlationLoop(w, r, loops) + }) srv := &http.Server{ Addr: ":7777", diff --git a/utils/build/docker/golang/app/echo/debugger.go b/utils/build/docker/golang/app/echo/debugger.go index 37a36a8fb80..f02170c58cc 100644 --- a/utils/build/docker/golang/app/echo/debugger.go +++ b/utils/build/docker/golang/app/echo/debugger.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "runtime" + "time" ) // The `debugger` feature allows attachment to specific lines of code. @@ -13,7 +14,6 @@ import ( type DebuggerController struct{} -// Dummy line // Dummy line func (d *DebuggerController) logProbe(w http.ResponseWriter, r *http.Request) { // Dummy line @@ -116,3 +116,35 @@ func (d *DebuggerController) budgets(w http.ResponseWriter, r *http.Request, loo func (d *DebuggerController) budgetStep(i, loops int) int { return i + loops // loops is referenced so the capture-expression probe can read it at this line. } + +//go:noinline +func (d *DebuggerController) correlation(w http.ResponseWriter, r *http.Request) { + result := d.correlationMiddle() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + runtime.KeepAlive(result) + w.Write([]byte(fmt.Sprintf("Correlation %d", result))) +} + +//go:noinline +func (d *DebuggerController) correlationMiddle() int { + result := d.correlationLeaf() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + return result +} + +//go:noinline +func (d *DebuggerController) correlationLeaf() int { + return 3 +} + +//go:noinline +func (d *DebuggerController) correlationLoop(w http.ResponseWriter, r *http.Request, loops int) { + total := 0 + for i := 0; i < loops; i++ { + total += i // correlation loop body line probe + time.Sleep(time.Second) + } + afterLoop := total // correlation loop sibling line probe + runtime.KeepAlive(afterLoop) + w.Write([]byte(fmt.Sprintf("Loop %d", afterLoop))) +} diff --git a/utils/build/docker/golang/app/echo/main.go b/utils/build/docker/golang/app/echo/main.go index 631076f9a44..dae11906908 100644 --- a/utils/build/docker/golang/app/echo/main.go +++ b/utils/build/docker/golang/app/echo/main.go @@ -399,6 +399,12 @@ func main() { d.budgets(c.Response().Writer, c.Request(), loops) return nil }) + r.Any("/debugger/correlation", echoHandleFunc(d.correlation)) + r.Any("/debugger/correlation/loop/:count", func(c echo.Context) error { + loops, _ := strconv.Atoi(c.Param("count")) + d.correlationLoop(c.Response().Writer, c.Request(), loops) + return nil + }) common.InitDatadog() go grpc.ListenAndServe() diff --git a/utils/build/docker/golang/app/gin/debugger.go b/utils/build/docker/golang/app/gin/debugger.go index 37a36a8fb80..f02170c58cc 100644 --- a/utils/build/docker/golang/app/gin/debugger.go +++ b/utils/build/docker/golang/app/gin/debugger.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "runtime" + "time" ) // The `debugger` feature allows attachment to specific lines of code. @@ -13,7 +14,6 @@ import ( type DebuggerController struct{} -// Dummy line // Dummy line func (d *DebuggerController) logProbe(w http.ResponseWriter, r *http.Request) { // Dummy line @@ -116,3 +116,35 @@ func (d *DebuggerController) budgets(w http.ResponseWriter, r *http.Request, loo func (d *DebuggerController) budgetStep(i, loops int) int { return i + loops // loops is referenced so the capture-expression probe can read it at this line. } + +//go:noinline +func (d *DebuggerController) correlation(w http.ResponseWriter, r *http.Request) { + result := d.correlationMiddle() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + runtime.KeepAlive(result) + w.Write([]byte(fmt.Sprintf("Correlation %d", result))) +} + +//go:noinline +func (d *DebuggerController) correlationMiddle() int { + result := d.correlationLeaf() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + return result +} + +//go:noinline +func (d *DebuggerController) correlationLeaf() int { + return 3 +} + +//go:noinline +func (d *DebuggerController) correlationLoop(w http.ResponseWriter, r *http.Request, loops int) { + total := 0 + for i := 0; i < loops; i++ { + total += i // correlation loop body line probe + time.Sleep(time.Second) + } + afterLoop := total // correlation loop sibling line probe + runtime.KeepAlive(afterLoop) + w.Write([]byte(fmt.Sprintf("Loop %d", afterLoop))) +} diff --git a/utils/build/docker/golang/app/gin/main.go b/utils/build/docker/golang/app/gin/main.go index 1b8b3e4a913..9fe8822faed 100644 --- a/utils/build/docker/golang/app/gin/main.go +++ b/utils/build/docker/golang/app/gin/main.go @@ -384,6 +384,11 @@ func main() { loops, _ := strconv.Atoi(ctx.Param("count")) d.budgets(ctx.Writer, ctx.Request, loops) }) + r.Any("/debugger/correlation", ginHandleFunc(d.correlation)) + r.Any("/debugger/correlation/loop/:count", func(ctx *gin.Context) { + loops, _ := strconv.Atoi(ctx.Param("count")) + d.correlationLoop(ctx.Writer, ctx.Request, loops) + }) srv := &http.Server{ Addr: ":7777", diff --git a/utils/build/docker/golang/app/net-http-orchestrion/debugger.go b/utils/build/docker/golang/app/net-http-orchestrion/debugger.go index 37a36a8fb80..f02170c58cc 100644 --- a/utils/build/docker/golang/app/net-http-orchestrion/debugger.go +++ b/utils/build/docker/golang/app/net-http-orchestrion/debugger.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "runtime" + "time" ) // The `debugger` feature allows attachment to specific lines of code. @@ -13,7 +14,6 @@ import ( type DebuggerController struct{} -// Dummy line // Dummy line func (d *DebuggerController) logProbe(w http.ResponseWriter, r *http.Request) { // Dummy line @@ -116,3 +116,35 @@ func (d *DebuggerController) budgets(w http.ResponseWriter, r *http.Request, loo func (d *DebuggerController) budgetStep(i, loops int) int { return i + loops // loops is referenced so the capture-expression probe can read it at this line. } + +//go:noinline +func (d *DebuggerController) correlation(w http.ResponseWriter, r *http.Request) { + result := d.correlationMiddle() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + runtime.KeepAlive(result) + w.Write([]byte(fmt.Sprintf("Correlation %d", result))) +} + +//go:noinline +func (d *DebuggerController) correlationMiddle() int { + result := d.correlationLeaf() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + return result +} + +//go:noinline +func (d *DebuggerController) correlationLeaf() int { + return 3 +} + +//go:noinline +func (d *DebuggerController) correlationLoop(w http.ResponseWriter, r *http.Request, loops int) { + total := 0 + for i := 0; i < loops; i++ { + total += i // correlation loop body line probe + time.Sleep(time.Second) + } + afterLoop := total // correlation loop sibling line probe + runtime.KeepAlive(afterLoop) + w.Write([]byte(fmt.Sprintf("Loop %d", afterLoop))) +} diff --git a/utils/build/docker/golang/app/net-http-orchestrion/main.go b/utils/build/docker/golang/app/net-http-orchestrion/main.go index b9cda1a1f41..1689e75c17d 100644 --- a/utils/build/docker/golang/app/net-http-orchestrion/main.go +++ b/utils/build/docker/golang/app/net-http-orchestrion/main.go @@ -646,6 +646,11 @@ func main() { loops, _ := strconv.Atoi(r.PathValue("count")) d.budgets(w, r, loops) }) + mux.HandleFunc("/debugger/correlation", d.correlation) + mux.HandleFunc("/debugger/correlation/loop/{count}", func(w http.ResponseWriter, r *http.Request) { + loops, _ := strconv.Atoi(r.PathValue("count")) + d.correlationLoop(w, r, loops) + }) srv := &http.Server{ Addr: ":7777", diff --git a/utils/build/docker/golang/app/net-http/debugger.go b/utils/build/docker/golang/app/net-http/debugger.go index 37a36a8fb80..f02170c58cc 100644 --- a/utils/build/docker/golang/app/net-http/debugger.go +++ b/utils/build/docker/golang/app/net-http/debugger.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "runtime" + "time" ) // The `debugger` feature allows attachment to specific lines of code. @@ -13,7 +14,6 @@ import ( type DebuggerController struct{} -// Dummy line // Dummy line func (d *DebuggerController) logProbe(w http.ResponseWriter, r *http.Request) { // Dummy line @@ -116,3 +116,35 @@ func (d *DebuggerController) budgets(w http.ResponseWriter, r *http.Request, loo func (d *DebuggerController) budgetStep(i, loops int) int { return i + loops // loops is referenced so the capture-expression probe can read it at this line. } + +//go:noinline +func (d *DebuggerController) correlation(w http.ResponseWriter, r *http.Request) { + result := d.correlationMiddle() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + runtime.KeepAlive(result) + w.Write([]byte(fmt.Sprintf("Correlation %d", result))) +} + +//go:noinline +func (d *DebuggerController) correlationMiddle() int { + result := d.correlationLeaf() + time.Sleep(400 * time.Millisecond) // space the probed call sites in time + return result +} + +//go:noinline +func (d *DebuggerController) correlationLeaf() int { + return 3 +} + +//go:noinline +func (d *DebuggerController) correlationLoop(w http.ResponseWriter, r *http.Request, loops int) { + total := 0 + for i := 0; i < loops; i++ { + total += i // correlation loop body line probe + time.Sleep(time.Second) + } + afterLoop := total // correlation loop sibling line probe + runtime.KeepAlive(afterLoop) + w.Write([]byte(fmt.Sprintf("Loop %d", afterLoop))) +} diff --git a/utils/build/docker/golang/app/net-http/main.go b/utils/build/docker/golang/app/net-http/main.go index dafa5e66da1..1f54502fe63 100644 --- a/utils/build/docker/golang/app/net-http/main.go +++ b/utils/build/docker/golang/app/net-http/main.go @@ -797,6 +797,11 @@ func main() { loops, _ := strconv.Atoi(r.PathValue("count")) d.budgets(w, r, loops) }) + mux.HandleFunc("/debugger/correlation", d.correlation) + mux.HandleFunc("/debugger/correlation/loop/{count}", func(w http.ResponseWriter, r *http.Request) { + loops, _ := strconv.Atoi(r.PathValue("count")) + d.correlationLoop(w, r, loops) + }) srv := &http.Server{ Addr: ":7777", From 1f332f2e4d7c9fa256fa958eb1e2c0dfd47afbc7 Mon Sep 17 00:00:00 2001 From: dudik <8845578+dudikeleti@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:46:55 +0200 Subject: [PATCH 02/13] Enable .NET coordinated snapshot sampling from 3.55.0 dd-trace-dotnet#9266 shares one sampling decision per trace. Turn on Test_Debugger_Coordinated_Sampling for that release and add the /debugger/correlation weblog methods it probes. --- docs/understand/weblogs/end-to-end_weblog.md | 4 +++ manifests/dotnet.yml | 2 +- .../weblog/Controllers/DebuggerController.cs | 25 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/understand/weblogs/end-to-end_weblog.md b/docs/understand/weblogs/end-to-end_weblog.md index 967040e6cdf..69f2495434b 100644 --- a/docs/understand/weblogs/end-to-end_weblog.md +++ b/docs/understand/weblogs/end-to-end_weblog.md @@ -972,6 +972,10 @@ These endpoints are used for the `Dynamic Instrumentation` tests. Executes the same probe line `loops` times during one request. It is used to verify per-probe and global Dynamic Instrumentation rate limits. +#### GET /debugger/correlation + +Calls `Correlation`, `CorrelationMiddle`, and `CorrelationLeaf` in one request, pausing between the probed method exits. Used to check that snapshot sampling emits a whole probe chain together. + #### GET /debugger/snapshot/capture-timeout Creates a collection containing `collectionSize` object chains. Each chain has a leaf wrapped by diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 2b16c30cf23..42b7f92f8ac 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -786,7 +786,7 @@ manifest: - declaration: bug (DEBUG-5101) component_version: '<3.45.0' tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses: v2.53.0 - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: v3.55.0 # DEBUG-5828: per-trace coordinated snapshot sampling (#9266) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: v2.53.0 diff --git a/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs b/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs index 818e978ebb9..cfdaac2ebd7 100644 --- a/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs +++ b/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Mvc.ModelBinding.Binders; using Microsoft.AspNetCore.Routing; using System.Collections.Generic; +using System.Runtime.CompilerServices; using System.Threading.Tasks; using weblog.Models.Debugger; @@ -182,5 +183,29 @@ private static Dictionary CreatePiiUser() ["name"] = "NON_SENSITIVE_VALUE", }; } + + [HttpGet("correlation")] + [Consumes("application/json", "application/xml")] + [MethodImpl(MethodImplOptions.NoInlining)] + public IActionResult Correlation() + { + int result = CorrelationMiddle(); + System.Threading.Thread.Sleep(400); // space the probed call sites in time + return Content($"Correlation {result}"); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private int CorrelationMiddle() + { + int result = CorrelationLeaf(); + System.Threading.Thread.Sleep(400); // space the probed call sites in time + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private int CorrelationLeaf() + { + return 3; + } } } From 9cd9daacb435624ebc82d8dde7c37a023d2edb9b Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 7 Oct 2026 22:16:01 -0400 Subject: [PATCH 03/13] fix(dotnet): keep the probed lines of the debugger controller in place The `using System.Runtime.CompilerServices;` added for the correlation methods moved every line of DebuggerController.cs down by one, so all .NET line probes in the debugger tests targeted the wrong lines. Fully qualify the `MethodImpl` attributes instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../docker/dotnet/weblog/Controllers/DebuggerController.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs b/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs index cfdaac2ebd7..d159e702521 100644 --- a/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs +++ b/utils/build/docker/dotnet/weblog/Controllers/DebuggerController.cs @@ -4,7 +4,6 @@ using Microsoft.AspNetCore.Mvc.ModelBinding.Binders; using Microsoft.AspNetCore.Routing; using System.Collections.Generic; -using System.Runtime.CompilerServices; using System.Threading.Tasks; using weblog.Models.Debugger; @@ -186,7 +185,7 @@ private static Dictionary CreatePiiUser() [HttpGet("correlation")] [Consumes("application/json", "application/xml")] - [MethodImpl(MethodImplOptions.NoInlining)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public IActionResult Correlation() { int result = CorrelationMiddle(); @@ -194,7 +193,7 @@ public IActionResult Correlation() return Content($"Correlation {result}"); } - [MethodImpl(MethodImplOptions.NoInlining)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private int CorrelationMiddle() { int result = CorrelationLeaf(); @@ -202,7 +201,7 @@ private int CorrelationMiddle() return result; } - [MethodImpl(MethodImplOptions.NoInlining)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private int CorrelationLeaf() { return 3; From 09a45442403ff9f1b7673a55d6f045c0d95fc3c2 Mon Sep 17 00:00:00 2001 From: ddsign Date: Mon, 24 Aug 2026 12:35:18 -0400 Subject: [PATCH 04/13] test(debugger): accept flat dotted dd.trace_id in correlation gate The coordinated-sampling gate reads dd.trace_id to group snapshots by trace. The Ruby tracer emits it as a flat dotted key at the envelope root ("dd.trace_id"), matching the Java/.NET/Go dotted-key convention, while Python/Node/Ruby nest it under a dd object. The test read the nested form only, so dotted-key snapshots resolved to None and failed the presence assertion before any sampling logic ran. Read both shapes, mirroring the dual-shape lookup already used for dd.span_id in test_debugger_exception_replay.py. Scope: the reported feature-564 test only. runtime_id (camelCase) and the generation token remain nested/absent and stay out of scope. (cherry picked from commit e3da2e0fde69618f269a4fe7f764f20d5b6706b1) --- tests/debugger/test_debugger_snapshot_correlation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/debugger/test_debugger_snapshot_correlation.py b/tests/debugger/test_debugger_snapshot_correlation.py index a3c60bc64fb..61deb5608b6 100644 --- a/tests/debugger/test_debugger_snapshot_correlation.py +++ b/tests/debugger/test_debugger_snapshot_correlation.py @@ -60,7 +60,9 @@ def test_coordinated_per_trace_sampling(self): probes_by_trace: dict[str, set[str]] = {} for probe_id in self.probe_ids: for snapshot in self.probe_snapshots.get(probe_id, []): - trace_id = snapshot.get("dd", {}).get("trace_id") + # Tolerate the nested dd object (Python/Node/Ruby nest) and the flat dotted + # key (Java/.NET/Go/Ruby dotted) for the trace id. + trace_id = snapshot.get("dd", {}).get("trace_id") or snapshot.get("dd.trace_id") assert trace_id, f"snapshot for probe {probe_id} is missing dd.trace_id on the wire" probes_by_trace.setdefault(trace_id, set()).add(probe_id) From a6437dc948be7ad90a559326a93545589f2428aa Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 22 Oct 2026 09:32:19 +0200 Subject: [PATCH 05/13] docs(debugger): document the correlation loop endpoint Co-Authored-By: Claude Opus 5.5 --- docs/understand/weblogs/end-to-end_weblog.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/understand/weblogs/end-to-end_weblog.md b/docs/understand/weblogs/end-to-end_weblog.md index 69f2495434b..23fa0d6127b 100644 --- a/docs/understand/weblogs/end-to-end_weblog.md +++ b/docs/understand/weblogs/end-to-end_weblog.md @@ -976,6 +976,12 @@ Dynamic Instrumentation rate limits. Calls `Correlation`, `CorrelationMiddle`, and `CorrelationLeaf` in one request, pausing between the probed method exits. Used to check that snapshot sampling emits a whole probe chain together. +#### GET /debugger/correlation/loop/ + +Runs a loop `count` times in one request, sleeping one second per iteration. The loop body and the line right +after the loop are probed. Used to check that a probe emits at most one snapshot per trace, without starving a +sibling probe in the same trace. + #### GET /debugger/snapshot/capture-timeout Creates a collection containing `collectionSize` object chains. Each chain has a leaf wrapped by From 7c0b85c6c380c3032c131657436e1e71b37de405 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 22 Oct 2026 15:46:06 +0200 Subject: [PATCH 06/13] test(debugger): make the coordinated sampling test discriminating Every probe had the same one snapshot per second limit, and each request hits the probes at the same offsets, so independent per-probe samplers stay in lockstep and emit whole chains too. The test passed without coordinated sampling, e.g. for Node.js before coordinated sampling was implemented. Let the leaf, which is the first probe hit in each request and therefore decides for the trace, sample every request, and keep the other probes at one snapshot per second. A request takes about 800ms, so without coordination those probes are skipped in about every other request and the chains split. Replaying the scenario against Node.js weblog routes: 15 of 30 chains were partial without coordinated sampling, and all 30 were complete with it. Co-Authored-By: Claude Opus 5.5 --- .../test_debugger_snapshot_correlation.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/debugger/test_debugger_snapshot_correlation.py b/tests/debugger/test_debugger_snapshot_correlation.py index 61deb5608b6..60d92eda630 100644 --- a/tests/debugger/test_debugger_snapshot_correlation.py +++ b/tests/debugger/test_debugger_snapshot_correlation.py @@ -16,12 +16,17 @@ class Test_Debugger_Coordinated_Sampling(debugger.BaseDebuggerTest): """The sampling decision is made once per trace, so a trace emits its whole probe chain or none of it. - Correlation rides on the dd.trace_id already in the snapshot envelope. The endpoint spaces its - probed call sites in time, so today's independent per-probe rate limiters land in different - windows and emit partial chains, which is what makes the assertion discriminating. + Correlation rides on the dd.trace_id already in the snapshot envelope. The probes get different + per-probe rates, so independent per-probe sampling emits partial chains, which is what makes the + assertion discriminating. """ _INVOCATIONS = 30 + # The leaf is the first probe hit in each request, so it makes the trace's sampling decision, and + # its rate allows every request to be sampled. A request takes about 800ms, so the probes limited + # to one snapshot per second are skipped in about every other request unless they follow the + # trace's decision. + _SNAPSHOTS_PER_SECOND = {"CorrelationLeaf": 5, "CorrelationMiddle": 1, "Correlation": 1} def setup_coordinated_per_trace_sampling(self): self.initialize_weblog_remote_config() @@ -29,10 +34,7 @@ def setup_coordinated_per_trace_sampling(self): probes = debugger.read_probes("probe_snapshot_log_correlation") for probe in probes: probe["id"] = debugger.generate_probe_id("log") - # This is the per-probe time limiter, not a per-trace rate: it is set only to put the - # chain under sampling pressure so that an uncoordinated implementation splits it. - # Swap it for the per-trace rate once one exists. - probe["sampling"] = {"snapshotsPerSecond": 1} + probe["sampling"] = {"snapshotsPerSecond": self._SNAPSHOTS_PER_SECOND[probe["where"]["methodName"]]} self.set_probes(probes) self.send_rc_probes() From 36a57e6e019e8619b7d4e6142e6871036454aee7 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 22 Oct 2026 09:38:19 +0200 Subject: [PATCH 07/13] test(debugger): run the snapshot correlation tests for Node.js Add the `/debugger/correlation` and `/debugger/correlation/loop/:count` endpoints to the Node.js Express, TypeScript Express and Fastify weblogs, mirroring the Go endpoints: the probed functions return 400ms apart, and the loop sleeps for a second per iteration. The debugger scenarios run the weblogs with IAST enabled, and Dynamic Instrumentation places line probes two lines too early in files that the IAST rewriter rewrites. So the endpoints avoid everything the rewriter instruments (`+`, `+=`, template literals and some string methods) and respond with constant strings. Node.js doesn't support method probes, so the correlation tests point each probe at the line its method returns on for Node.js instead, using new Node.js entries in the line map. The new routes move the capture timeout line from 157 to 159, and the budgets line from 163 to 165. Enable `Test_Debugger_Coordinated_Sampling` and `test_per_span_budget` from v6.21.0 and v5.132.0, the releases that ship coordinated snapshot sampling (DataDog/dd-trace-js#10677). Co-Authored-By: Claude Opus 5.5 --- manifests/nodejs.yml | 7 +++- tests/debugger/utils.py | 12 ++++-- .../docker/nodejs/express/debugger/index.js | 40 ++++++++++++++++++- .../express4-typescript/debugger/index.ts | 40 ++++++++++++++++++- .../docker/nodejs/fastify/debugger/index.js | 40 ++++++++++++++++++- 5 files changed, 127 insertions(+), 12 deletions(-) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 94e3aa27549..dc24792600e 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1788,7 +1788,7 @@ manifest: fastify: *ref_5_46_0 uds-express4: *ref_5_46_0 tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_log_line_capture_expression_budgets: '>=6.16.0 || ^5.127.0' - tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: '>=6.21.0 || ^5.132.0' tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: missing_feature (Method probes are not supported by Node.js) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: - weblog_declaration: @@ -1847,7 +1847,10 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Line_Probe_Statuses::test_metric_line_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Line_Probe_Statuses::test_span_decoration_line_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses: missing_feature - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: + - weblog_declaration: + "*": '>=6.21.0 || ^5.132.0' + nextjs: irrelevant tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: missing_feature (feature not implented) diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index f60cfba3294..7d319104220 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -190,7 +190,7 @@ def initialize_weblog_remote_config(self) -> None: def method_and_language_to_line_number(self, method: str, language: str) -> list: """method_and_language_to_line_number returns the respective line number given the method and language""" definitions: dict[str, dict[str, list[int]]] = { - "Budgets": {"java": [140], "dotnet": [138], "python": [144], "golang": [117], "nodejs": [163]}, + "Budgets": {"java": [140], "dotnet": [138], "python": [144], "golang": [117], "nodejs": [165]}, "LogProbe": {"nodejs": [20]}, "Pii": {"java": [66], "dotnet": [66], "python": [66], "ruby": [66], "nodejs": [64]}, "Expression": {"java": [73], "dotnet": [76], "python": [74], "ruby": [82], "nodejs": [82], "golang": [71]}, @@ -212,9 +212,13 @@ def method_and_language_to_line_number(self, method: str, language: str) -> list "CollectionOperations": {"java": [116], "dotnet": [116], "python": [125], "ruby": [162], "nodejs": [120]}, "Nulls": {"java": [132], "dotnet": [129], "python": [138], "ruby": [192], "nodejs": [126]}, "SnapshotLimits": {"java": [155], "python": [174], "nodejs": [136], "ruby": [233], "dotnet": [152]}, - "CaptureTimeout": {"java": [174], "nodejs": [157], "dotnet": [173]}, - "CorrelationLoopBody": {"golang": [144]}, - "CorrelationLoopSibling": {"golang": [147]}, + "CaptureTimeout": {"java": [174], "nodejs": [159], "dotnet": [173]}, + # Lines the probed methods of the correlation endpoint return on, for tracers without method probes + "Correlation": {"nodejs": [178]}, + "CorrelationMiddle": {"nodejs": [184]}, + "CorrelationLeaf": {"nodejs": [188]}, + "CorrelationLoopBody": {"golang": [144], "nodejs": [194]}, + "CorrelationLoopSibling": {"golang": [147], "nodejs": [197]}, } return definitions.get(method, {}).get(language, []) diff --git a/utils/build/docker/nodejs/express/debugger/index.js b/utils/build/docker/nodejs/express/debugger/index.js index e3a68bf4dbf..ec4f7028b03 100644 --- a/utils/build/docker/nodejs/express/debugger/index.js +++ b/utils/build/docker/nodejs/express/debugger/index.js @@ -143,6 +143,8 @@ module.exports = { }) app.get('/debugger/budgets/:loops', budgets) + app.get('/debugger/correlation', correlationHandler) + app.get('/debugger/correlation/loop/:count', correlationLoopHandler) } } @@ -154,13 +156,47 @@ function captureTimeoutFixture (collectionSize, nestingDepth) { } return nested }) - return 'Capture timeout probe' // This needs to be line 157 + return 'Capture timeout probe' // This needs to be line 159 } function budgets (request, reply) { const loops = Number(request.params.loops) for (let iteration = 0; iteration < loops; iteration++) { - const currentIteration = iteration // This needs to be line 163 + const currentIteration = iteration // This needs to be line 165 } return reply.send('Budgets') } + +async function correlationHandler (req, res) { + await correlation() + res.send('Correlation') +} + +async function correlation () { + const value = await correlationMiddle() + await sleep(400) + return value // This needs to be line 178 +} + +async function correlationMiddle () { + const value = correlationLeaf() + await sleep(400) + return value // This needs to be line 184 +} + +function correlationLeaf () { + return 3 // This needs to be line 188 +} + +async function correlationLoopHandler (req, res) { + const count = Number(req.params.count) + for (let i = 0; i < count; i++) { + const iteration = i // This needs to be line 194 + await sleep(1000) + } + res.send('Loop') // This needs to be line 197 +} + +function sleep (ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/utils/build/docker/nodejs/express4-typescript/debugger/index.ts b/utils/build/docker/nodejs/express4-typescript/debugger/index.ts index 8b7076ac94c..eca43368392 100644 --- a/utils/build/docker/nodejs/express4-typescript/debugger/index.ts +++ b/utils/build/docker/nodejs/express4-typescript/debugger/index.ts @@ -143,6 +143,8 @@ export function initRoutes (app: Express) { }) app.get('/debugger/budgets/:loops', budgets) + app.get('/debugger/correlation', correlationHandler) + app.get('/debugger/correlation/loop/:count', correlationLoopHandler) } function captureTimeoutFixture (collectionSize: number, nestingDepth: number): string { @@ -154,13 +156,47 @@ function captureTimeoutFixture (collectionSize: number, nestingDepth: number): s return nested }) - return 'Capture timeout probe' // This needs to be line 157 + return 'Capture timeout probe' // This needs to be line 159 } function budgets (request: Request, reply: Response): Response { const loops = Number(request.params.loops) for (let iteration = 0; iteration < loops; iteration++) { - const currentIteration = iteration // This needs to be line 163 + const currentIteration = iteration // This needs to be line 165 } return reply.send('Budgets') } + +async function correlationHandler (req: Request, res: Response): Promise { + await correlation() + res.send('Correlation') +} + +async function correlation (): Promise { + const value = await correlationMiddle() + await sleep(400) + return value // This needs to be line 178 +} + +async function correlationMiddle (): Promise { + const value = correlationLeaf() + await sleep(400) + return value // This needs to be line 184 +} + +function correlationLeaf (): number { + return 3 // This needs to be line 188 +} + +async function correlationLoopHandler (req: Request, res: Response): Promise { + const count = Number(req.params.count) + for (let i = 0; i < count; i++) { + const iteration = i // This needs to be line 194 + await sleep(1000) + } + res.send('Loop') // This needs to be line 197 +} + +function sleep (ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/utils/build/docker/nodejs/fastify/debugger/index.js b/utils/build/docker/nodejs/fastify/debugger/index.js index 33edf1f2335..5caf0e0c839 100644 --- a/utils/build/docker/nodejs/fastify/debugger/index.js +++ b/utils/build/docker/nodejs/fastify/debugger/index.js @@ -143,6 +143,8 @@ module.exports = { }) fastify.get('/debugger/budgets/:loops', budgets) + fastify.get('/debugger/correlation', correlationHandler) + fastify.get('/debugger/correlation/loop/:count', correlationLoopHandler) } } @@ -154,13 +156,47 @@ function captureTimeoutFixture (collectionSize, nestingDepth) { } return nested }) - return 'Capture timeout probe' // This needs to be line 157 + return 'Capture timeout probe' // This needs to be line 159 } function budgets (request, reply) { const loops = Number(request.params.loops) for (let iteration = 0; iteration < loops; iteration++) { - const currentIteration = iteration // This needs to be line 163 + const currentIteration = iteration // This needs to be line 165 } return reply.send('Budgets') } + +async function correlationHandler (request, reply) { + await correlation() + return 'Correlation' +} + +async function correlation () { + const value = await correlationMiddle() + await sleep(400) + return value // This needs to be line 178 +} + +async function correlationMiddle () { + const value = correlationLeaf() + await sleep(400) + return value // This needs to be line 184 +} + +function correlationLeaf () { + return 3 // This needs to be line 188 +} + +async function correlationLoopHandler (request, reply) { + const count = Number(request.params.count) + for (let i = 0; i < count; i++) { + const iteration = i // This needs to be line 194 + await sleep(1000) + } + return 'Loop' // This needs to be line 197 +} + +function sleep (ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} From 7d189849ff052a27ecbb4e1444a76112a059693f Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 22 Oct 2026 09:39:05 +0200 Subject: [PATCH 08/13] chore(nodejs): refresh the Node.js base images The new debugger fixtures change the content of the express4, express4-typescript, express5 and fastify base images. Co-Authored-By: Claude Opus 5.5 --- utils/build/docker/base-images.lock.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/build/docker/base-images.lock.json b/utils/build/docker/base-images.lock.json index d2f79d4f9ca..578a921c155 100644 --- a/utils/build/docker/base-images.lock.json +++ b/utils/build/docker/base-images.lock.json @@ -2,10 +2,10 @@ "version": 1, "images": { "system_tests_base_java_lambda_java_lambda_runtime": "datadog/system-tests:java-lambda-runtime.base-e9a494d2bc57", - "system_tests_base_nodejs_express4": "datadog/system-tests:express4.base-7a16285d66a9", - "system_tests_base_nodejs_express4_typescript": "datadog/system-tests:express4-typescript.base-be5138693a38", - "system_tests_base_nodejs_express5": "datadog/system-tests:express5.base-fb385bd8cf2f", - "system_tests_base_nodejs_fastify": "datadog/system-tests:fastify.base-8f4f6cf42bfb", + "system_tests_base_nodejs_express4": "datadog/system-tests:express4.base-527c2a306057", + "system_tests_base_nodejs_express4_typescript": "datadog/system-tests:express4-typescript.base-070fd1ba6efd", + "system_tests_base_nodejs_express5": "datadog/system-tests:express5.base-5bff656d2059", + "system_tests_base_nodejs_fastify": "datadog/system-tests:fastify.base-d8786a519f91", "system_tests_base_nodejs_lambda_nodejs_lambda_runtime": "datadog/system-tests:nodejs-lambda-runtime.base-93b9660a8ce9", "system_tests_base_nodejs_nextjs": "datadog/system-tests:nextjs.base-5c42165eff3e", "system_tests_base_php_apache_mod_7_0": "datadog/system-tests:apache-mod-7.0.base-817e138f4813", From 4f9140cd32f11520da71db19b1774d7808cecea7 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 22 Oct 2026 11:12:38 +0200 Subject: [PATCH 09/13] chore(nodejs): mirror the refreshed Node.js base images Co-Authored-By: Claude Opus 5.5 --- mirror_images.lock.yaml | 32 ++++++++++++++++---------------- mirror_images.yaml | 8 ++++---- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mirror_images.lock.yaml b/mirror_images.lock.yaml index fc975ace23a..8562942aaf5 100644 --- a/mirror_images.lock.yaml +++ b/mirror_images.lock.yaml @@ -110,26 +110,26 @@ images: digest: sha256:c2d8a505dfd121933b113e0717f8fbe2e8352f4533d99c85cc5456d9aaf77bde target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:django-py3.13.base-8fa5fb488bac tag: django-py3.13.base-8fa5fb488bac - datadog/system-tests:express4-typescript.base-be5138693a38: - digest: sha256:6e66aa1b5da64a0d04a108c78f35c3f9f26a941f8233160c5eb78f08d0d3834d - target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4-typescript.base-be5138693a38 - tag: express4-typescript.base-be5138693a38 - datadog/system-tests:express4.base-7a16285d66a9: - digest: sha256:3b3365bb8748df3ff993459149705c08a127589ccba72a1a5691f8905b742db2 - target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4.base-7a16285d66a9 - tag: express4.base-7a16285d66a9 - datadog/system-tests:express5.base-fb385bd8cf2f: - digest: sha256:bd622fa20feba4e54e5c4aebe3e069b9f9b5ffb8a3d016c66749640e44c943fd - target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express5.base-fb385bd8cf2f - tag: express5.base-fb385bd8cf2f + datadog/system-tests:express4-typescript.base-070fd1ba6efd: + digest: sha256:c805fcf6a950f68d3b1fe25102dfbe4a3175374857475ad417154e1e53ebc959 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4-typescript.base-070fd1ba6efd + tag: express4-typescript.base-070fd1ba6efd + datadog/system-tests:express4.base-527c2a306057: + digest: sha256:5960fa4fcfdd56fee4754847c6400c78b6d1d4cc3f6093954ea9fa959ef6cd24 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4.base-527c2a306057 + tag: express4.base-527c2a306057 + datadog/system-tests:express5.base-5bff656d2059: + digest: sha256:56f6df4d24495b11e13e122f117d0938a3b86fc9b14c4d30e94bfb88e2999946 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express5.base-5bff656d2059 + tag: express5.base-5bff656d2059 datadog/system-tests:fastapi.base-7f1b67ecf0c2: digest: sha256:6ee0374d82b4cb777189b4e5f9a6ceb8697ee92be2ab455691445b1c86e2df4a target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastapi.base-7f1b67ecf0c2 tag: fastapi.base-7f1b67ecf0c2 - datadog/system-tests:fastify.base-8f4f6cf42bfb: - digest: sha256:4ebcb8a7ec9cf2152633cec3d907d90ce50c076420e98e15e2322f9bb25682d8 - target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastify.base-8f4f6cf42bfb - tag: fastify.base-8f4f6cf42bfb + datadog/system-tests:fastify.base-d8786a519f91: + digest: sha256:e7006ea8fb96c22809b44721f6aefaaa249f2d49de5cfaeff9ad5df290c13526 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastify.base-d8786a519f91 + tag: fastify.base-d8786a519f91 datadog/system-tests:flask-poc.base-ac207107ea25: digest: sha256:1df044be7f3766d2f8208420b010d485bf270f98f955344997029ac732b3fca4 target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:flask-poc.base-ac207107ea25 diff --git a/mirror_images.yaml b/mirror_images.yaml index a422e046b5f..c1f3b879448 100644 --- a/mirror_images.yaml +++ b/mirror_images.yaml @@ -35,11 +35,11 @@ - "datadog/system-tests:apache-mod-8.2.base-5771d89e2b55" - "datadog/system-tests:django-poc.base-1d15b551f35b" - "datadog/system-tests:django-py3.13.base-8fa5fb488bac" -- "datadog/system-tests:express4-typescript.base-be5138693a38" -- "datadog/system-tests:express4.base-7a16285d66a9" -- "datadog/system-tests:express5.base-fb385bd8cf2f" +- "datadog/system-tests:express4-typescript.base-070fd1ba6efd" +- "datadog/system-tests:express4.base-527c2a306057" +- "datadog/system-tests:express5.base-5bff656d2059" - "datadog/system-tests:fastapi.base-7f1b67ecf0c2" -- "datadog/system-tests:fastify.base-8f4f6cf42bfb" +- "datadog/system-tests:fastify.base-d8786a519f91" - "datadog/system-tests:flask-poc.base-ac207107ea25" - "datadog/system-tests:golang_buddy-v2" - "datadog/system-tests:java-lambda-runtime.base-e9a494d2bc57" From 1ed6a5b35d2157dbc656b4749403c134a605c777 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 22 Oct 2026 15:57:43 +0200 Subject: [PATCH 10/13] test(debugger): add a line-probe variant of the coordinated sampling test Node.js doesn't support method probes. Instead of branching on the library in the test, add `Test_Debugger_Coordinated_Sampling_Line_Probe`, which points each probe at the line its method returns on, using the line map. Enable it for Node.js, and mark the method-probe variant as irrelevant there, like the evaluation timeout guardrail test variants. The other libraries keep the method-probe variant. Co-Authored-By: Claude Opus 5.5 --- manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 3 +- manifests/php.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + .../test_debugger_snapshot_correlation.py | 36 ++++++++++++++----- 8 files changed, 36 insertions(+), 9 deletions(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 42b7f92f8ac..06493fa0f96 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -787,6 +787,7 @@ manifest: component_version: '<3.45.0' tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses: v2.53.0 tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: v3.55.0 # DEBUG-5828: per-trace coordinated snapshot sampling (#9266) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: v2.53.0 diff --git a/manifests/golang.yml b/manifests/golang.yml index 11b0a6ce6b2..55c737faf8a 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1705,6 +1705,7 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_decoration_method_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_method_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: v2.2.3 diff --git a/manifests/java.yml b/manifests/java.yml index 86eadaed9e3..1d8b5e28f0a 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3398,6 +3398,7 @@ manifest: spring-boot-wildfly: v1.38.0 uds-spring-boot: v1.38.0 tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index dc24792600e..669ea0176ed 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1847,7 +1847,8 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Line_Probe_Statuses::test_metric_line_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Line_Probe_Statuses::test_span_decoration_line_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses: missing_feature - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: irrelevant (Method-probe variant is not used by Node.js) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: - weblog_declaration: "*": '>=6.21.0 || ^5.132.0' nextjs: irrelevant diff --git a/manifests/php.yml b/manifests/php.yml index 4322a86fd62..42177497a32 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1221,6 +1221,7 @@ manifest: - declaration: flaky (DEBUG-3814) component_version: '>1.8.3' tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: missing_feature diff --git a/manifests/python.yml b/manifests/python.yml index cf5b14fc019..6dd98631bb0 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2180,6 +2180,7 @@ manifest: - declaration: bug (DEBUG-3127) component_version: 2.16.1 tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index d2f64dc6e95..49a66ac823a 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2064,6 +2064,7 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_decoration_method_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_method_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: diff --git a/tests/debugger/test_debugger_snapshot_correlation.py b/tests/debugger/test_debugger_snapshot_correlation.py index 60d92eda630..5b663b225dd 100644 --- a/tests/debugger/test_debugger_snapshot_correlation.py +++ b/tests/debugger/test_debugger_snapshot_correlation.py @@ -4,16 +4,12 @@ import tests.debugger.utils as debugger -from utils import scenarios, features, rfc, slow +from utils import context, scenarios, features, rfc, slow +_RFC = "https://docs.google.com/document/d/17BQ1cEcJuumpMWMtBItIx9x_akG99dzjBA1NoaBFuBE/edit?tab=t.4l0zjxovcnz3#heading=h.kejjaf501thi" -@rfc( - "https://docs.google.com/document/d/17BQ1cEcJuumpMWMtBItIx9x_akG99dzjBA1NoaBFuBE/edit?tab=t.4l0zjxovcnz3#heading=h.kejjaf501thi" -) -@features.debugger_snapshot_correlation -@scenarios.debugger_probes_snapshot -@slow -class Test_Debugger_Coordinated_Sampling(debugger.BaseDebuggerTest): + +class _DebuggerCoordinatedSamplingTest(debugger.BaseDebuggerTest): """The sampling decision is made once per trace, so a trace emits its whole probe chain or none of it. Correlation rides on the dd.trace_id already in the snapshot envelope. The probes get different @@ -28,6 +24,9 @@ class Test_Debugger_Coordinated_Sampling(debugger.BaseDebuggerTest): # trace's decision. _SNAPSHOTS_PER_SECOND = {"CorrelationLeaf": 5, "CorrelationMiddle": 1, "Correlation": 1} + def _prepare_probe(self, probe: dict) -> None: + pass + def setup_coordinated_per_trace_sampling(self): self.initialize_weblog_remote_config() @@ -35,6 +34,7 @@ def setup_coordinated_per_trace_sampling(self): for probe in probes: probe["id"] = debugger.generate_probe_id("log") probe["sampling"] = {"snapshotsPerSecond": self._SNAPSHOTS_PER_SECOND[probe["where"]["methodName"]]} + self._prepare_probe(probe) self.set_probes(probes) self.send_rc_probes() @@ -77,6 +77,26 @@ def test_coordinated_per_trace_sampling(self): assert full, "no trace emitted the full probe chain" +@rfc(_RFC) +@features.debugger_snapshot_correlation +@scenarios.debugger_probes_snapshot +@slow +class Test_Debugger_Coordinated_Sampling(_DebuggerCoordinatedSamplingTest): + """Probes the methods of the correlation endpoint.""" + + +@rfc(_RFC) +@features.debugger_snapshot_correlation +@scenarios.debugger_probes_snapshot +@slow +class Test_Debugger_Coordinated_Sampling_Line_Probe(_DebuggerCoordinatedSamplingTest): + """Probes the line each method of the correlation endpoint returns on, for tracers without method probes.""" + + def _prepare_probe(self, probe: dict) -> None: + lines = self.method_and_language_to_line_number(probe["where"]["methodName"], context.library.name) + self._rewrite_where_for_lines([probe], lines) + + # Follow-on work, split out of feature 564 (which is coordinated sampling via trace_id only). There # is no dashboard feature for these yet, and they are simply not implemented (not blocked on any # schema negotiation), so they are not reported. From 4afa95470c3a8b5612bc63969e884cb560da28f1 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Fri, 23 Oct 2026 07:00:06 -0400 Subject: [PATCH 11/13] test(debugger): drop the correlation fields test and fix review findings Remove Test_Debugger_Snapshot_Correlation_Fields. No tracer sends the generation token it asserts on, and nothing defines that field yet. Its thread_id check also fails Java, which sends the key flat, and PHP, which has no logger object. The runtime ID part is dropped for now. Rename test_per_span_budget to test_per_trace_budget, since tracers cap each probe at one snapshot per trace, not per span. Count every collected snapshot, so tracers that send the flat debugger.snapshot key are counted. Use a ref for the Node.js 6.21.0 / 5.132.0 versions. Declare the Java entries incomplete_test_app. dd-trace-java 1.67.0 has coordinated sampling and the per-trace cap, but the Java weblog has no /debugger/correlation endpoints. Environment: Datadog workspace Co-Authored-By: Claude Sonnet 5.5 --- manifests/dotnet.yml | 4 +- manifests/golang.yml | 4 +- manifests/java.yml | 6 +- manifests/nodejs.yml | 7 +- manifests/php.yml | 4 +- manifests/python.yml | 4 +- manifests/ruby.yml | 4 +- tests/debugger/test_debugger_probe_budgets.py | 18 +++-- .../test_debugger_snapshot_correlation.py | 72 ------------------- 9 files changed, 18 insertions(+), 105 deletions(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 06493fa0f96..2ae0511b539 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -763,7 +763,7 @@ manifest: weblog: [uds, poc] - declaration: bug (DEBUG-4746) excluded_weblog: [uds, poc] - tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_trace_budget: missing_feature (per-trace snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: irrelevant (Dotnet emits evaluation errors as span tags, not error snapshots to the debugger endpoint) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: v2.53.0 tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_default_max_length: @@ -788,8 +788,6 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses: v2.53.0 tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: v3.55.0 # DEBUG-5828: per-trace coordinated snapshot sampling (#9266) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: v2.53.0 tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_event_metadata: v3.44.0 tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_symdb_upload: # Modified by easy win activation script diff --git a/manifests/golang.yml b/manifests/golang.yml index 55c737faf8a..f67e4aacdb7 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1667,7 +1667,7 @@ manifest: tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction: missing_feature (feature not implemented) tests/debugger/test_debugger_pii.py::Test_Debugger_PII_Redaction_Excluded_Identifiers: missing_feature (feature not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets: v2.2.3 - tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_trace_budget: missing_feature (per-trace snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: missing_feature (span decoration probes not yet implemented for Go) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: v2.2.3 tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_default_max_collection_size: missing_feature (no /debugger/snapshot/limits endpoint) @@ -1706,8 +1706,6 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_method_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: v2.2.3 # TODO(andrei): un-skip when datadog-agent v7.79.0 is released (Go's symdb upload comes from the agent, not from dd-trace-go) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_event_metadata: # TODO: a lower version might be supported diff --git a/manifests/java.yml b/manifests/java.yml index 1d8b5e28f0a..2c09fbe72d9 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3316,7 +3316,7 @@ manifest: spring-boot-undertow: v1.38.0 spring-boot-wildfly: v1.38.0 uds-spring-boot: v1.38.0 - tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_trace_budget: incomplete_test_app (no /debugger/correlation/loop endpoint) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: bug (DEBUG-3797) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: - weblog_declaration: @@ -3397,10 +3397,8 @@ manifest: spring-boot-undertow: v1.38.0 spring-boot-wildfly: v1.38.0 uds-spring-boot: v1.38.0 - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) + tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: incomplete_test_app (no /debugger/correlation endpoint) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: - weblog_declaration: "*": missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 669ea0176ed..6e857dd851b 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -110,6 +110,7 @@ refs: - &ref_6_17_0 '>=6.17.0 || ^5.128.0' - &ref_6_20_0 '>=6.20.0 || ^5.131.0' - &ref_6_20_1 '>=6.20.1 || ^5.131.1' + - &ref_6_21_0 '>=6.21.0 || ^5.132.0' manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_AIGuardEvent_Tag: - weblog_declaration: @@ -1788,7 +1789,7 @@ manifest: fastify: *ref_5_46_0 uds-express4: *ref_5_46_0 tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_log_line_capture_expression_budgets: '>=6.16.0 || ^5.127.0' - tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: '>=6.21.0 || ^5.132.0' + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_trace_budget: *ref_6_21_0 tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: missing_feature (Method probes are not supported by Node.js) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: - weblog_declaration: @@ -1850,10 +1851,8 @@ manifest: tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: irrelevant (Method-probe variant is not used by Node.js) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: - weblog_declaration: - "*": '>=6.21.0 || ^5.132.0' + "*": *ref_6_21_0 nextjs: irrelevant - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: missing_feature (feature not implented) tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: - weblog_declaration: diff --git a/manifests/php.yml b/manifests/php.yml index 42177497a32..128ebe5dc9f 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1190,7 +1190,7 @@ manifest: laravel11x: incomplete_test_app symfony7x: incomplete_test_app tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_log_line_capture_expression_budgets: missing_feature - tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_trace_budget: missing_feature (per-trace snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: irrelevant tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: missing_feature tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) @@ -1222,8 +1222,6 @@ manifest: component_version: '>1.8.3' tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: missing_feature tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: - weblog_declaration: diff --git a/manifests/python.yml b/manifests/python.yml index 6dd98631bb0..2c19e1d19f4 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2119,7 +2119,7 @@ manifest: "*": missing_feature *flask : v2.11.0 tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_log_line_capture_expression_budgets: bug (DEBUG-5743) - tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented; tracer uses a time-based budget) + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_trace_budget: missing_feature (per-trace snapshot budget of 1 not implemented; tracer uses a time-based budget) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_span_probe_expression_budgets: irrelevant (Python emits evaluation errors as span tags, not error snapshots to the debugger endpoint) tests/debugger/test_debugger_probe_snapshot.py: - weblog_declaration: @@ -2181,8 +2181,6 @@ manifest: component_version: 2.16.1 tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: - weblog_declaration: "*": missing_feature diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 49a66ac823a..2753fefb07c 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1998,7 +1998,7 @@ manifest: rails80: missing_feature (feature not implemented) uds-rails: missing_feature (feature not implemented) tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_log_line_budgets: missing_feature (Not yet implemented) - tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_span_budget: missing_feature (per-span snapshot budget of 1 not implemented) + tests/debugger/test_debugger_probe_budgets.py::Test_Debugger_Probe_Budgets::test_per_trace_budget: missing_feature (per-trace snapshot budget of 1 not implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots: - weblog_declaration: "*": irrelevant @@ -2065,8 +2065,6 @@ manifest: tests/debugger/test_debugger_probe_status.py::Test_Debugger_Method_Probe_Statuses::test_span_method_status: missing_feature (Not yet implemented) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling: missing_feature (no per-trace coordinated sampling; probabilistic per-trace rate not implemented) tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Coordinated_Sampling_Line_Probe: irrelevant (Line-probe variant is only used by Node.js) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_generation_token: missing_feature (execution-context generation token not yet emitted) - tests/debugger/test_debugger_snapshot_correlation.py::Test_Debugger_Snapshot_Correlation_Fields::test_runtime_id_in_envelope: missing_feature (runtime_id not yet present in the snapshot envelope) tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: - weblog_declaration: "*": irrelevant diff --git a/tests/debugger/test_debugger_probe_budgets.py b/tests/debugger/test_debugger_probe_budgets.py index 2e01d47ba98..f0a4db16212 100644 --- a/tests/debugger/test_debugger_probe_budgets.py +++ b/tests/debugger/test_debugger_probe_budgets.py @@ -169,7 +169,7 @@ def test_span_probe_expression_budgets(self): f"Span probe expression error logging should be rate-limited." ) - def setup_per_span_budget(self): + def setup_per_trace_budget(self): self.initialize_weblog_remote_config() language = self.get_tracer()["language"] @@ -185,20 +185,18 @@ def setup_per_span_budget(self): self.send_rc_probes() self.wait_for_all_probes(statuses=["INSTALLED"]) - # The loop sleeps one second per iteration, so the time-based budget above would emit more - # than one snapshot while a span-scoped budget emits exactly one. + # The loop sleeps one second per iteration, so a time-based budget would emit more than one + # snapshot while a trace-scoped budget emits exactly one. self.send_weblog_request("/debugger/correlation/loop/3") self.wait_for_all_probes(statuses=["EMITTING"]) self.wait_for_all_snapshots() - def test_per_span_budget(self): - """A probe in a loop emits one snapshot per span and does not starve a sibling probe.""" + def test_per_trace_budget(self): + """A probe in a loop emits one snapshot per trace and does not starve a sibling probe.""" self._assert() - # A single request is a single span, so every probe must have emitted exactly once. + # A single request is a single trace, so every probe must have emitted exactly once. for probe_id in self.probe_ids: - snapshots = [s for s in self.probe_snapshots.get(probe_id, []) if s.get("debugger", {}).get("snapshot")] - assert len(snapshots) == 1, ( - f"probe {probe_id} emitted {len(snapshots)} snapshots for one span, expected exactly 1" - ) + count = len(self.probe_snapshots.get(probe_id, [])) + assert count == 1, f"probe {probe_id} emitted {count} snapshots in one trace, expected exactly 1" diff --git a/tests/debugger/test_debugger_snapshot_correlation.py b/tests/debugger/test_debugger_snapshot_correlation.py index 5b663b225dd..220dc22aa0e 100644 --- a/tests/debugger/test_debugger_snapshot_correlation.py +++ b/tests/debugger/test_debugger_snapshot_correlation.py @@ -95,75 +95,3 @@ class Test_Debugger_Coordinated_Sampling_Line_Probe(_DebuggerCoordinatedSampling def _prepare_probe(self, probe: dict) -> None: lines = self.method_and_language_to_line_number(probe["where"]["methodName"], context.library.name) self._rewrite_where_for_lines([probe], lines) - - -# Follow-on work, split out of feature 564 (which is coordinated sampling via trace_id only). There -# is no dashboard feature for these yet, and they are simply not implemented (not blocked on any -# schema negotiation), so they are not reported. -@features.not_reported -@scenarios.debugger_probes_snapshot -@slow -class Test_Debugger_Snapshot_Correlation_Fields(debugger.BaseDebuggerTest): - """Identity fields that make a snapshot correlatable: the per-process runtime_id in the envelope, - and a generation token that disambiguates a reused thread id. - """ - - def _setup(self): - self.initialize_weblog_remote_config() - - probes = debugger.read_probes("probe_snapshot_log_correlation") - for probe in probes: - probe["id"] = debugger.generate_probe_id("log") - self.set_probes(probes) - - self.send_rc_probes() - if not self.wait_for_all_probes(statuses=["INSTALLED"], timeout=60): - self.setup_failures.append("Probes did not reach INSTALLED status") - return - - self.send_weblog_request("/debugger/correlation") - self.wait_for_all_probes(statuses=["EMITTING"]) - self.wait_for_all_snapshots() - - def _assert(self): - self.collect() - - self.assert_setup_ok() - self.assert_rc_state_not_error() - self.assert_all_weblog_responses_ok() - - def setup_runtime_id_in_envelope(self): - self._setup() - - def test_runtime_id_in_envelope(self): - """Every snapshot carries the same non-empty runtime_id, since all come from one process.""" - self._assert() - - runtime_ids = set() - for probe_id in self.probe_ids: - snapshots = self.probe_snapshots.get(probe_id, []) - assert snapshots, f"no snapshot was received for probe {probe_id}" - for snapshot in snapshots: - # The envelope does not carry it today, so tolerate the root or the "dd" object. - runtime_id = snapshot.get("runtime_id") or snapshot.get("dd", {}).get("runtime_id") - assert runtime_id, f"snapshot for probe {probe_id} is missing a non-empty runtime_id" - runtime_ids.add(runtime_id) - - assert len(runtime_ids) == 1, f"expected one runtime_id across the run, got {runtime_ids}" - - def setup_generation_token(self): - self._setup() - - def test_generation_token(self): - """Every snapshot carries a generation token alongside thread_id.""" - self._assert() - - for probe_id in self.probe_ids: - snapshots = self.probe_snapshots.get(probe_id, []) - assert snapshots, f"no snapshot was received for probe {probe_id}" - for snapshot in snapshots: - context = snapshot.get("logger", {}) - assert "thread_id" in context, f"snapshot for probe {probe_id} is missing thread_id" - assert context.get("generation") not in (None, ""), ( - f"snapshot for probe {probe_id} is missing the execution-context generation token" - ) From bd9310b6942bc46c3028291cdbd76ab0f29a1c9b Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Fri, 23 Oct 2026 07:04:56 -0400 Subject: [PATCH 12/13] docs(debugger): state the RFC's coordinated sampling rule in the test The RFC says the first snapshot probe hit in a trace decides for the whole trace, and that once it emits, every probe in the trace may emit until its cap of one snapshot per trace. The test relies on that: the leaf samples every request, and the other probes emit past their own one-per-second rate. Say so in the class docstring. The leaf is only the first probe hit for line probes on the return lines. Method probes may count the outer method first, so reword that comment. Environment: Datadog workspace Co-Authored-By: Claude Opus 5.5 --- tests/debugger/test_debugger_snapshot_correlation.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/debugger/test_debugger_snapshot_correlation.py b/tests/debugger/test_debugger_snapshot_correlation.py index 220dc22aa0e..02cf424c40e 100644 --- a/tests/debugger/test_debugger_snapshot_correlation.py +++ b/tests/debugger/test_debugger_snapshot_correlation.py @@ -12,16 +12,20 @@ class _DebuggerCoordinatedSamplingTest(debugger.BaseDebuggerTest): """The sampling decision is made once per trace, so a trace emits its whole probe chain or none of it. + As the RFC specifies, the first snapshot probe hit in a trace decides for the whole trace. If it + drops, every probe in the trace drops. If it emits, every probe in the trace may emit until it + reaches its cap of one snapshot per trace, even where its own per-probe rate would have skipped it. + Correlation rides on the dd.trace_id already in the snapshot envelope. The probes get different per-probe rates, so independent per-probe sampling emits partial chains, which is what makes the assertion discriminating. """ _INVOCATIONS = 30 - # The leaf is the first probe hit in each request, so it makes the trace's sampling decision, and - # its rate allows every request to be sampled. A request takes about 800ms, so the probes limited - # to one snapshot per second are skipped in about every other request unless they follow the - # trace's decision. + # The leaf samples every request on its own. A request takes about 800ms, so without coordination + # the probes limited to one snapshot per second are skipped in about every other request while the + # leaf is not. With line probes on the return lines, the leaf is also the first probe hit, so it + # makes the trace's decision. _SNAPSHOTS_PER_SECOND = {"CorrelationLeaf": 5, "CorrelationMiddle": 1, "Correlation": 1} def _prepare_probe(self, probe: dict) -> None: From b1b9f40c93da6852e4dbb259b64b2442314d7acd Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Fri, 23 Oct 2026 09:23:36 -0400 Subject: [PATCH 13/13] docs(debugger): stop listing languages in the trace ID comment The comment named Ruby as sending both the nested dd object and the flat dd.trace_id key, and Go as sending the flat key. In CI, Ruby sends only the flat key and Go sends neither. Describe the two formats without language lists, which go stale. Environment: Datadog workspace Co-Authored-By: Claude Opus 5.5 --- tests/debugger/test_debugger_snapshot_correlation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/debugger/test_debugger_snapshot_correlation.py b/tests/debugger/test_debugger_snapshot_correlation.py index 02cf424c40e..4b7b38bef49 100644 --- a/tests/debugger/test_debugger_snapshot_correlation.py +++ b/tests/debugger/test_debugger_snapshot_correlation.py @@ -66,8 +66,7 @@ def test_coordinated_per_trace_sampling(self): probes_by_trace: dict[str, set[str]] = {} for probe_id in self.probe_ids: for snapshot in self.probe_snapshots.get(probe_id, []): - # Tolerate the nested dd object (Python/Node/Ruby nest) and the flat dotted - # key (Java/.NET/Go/Ruby dotted) for the trace id. + # Tracers send either a nested dd object or a flat dd.trace_id key. trace_id = snapshot.get("dd", {}).get("trace_id") or snapshot.get("dd.trace_id") assert trace_id, f"snapshot for probe {probe_id} is missing dd.trace_id on the wire" probes_by_trace.setdefault(trace_id, set()).add(probe_id)