From 194539dc518f1ea2b30845237457240e96e1355c Mon Sep 17 00:00:00 2001 From: "helen.han" Date: Thu, 24 Sep 2026 18:30:34 -0400 Subject: [PATCH 01/10] Add parametric test for UFC serialId threaded through flagMetadata Every UFC split has a stable per-org serialId used by downstream consumers (APM span enrichment, exposure caching, evaluation-metrics correlation) to identify the exact (allocation, variant) pair that was assigned. Today only the Node server SDK (openfeature-js-client#269) exposes this via flagMetadata["__dd_split_serial_id"]; this test generalizes the contract so every SDK can be validated the same way. Adds a dedicated fixture covering: a plain serialId, serialId=0 (a valid ID that must not be dropped by a truthiness check), a split with no serialId (must not fabricate one), and two allocations with different serialIds selected via targeting. All languages are marked missing_feature: verified directly against the Node.js and Go parametric weblog handlers that neither returns flagMetadata from POST /ffe/evaluate today. The new feature_flags_serial_id_metadata marker uses NOT_REPORTED_ID pending a real feature-parity page. --- manifests/cpp.yml | 1 + manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/php.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + .../test_ffe/serial-id-metadata-flags.json | 97 +++++++ .../test_ffe/test_serial_id_metadata.py | 257 ++++++++++++++++++ utils/_features.py | 8 + 11 files changed, 370 insertions(+) create mode 100644 tests/parametric/test_ffe/serial-id-metadata-flags.json create mode 100644 tests/parametric/test_ffe/test_serial_id_metadata.py diff --git a/manifests/cpp.yml b/manifests/cpp.yml index f47379886b7..3dbff5bfdb1 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -124,6 +124,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: missing_feature (Automatic baggage propagation is not implemented) tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (cpp is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=2.1.1' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 1a06585da38..63d456a3e5e 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -965,6 +965,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: missing_feature (FFL-2703 tracks .NET agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: missing_feature tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_valid: missing_feature (Need to remove b3=b3multi alias) tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_inject_valid: missing_feature (Need to remove b3=b3multi alias) diff --git a/manifests/golang.yml b/manifests/golang.yml index cdf890f8a9d..66f2acc104c 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1631,6 +1631,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (DD_TRACE_PROPAGATION_EXTRACT_FIRST=true is unsupported; dd-trace-go panics when extraction fails) tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=2.10.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/java.yml b/manifests/java.yml index 3dfa97d6905..fd289b19138 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3963,6 +3963,7 @@ manifest: : v1.65.0 tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0 + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=1.64.2' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 9d1025d7d09..3476d912f50 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2348,6 +2348,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: incomplete_test_app (The parametric test app does not preserve baggage after extraction) tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0 + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: "missing_feature (dd-trace-js#8343)" tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_invalid: # TODO: a lower version might be supported - declaration: missing_feature (Need to remove b3=b3multi alias) diff --git a/manifests/php.yml b/manifests/php.yml index 266887892a9..621695f0a7f 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1476,6 +1476,7 @@ manifest: : missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=1.23.3' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/python.yml b/manifests/python.yml index cee16cef26a..405640a64fc 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2005,6 +2005,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: v2.20.0 tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0 + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=4.9.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/ruby.yml b/manifests/ruby.yml index e5c9c253985..0e8575362dd 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2419,6 +2419,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_ffe/test_configuration_sources.py: missing_feature (FFL-2701 tracks Ruby agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0 + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: missing_feature tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_extract_invalid: # Easy win for all weblogs and version 2.27.0 - declaration: missing_feature (Missing for 2.x) diff --git a/tests/parametric/test_ffe/serial-id-metadata-flags.json b/tests/parametric/test_ffe/serial-id-metadata-flags.json new file mode 100644 index 00000000000..35bf0ab4c74 --- /dev/null +++ b/tests/parametric/test_ffe/serial-id-metadata-flags.json @@ -0,0 +1,97 @@ +{ + "createdAt": "2024-01-01T00:00:00.000Z", + "format": "SERVER", + "environment": { "name": "Test" }, + "flags": { + "serial-id-basic-flag": { + "key": "serial-id-basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { "key": "on", "value": true }, + "off": { "key": "off", "value": false } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { "variationKey": "on", "shards": [], "serialId": 42 } + ], + "doLog": true + } + ] + }, + "serial-id-zero-flag": { + "key": "serial-id-zero-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { "key": "on", "value": true }, + "off": { "key": "off", "value": false } + }, + "allocations": [ + { + "key": "zero-allocation", + "rules": [], + "splits": [ + { "variationKey": "on", "shards": [], "serialId": 0 } + ], + "doLog": true + } + ] + }, + "serial-id-missing-flag": { + "key": "serial-id-missing-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { "key": "control", "value": "control" }, + "treatment": { "key": "treatment", "value": "treatment" } + }, + "allocations": [ + { + "key": "no-serial-allocation", + "rules": [], + "splits": [ + { "variationKey": "control", "shards": [] } + ], + "doLog": true + } + ] + }, + "serial-id-targeting-flag": { + "key": "serial-id-targeting-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "vip": { "key": "vip", "value": "vip" }, + "standard": { "key": "standard", "value": "standard" } + }, + "allocations": [ + { + "key": "vip-allocation", + "rules": [ + { + "conditions": [ + { "attribute": "segment", "operator": "ONE_OF", "value": ["vip"] } + ] + } + ], + "splits": [ + { "variationKey": "vip", "shards": [], "serialId": 201 } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { "variationKey": "standard", "shards": [], "serialId": 200 } + ], + "doLog": true + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py new file mode 100644 index 00000000000..2c74d124541 --- /dev/null +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -0,0 +1,257 @@ +"""Test that UFC split serial IDs are threaded through OpenFeature flagMetadata. + +Every evaluated (allocation, variant) pair has a stable per-org serial ID +(`Split.serialId` in the UFC wire format). Downstream consumers -- APM span +enrichment, exposure caching, evaluation-metrics correlation -- need that +serial ID without re-deriving it from (flag_key, allocation_key, variant_key). + +The contract under test: the SDK's OpenFeature EvaluationDetails-equivalent +object must expose the selected split's serial ID via `flagMetadata` under +the internal key `__dd_split_serial_id`, matching the value present in the +UFC config pushed over Remote Config. This is the field +`openfeature-js-client#269` introduced for the Node server SDK; this test +generalizes the contract to every parametric-tested SDK. + +This module tests: +1. flagMetadata carries the exact serialId of the selected split. +2. Zero is a valid serial ID and must not be treated as "missing" (falsy-check trap). +3. A split with no serialId must not fabricate one; flagMetadata omits the key or reports null. +4. flagMetadata reflects whichever split targeting actually selected, not always the first one. +5. Repeated evaluations of the same context return the same serial ID. +""" + +import json +import time +from pathlib import Path +from typing import Any + +import pytest + +from utils import features, scenarios +from utils.dd_constants import RemoteConfigApplyState +from utils.docker_fixtures import TestAgentAPI +from tests.parametric.conftest import APMLibrary + +RC_PRODUCT = "FFE_FLAGS" +RC_PATH = f"datadog/2/{RC_PRODUCT}" +FFE_READY_RETRY_ATTEMPTS = 10 +FFE_READY_RETRY_INTERVAL_SECONDS = 0.2 + +SERIAL_ID_METADATA_KEY = "__dd_split_serial_id" + +parametrize = pytest.mark.parametrize + + +def _load_serial_id_metadata_fixture() -> dict[str, Any]: + """Load the UFC fixture file for serial ID metadata tests.""" + fixture_path = Path(__file__).parent / "serial-id-metadata-flags.json" + + if not fixture_path.exists(): + pytest.skip(f"Fixture file not found: {fixture_path}") + + with fixture_path.open() as f: + return json.load(f) + + +UFC_SERIAL_ID_METADATA_DATA = _load_serial_id_metadata_fixture() + +DEFAULT_ENVVARS = { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", +} + + +def _set_and_wait_ffe_rc( + test_agent: TestAgentAPI, ufc_data: dict[str, Any], config_id: str | None = None +) -> dict[str, Any]: + """Set FFE Remote Config and wait for it to be acknowledged.""" + if not config_id: + config_id = str(hash(json.dumps(ufc_data, sort_keys=True))) + + test_agent.set_remote_config(path=f"{RC_PATH}/{config_id}/config", payload=ufc_data) + return test_agent.wait_for_rc_apply_state(RC_PRODUCT, state=RemoteConfigApplyState.ACKNOWLEDGED, clear=True) + + +def _is_ffe_waiting_for_rc(result: dict[str, Any]) -> bool: + provider_state = result.get("providerState") + return result.get("errorCode") == "PROVIDER_NOT_READY" or ( + isinstance(provider_state, dict) and provider_state.get("hasConfig") is False + ) + + +def _ffe_evaluate_with_rc_retry( + test_library: APMLibrary, + *, + flag: str, + variation_type: str, + default_value: bool | str | float | dict[str, Any], + targeting_key: str, + attributes: dict[str, Any] | None = None, +) -> dict[str, Any]: + result = test_library.ffe_evaluate( + flag=flag, + variation_type=variation_type, + default_value=default_value, + targeting_key=targeting_key, + attributes=attributes, + ) + for _ in range(FFE_READY_RETRY_ATTEMPTS - 1): + if not _is_ffe_waiting_for_rc(result): + return result + time.sleep(FFE_READY_RETRY_INTERVAL_SECONDS) + result = test_library.ffe_evaluate( + flag=flag, + variation_type=variation_type, + default_value=default_value, + targeting_key=targeting_key, + attributes=attributes, + ) + + return result + + +@scenarios.parametric +@features.feature_flags_serial_id_metadata +class Test_FFE_Serial_Id_Metadata: + """Test that the selected split's serial ID is threaded through flagMetadata.""" + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + """The serial ID of the selected split must appear in flagMetadata.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-basic-flag", + variation_type="BOOLEAN", + default_value=False, + targeting_key="user-1", + ) + assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" + + flag_metadata = result.get("flagMetadata") or {} + assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == 42, ( + f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == 42 for the selected split, " + f"got flagMetadata={flag_metadata}" + ) + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_zero_is_not_treated_as_missing( + self, test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + """Serial ID 0 is a real, valid ID and must survive falsy-value checks.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-zero-flag", + variation_type="BOOLEAN", + default_value=False, + targeting_key="user-1", + ) + assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" + + flag_metadata = result.get("flagMetadata") or {} + assert SERIAL_ID_METADATA_KEY in flag_metadata, ( + f"flagMetadata is missing '{SERIAL_ID_METADATA_KEY}' entirely; a 0 serial ID must not be " + f"dropped by a truthiness check. flagMetadata={flag_metadata}" + ) + assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == 0, ( + f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == 0, got flagMetadata={flag_metadata}" + ) + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_absent_when_split_has_no_serial_id( + self, test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + """A split without a serialId must not have one fabricated for it.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-missing-flag", + variation_type="STRING", + default_value="default", + targeting_key="user-1", + ) + assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" + + flag_metadata = result.get("flagMetadata") or {} + assert flag_metadata.get(SERIAL_ID_METADATA_KEY) is None, ( + f"Expected no serial ID for a split without one, got " + f"flagMetadata['{SERIAL_ID_METADATA_KEY}']={flag_metadata.get(SERIAL_ID_METADATA_KEY)!r}" + ) + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_matches_selected_split_under_targeting( + self, test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + """flagMetadata must reflect whichever split targeting actually selected.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + vip_result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-targeting-flag", + variation_type="STRING", + default_value="default", + targeting_key="user-vip", + attributes={"segment": "vip"}, + ) + standard_result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-targeting-flag", + variation_type="STRING", + default_value="default", + targeting_key="user-standard", + attributes={"segment": "standard"}, + ) + + for label, result in (("vip", vip_result), ("standard", standard_result)): + assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data ({label}); result={result}" + + vip_metadata = vip_result.get("flagMetadata") or {} + standard_metadata = standard_result.get("flagMetadata") or {} + + assert vip_result.get("value") == "vip" + assert vip_metadata.get(SERIAL_ID_METADATA_KEY) == 201, ( + f"VIP targeting should select the split with serialId 201, got flagMetadata={vip_metadata}" + ) + + assert standard_result.get("value") == "standard" + assert standard_metadata.get(SERIAL_ID_METADATA_KEY) == 200, ( + f"Non-VIP targeting should select the split with serialId 200, got flagMetadata={standard_metadata}" + ) + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_stable_across_repeated_evaluations( + self, test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + """The same context evaluated twice must yield the same serial ID.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + first = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-basic-flag", + variation_type="BOOLEAN", + default_value=False, + targeting_key="user-repeat", + ) + second = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-basic-flag", + variation_type="BOOLEAN", + default_value=False, + targeting_key="user-repeat", + ) + + first_id = (first.get("flagMetadata") or {}).get(SERIAL_ID_METADATA_KEY) + second_id = (second.get("flagMetadata") or {}).get(SERIAL_ID_METADATA_KEY) + assert first_id == second_id == 42, ( + f"Expected stable serial ID 42 across repeated evaluations, got {first_id} then {second_id}" + ) diff --git a/utils/_features.py b/utils/_features.py index 0cd8b4acf13..38f5c3f8c6b 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2729,6 +2729,14 @@ def feature_flags_event_enrichment(test_object): """ return _mark_test_object(test_object, feature_id=551, owner=_Owner.ffe) + @staticmethod + def feature_flags_serial_id_metadata(test_object): + """Feature Flags Serial ID Metadata (UFC split serialId threaded through flagMetadata) + + No feature-parity page yet; using NOT_REPORTED_ID until one is filed. + """ + return _mark_test_object(test_object, feature_id=NOT_REPORTED_ID, owner=_Owner.ffe) + @staticmethod def appsec_extended_data_collection(test_object): """AppSec supports extended data collection including headers and body From cee928ab864178215025689b4fe0a9eb2d97d401 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 25 Sep 2026 00:59:20 +0000 Subject: [PATCH 02/10] Fix serial-ID metadata test discovery and validation Exclude the UFC configuration from evaluation-case discovery, disable the new suite for Rust, and require successful evaluation before accepting an absent serial ID. Apply the repository formatter to the new test module. Environment: Datadog workspace --- manifests/rust.yml | 1 + tests/parametric/test_ffe/test_dynamic_evaluation.py | 2 +- tests/parametric/test_ffe/test_serial_id_metadata.py | 9 +++++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/manifests/rust.yml b/manifests/rust.yml index 0a2d4fdca38..3cc8c2c7273 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -138,6 +138,7 @@ manifest: tests/parametric/test_extract_behavior.py: '>=0.4.1-dev' tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (rust is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: missing_feature tests/parametric/test_headers_b3.py: '>=0.2.1' # Modified by easy win activation script tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_extract_valid: missing_feature # Created by easy win activation script diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 7d48a1e0b7c..5ceea0560c9 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -41,7 +41,7 @@ def _get_test_case_files() -> list[str]: return [] # Exclude base fixtures that aren't test cases - excluded = {"flags-v1.json", "span-enrichment-flags.json"} + excluded = {"flags-v1.json", "serial-id-metadata-flags.json", "span-enrichment-flags.json"} return [f.name for f in test_data_dir.iterdir() if f.suffix == ".json" and f.name not in excluded] diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 2c74d124541..94cdbffc9ea 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -138,9 +138,7 @@ def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test ) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) - def test_serial_id_zero_is_not_treated_as_missing( - self, test_agent: TestAgentAPI, test_library: APMLibrary - ) -> None: + def test_serial_id_zero_is_not_treated_as_missing(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: """Serial ID 0 is a real, valid ID and must survive falsy-value checks.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" @@ -179,6 +177,9 @@ def test_serial_id_absent_when_split_has_no_serial_id( targeting_key="user-1", ) assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" + assert result.get("errorCode") in {None, ""}, f"FFE evaluation failed; result={result}" + assert result.get("reason") != "ERROR", f"FFE evaluation failed; result={result}" + assert result.get("value") == "control", f"Expected the configured control variation; result={result}" flag_metadata = result.get("flagMetadata") or {} assert flag_metadata.get(SERIAL_ID_METADATA_KEY) is None, ( @@ -190,7 +191,7 @@ def test_serial_id_absent_when_split_has_no_serial_id( def test_serial_id_matches_selected_split_under_targeting( self, test_agent: TestAgentAPI, test_library: APMLibrary ) -> None: - """flagMetadata must reflect whichever split targeting actually selected.""" + """FlagMetadata must reflect whichever split targeting actually selected.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" From 55d147d0806f010871fe906fe9fe2544543d1b59 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Fri, 25 Sep 2026 16:30:50 -0400 Subject: [PATCH 03/10] Require flagMetadata in serial-ID evaluations Fail evaluations that omit flagMetadata while allowing null or empty metadata when a split has no serial ID. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- .../test_ffe/test_serial_id_metadata.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 94cdbffc9ea..9c77b0a59a6 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -15,7 +15,8 @@ This module tests: 1. flagMetadata carries the exact serialId of the selected split. 2. Zero is a valid serial ID and must not be treated as "missing" (falsy-check trap). -3. A split with no serialId must not fabricate one; flagMetadata omits the key or reports null. +3. Evaluation details always expose flagMetadata; for a split with no serialId, the map is null/empty + or omits the serial ID key. 4. flagMetadata reflects whichever split targeting actually selected, not always the first one. 5. Repeated evaluations of the same context return the same serial ID. """ @@ -80,6 +81,17 @@ def _is_ffe_waiting_for_rc(result: dict[str, Any]) -> bool: ) +def _require_flag_metadata(result: dict[str, Any]) -> dict[str, Any]: + """Require evaluation details to expose flagMetadata, allowing a null map.""" + assert "flagMetadata" in result, f"FFE evaluation details did not emit flagMetadata; result={result}" + + flag_metadata = result["flagMetadata"] + assert flag_metadata is None or isinstance(flag_metadata, dict), ( + f"Expected flagMetadata to be a map or null, got {flag_metadata!r}" + ) + return flag_metadata or {} + + def _ffe_evaluate_with_rc_retry( test_library: APMLibrary, *, @@ -131,7 +143,7 @@ def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test ) assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" - flag_metadata = result.get("flagMetadata") or {} + flag_metadata = _require_flag_metadata(result) assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == 42, ( f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == 42 for the selected split, " f"got flagMetadata={flag_metadata}" @@ -152,7 +164,7 @@ def test_serial_id_zero_is_not_treated_as_missing(self, test_agent: TestAgentAPI ) assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" - flag_metadata = result.get("flagMetadata") or {} + flag_metadata = _require_flag_metadata(result) assert SERIAL_ID_METADATA_KEY in flag_metadata, ( f"flagMetadata is missing '{SERIAL_ID_METADATA_KEY}' entirely; a 0 serial ID must not be " f"dropped by a truthiness check. flagMetadata={flag_metadata}" @@ -181,7 +193,7 @@ def test_serial_id_absent_when_split_has_no_serial_id( assert result.get("reason") != "ERROR", f"FFE evaluation failed; result={result}" assert result.get("value") == "control", f"Expected the configured control variation; result={result}" - flag_metadata = result.get("flagMetadata") or {} + flag_metadata = _require_flag_metadata(result) assert flag_metadata.get(SERIAL_ID_METADATA_KEY) is None, ( f"Expected no serial ID for a split without one, got " f"flagMetadata['{SERIAL_ID_METADATA_KEY}']={flag_metadata.get(SERIAL_ID_METADATA_KEY)!r}" @@ -215,8 +227,8 @@ def test_serial_id_matches_selected_split_under_targeting( for label, result in (("vip", vip_result), ("standard", standard_result)): assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data ({label}); result={result}" - vip_metadata = vip_result.get("flagMetadata") or {} - standard_metadata = standard_result.get("flagMetadata") or {} + vip_metadata = _require_flag_metadata(vip_result) + standard_metadata = _require_flag_metadata(standard_result) assert vip_result.get("value") == "vip" assert vip_metadata.get(SERIAL_ID_METADATA_KEY) == 201, ( @@ -251,8 +263,8 @@ def test_serial_id_stable_across_repeated_evaluations( targeting_key="user-repeat", ) - first_id = (first.get("flagMetadata") or {}).get(SERIAL_ID_METADATA_KEY) - second_id = (second.get("flagMetadata") or {}).get(SERIAL_ID_METADATA_KEY) + first_id = _require_flag_metadata(first).get(SERIAL_ID_METADATA_KEY) + second_id = _require_flag_metadata(second).get(SERIAL_ID_METADATA_KEY) assert first_id == second_id == 42, ( f"Expected stable serial ID 42 across repeated evaluations, got {first_id} then {second_id}" ) From b7eb74b708c28ba10a594b810e5be26cb48dd2b5 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Mon, 28 Sep 2026 15:54:36 -0400 Subject: [PATCH 04/10] Require missing serial ID to be omitted Tighten the missing-serial-ID test so an SDK cannot serialize an absent UFC serial ID as a null metadata entry. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- tests/parametric/test_ffe/test_serial_id_metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 9c77b0a59a6..536094775d1 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -15,8 +15,8 @@ This module tests: 1. flagMetadata carries the exact serialId of the selected split. 2. Zero is a valid serial ID and must not be treated as "missing" (falsy-check trap). -3. Evaluation details always expose flagMetadata; for a split with no serialId, the map is null/empty - or omits the serial ID key. +3. Evaluation details always expose flagMetadata; for a split with no serialId, the map must not + contain the serial ID key. 4. flagMetadata reflects whichever split targeting actually selected, not always the first one. 5. Repeated evaluations of the same context return the same serial ID. """ @@ -194,9 +194,9 @@ def test_serial_id_absent_when_split_has_no_serial_id( assert result.get("value") == "control", f"Expected the configured control variation; result={result}" flag_metadata = _require_flag_metadata(result) - assert flag_metadata.get(SERIAL_ID_METADATA_KEY) is None, ( - f"Expected no serial ID for a split without one, got " - f"flagMetadata['{SERIAL_ID_METADATA_KEY}']={flag_metadata.get(SERIAL_ID_METADATA_KEY)!r}" + assert SERIAL_ID_METADATA_KEY not in flag_metadata, ( + f"Expected flagMetadata to omit '{SERIAL_ID_METADATA_KEY}' for a split without a serial ID, " + f"got flagMetadata={flag_metadata}" ) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) From 37b4e44bff73ca675f54ced40a003360e273b905 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Mon, 28 Sep 2026 16:00:55 -0400 Subject: [PATCH 05/10] Enable span enrichment in serial ID tests Exercise serial-ID flag metadata under the required span-enrichment configuration for positive, zero, and missing UFC serial IDs. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- tests/parametric/test_ffe/test_serial_id_metadata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 536094775d1..8bcea2a48e6 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -5,10 +5,10 @@ enrichment, exposure caching, evaluation-metrics correlation -- need that serial ID without re-deriving it from (flag_key, allocation_key, variant_key). -The contract under test: the SDK's OpenFeature EvaluationDetails-equivalent -object must expose the selected split's serial ID via `flagMetadata` under -the internal key `__dd_split_serial_id`, matching the value present in the -UFC config pushed over Remote Config. This is the field +The contract under test: when span enrichment is enabled, the SDK's OpenFeature +EvaluationDetails-equivalent object must expose the selected split's serial ID +via `flagMetadata` under the internal key `__dd_split_serial_id`, matching the +value present in the UFC config pushed over Remote Config. This is the field `openfeature-js-client#269` introduced for the Node server SDK; this test generalizes the contract to every parametric-tested SDK. @@ -58,6 +58,7 @@ def _load_serial_id_metadata_fixture() -> dict[str, Any]: DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": "true", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", } From bed7532a03f7bd12ae3d6567081a2c90b46a7b4e Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Mon, 28 Sep 2026 17:17:59 -0400 Subject: [PATCH 06/10] Use approved pytest facade in serial ID tests Fix the import-policy failure when the PR is tested against current main. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- tests/parametric/test_ffe/test_serial_id_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 8bcea2a48e6..c893c067479 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -26,7 +26,7 @@ from pathlib import Path from typing import Any -import pytest +from utils import pytest from utils import features, scenarios from utils.dd_constants import RemoteConfigApplyState From 9547cefa87bc9a4e6ba47f0aa7678dd695a542b9 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Mon, 28 Sep 2026 17:26:01 -0400 Subject: [PATCH 07/10] Include pytest facade on serial ID test branch Bring in the exact upstream facade so imports work on the PR head as well as merged main. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- utils/pytest.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 utils/pytest.py diff --git a/utils/pytest.py b/utils/pytest.py new file mode 100644 index 00000000000..cb74ce04b15 --- /dev/null +++ b/utils/pytest.py @@ -0,0 +1,62 @@ +"""Approved pytest API for system tests. + +Test modules should import this facade with ``from utils import pytest``. +The facade intentionally exposes only the pytest APIs used by this repository. +In particular, force-skip markers are available only through the semantic +decorators exported by :mod:`utils`. +""" + +import pytest as _pytest # noqa: PT013 - keep the underlying pytest module private + + +Config = _pytest.Config +FixtureRequest = _pytest.FixtureRequest +Item = _pytest.Item +Mark = _pytest.Mark +MarkDecorator = _pytest.MarkDecorator +MonkeyPatch = _pytest.MonkeyPatch +CaptureFixture = _pytest.CaptureFixture +Session = _pytest.Session +CallInfo = _pytest.CallInfo +TempPathFactory = _pytest.TempPathFactory + +approx = _pytest.approx +exit = _pytest.exit # noqa: A001 - preserve the public pytest API name +fail = _pytest.fail +fixture = _pytest.fixture +param = _pytest.param +raises = _pytest.raises +skip = _pytest.skip +hookimpl = _pytest.hookimpl + + +class _AllowedMarks: + """Pytest markers approved for direct use by system tests.""" + + features = _pytest.mark.features + parametrize = _pytest.mark.parametrize + scenario = _pytest.mark.scenario + usefixtures = _pytest.mark.usefixtures + + +mark = _AllowedMarks() + +__all__ = [ + "CallInfo", + "Config", + "FixtureRequest", + "Item", + "Mark", + "MarkDecorator", + "MonkeyPatch", + "Session", + "approx", + "exit", + "fail", + "fixture", + "hookimpl", + "mark", + "param", + "raises", + "skip", +] From 7846a96bbc366c6cfeb5da3fc3857fe4502ca261 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Tue, 29 Sep 2026 11:56:14 -0400 Subject: [PATCH 08/10] Expose .NET FFE details in parametric adapter Forward provider metadata through public OpenFeature accessors; retain the serial ID test gate pending integer support. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- manifests/dotnet.yml | 2 +- .../dotnet/parametric/ApmTestApi.csproj | 1 + .../parametric/Endpoints/ApmTestApiFfe.cs | 119 ++++++++++++++++++ .../build/docker/dotnet/parametric/Program.cs | 1 + 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiFfe.cs diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 63d456a3e5e..8969dc1e298 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -965,7 +965,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: missing_feature (FFL-2703 tracks .NET agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script - tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature (Provider serial ID metadata is string-valued; integer support and release gate pending) tests/parametric/test_ffe/test_span_enrichment.py: missing_feature tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_valid: missing_feature (Need to remove b3=b3multi alias) tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_inject_valid: missing_feature (Need to remove b3=b3multi alias) diff --git a/utils/build/docker/dotnet/parametric/ApmTestApi.csproj b/utils/build/docker/dotnet/parametric/ApmTestApi.csproj index 9b1f4807601..b2fcbc99bf0 100644 --- a/utils/build/docker/dotnet/parametric/ApmTestApi.csproj +++ b/utils/build/docker/dotnet/parametric/ApmTestApi.csproj @@ -13,6 +13,7 @@ + diff --git a/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiFfe.cs b/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiFfe.cs new file mode 100644 index 00000000000..6836894540e --- /dev/null +++ b/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiFfe.cs @@ -0,0 +1,119 @@ +using Datadog.FeatureFlags.OpenFeature; +using OpenFeature; +using OpenFeature.Constant; +using OpenFeature.Model; +using System.Text.Json; + +namespace ApmTestApi.Endpoints; + +public static class ApmTestApiFfe +{ + private static FeatureClient? _client; + + public static void MapEndpoints(WebApplication app) + { + // Start configuration delivery before /ffe/start: tests wait for the RC + // acknowledgement before calling that endpoint. Do not block app startup. + var enabled = Environment.GetEnvironmentVariable("DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"); + var initialization = enabled is "true" or "1" ? InitializeAsync() : Task.CompletedTask; + app.MapPost("/ffe/start", async () => + { + await initialization; + return _client is null ? Results.StatusCode(503) : Results.Ok(true); + }); + app.MapPost("/ffe/evaluate", EvaluateAsync); + } + + private static async Task InitializeAsync() + { + await Api.Instance.SetProviderAsync(new DatadogProvider()); + _client = Api.Instance.GetClient(); + } + + private static async Task EvaluateAsync(JsonElement request) + { + if (_client is null) + { + return Results.Json(new { errorCode = "PROVIDER_NOT_READY", flagMetadata = new { } }); + } + + var context = EvaluationContext.Builder(); + if (request.TryGetProperty("targetingKey", out var targetingKey) && targetingKey.ValueKind != JsonValueKind.Null) + { + context.SetTargetingKey(targetingKey.GetString()!); + } + if (request.TryGetProperty("attributes", out var attributes) && attributes.ValueKind == JsonValueKind.Object) + { + foreach (var property in attributes.EnumerateObject()) + { + context.Set(property.Name, ToValue(property.Value)); + } + } + + var evaluationContext = context.Build(); + var flag = request.GetProperty("flag").GetString()!; + var defaultValue = request.GetProperty("defaultValue"); + return request.GetProperty("variationType").GetString()?.ToUpperInvariant() switch + { + "BOOLEAN" => ToResponse(await _client.GetBooleanDetailsAsync(flag, defaultValue.GetBoolean(), evaluationContext)), + "STRING" => ToResponse(await _client.GetStringDetailsAsync(flag, defaultValue.GetString()!, evaluationContext)), + "INTEGER" => ToResponse(await _client.GetIntegerDetailsAsync(flag, defaultValue.GetInt32(), evaluationContext)), + "NUMERIC" => ToResponse(await _client.GetDoubleDetailsAsync(flag, defaultValue.GetDouble(), evaluationContext)), + "JSON" => ToResponse(await _client.GetObjectDetailsAsync(flag, ToValue(defaultValue), evaluationContext)), + _ => Results.BadRequest(new { error = "Unsupported variationType" }), + }; + } + + private static IResult ToResponse(FlagEvaluationDetails details) => Results.Json(new + { + value = details.Value is Value value ? ToPlainObject(value) : (object?)details.Value, + reason = details.Reason?.ToUpperInvariant(), + variant = details.Variant, + errorCode = details.ErrorType == ErrorType.None ? null : JsonNamingPolicy.SnakeCaseUpper.ConvertName(details.ErrorType.ToString()), + errorMessage = details.ErrorMessage, + flagMetadata = ReadMetadata(details.FlagMetadata), + }); + + private static Dictionary ReadMetadata(ImmutableMetadata? metadata) + { + var result = new Dictionary(); + if (metadata is null) + { + return result; + } + + // OpenFeature .NET exposes typed accessors, but no metadata enumeration. + // Forward the provider's known keys without coercing values or reading UFC. + foreach (var key in new[] { "__dd_allocation_key", "__dd_do_log", "__dd_split_serial_id" }) + { + object? value = metadata.GetString(key) ?? (object?)metadata.GetInt(key) + ?? metadata.GetDouble(key) ?? (object?)metadata.GetBool(key); + if (value is not null) + { + result[key] = value; + } + } + return result; + } + + private static Value ToValue(JsonElement value) => value.ValueKind switch + { + JsonValueKind.True => new Value(true), + JsonValueKind.False => new Value(false), + JsonValueKind.String => new Value(value.GetString()!), + JsonValueKind.Number => new Value(value.GetDouble()), + JsonValueKind.Array => new Value(value.EnumerateArray().Select(ToValue).ToList()), + JsonValueKind.Object => new Value(new Structure(value.EnumerateObject().ToDictionary(p => p.Name, p => ToValue(p.Value)))), + _ => new Value(), + }; + + private static object? ToPlainObject(Value value) + { + if (value.IsNull) return null; + if (value.IsBoolean) return value.AsBoolean; + if (value.IsString) return value.AsString; + if (value.IsNumber) return value.AsDouble; + if (value.IsList) return value.AsList!.Select(ToPlainObject).ToArray(); + return value.AsStructure!.AsDictionary().ToDictionary(p => p.Key, p => ToPlainObject(p.Value)); + } +} diff --git a/utils/build/docker/dotnet/parametric/Program.cs b/utils/build/docker/dotnet/parametric/Program.cs index ffc43036ed8..ee5bdadeae9 100644 --- a/utils/build/docker/dotnet/parametric/Program.cs +++ b/utils/build/docker/dotnet/parametric/Program.cs @@ -11,6 +11,7 @@ // Map endpoints ApmTestApi.Endpoints.ApmTestApi.MapApmTraceEndpoints(app, logger); ApmTestApi.Endpoints.ApmTestApiOtel.MapApmOtelEndpoints(app, otelLogger); +ApmTestApi.Endpoints.ApmTestApiFfe.MapEndpoints(app); if (!int.TryParse(Environment.GetEnvironmentVariable("APM_TEST_CLIENT_SERVER_PORT"), out var port)) { From b2d7b019f8a043391a5ad0303072d4be9661b2bb Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Tue, 29 Sep 2026 16:32:11 -0400 Subject: [PATCH 09/10] Allow string serial ID metadata only for .NET Enable the verified .NET serial ID tests while preserving numeric expectations for other SDKs and key omission for missing IDs. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- manifests/dotnet.yml | 2 +- .../test_ffe/test_serial_id_metadata.py | 30 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 8969dc1e298..08e325cd8c2 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -965,7 +965,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: missing_feature (FFL-2703 tracks .NET agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script - tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature (Provider serial ID metadata is string-valued; integer support and release gate pending) + tests/parametric/test_ffe/test_serial_id_metadata.py: v3.54.0 tests/parametric/test_ffe/test_span_enrichment.py: missing_feature tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_valid: missing_feature (Need to remove b3=b3multi alias) tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_inject_valid: missing_feature (Need to remove b3=b3multi alias) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index c893c067479..bfd599a8b9d 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -8,7 +8,9 @@ The contract under test: when span enrichment is enabled, the SDK's OpenFeature EvaluationDetails-equivalent object must expose the selected split's serial ID via `flagMetadata` under the internal key `__dd_split_serial_id`, matching the -value present in the UFC config pushed over Remote Config. This is the field +value present in the UFC config pushed over Remote Config. Only .NET represents +the serial ID as a decimal string, preserving Int64 IDs through OpenFeature's +metadata API; all other SDKs must return a numeric value. This is the field `openfeature-js-client#269` introduced for the Node server SDK; this test generalizes the contract to every parametric-tested SDK. @@ -28,7 +30,7 @@ from utils import pytest -from utils import features, scenarios +from utils import context, features, scenarios from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary @@ -43,6 +45,11 @@ parametrize = pytest.mark.parametrize +def _expected_serial_id(serial_id: int) -> int | str: + """Only .NET uses strings to preserve Int64 IDs in OpenFeature metadata.""" + return str(serial_id) if context.library.name == "dotnet" else serial_id + + def _load_serial_id_metadata_fixture() -> dict[str, Any]: """Load the UFC fixture file for serial ID metadata tests.""" fixture_path = Path(__file__).parent / "serial-id-metadata-flags.json" @@ -145,8 +152,9 @@ def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" flag_metadata = _require_flag_metadata(result) - assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == 42, ( - f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == 42 for the selected split, " + expected_id = _expected_serial_id(42) + assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == expected_id, ( + f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == {expected_id!r} for the selected split, " f"got flagMetadata={flag_metadata}" ) @@ -170,8 +178,9 @@ def test_serial_id_zero_is_not_treated_as_missing(self, test_agent: TestAgentAPI f"flagMetadata is missing '{SERIAL_ID_METADATA_KEY}' entirely; a 0 serial ID must not be " f"dropped by a truthiness check. flagMetadata={flag_metadata}" ) - assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == 0, ( - f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == 0, got flagMetadata={flag_metadata}" + expected_id = _expected_serial_id(0) + assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == expected_id, ( + f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == {expected_id!r}, got flagMetadata={flag_metadata}" ) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) @@ -232,12 +241,12 @@ def test_serial_id_matches_selected_split_under_targeting( standard_metadata = _require_flag_metadata(standard_result) assert vip_result.get("value") == "vip" - assert vip_metadata.get(SERIAL_ID_METADATA_KEY) == 201, ( + assert vip_metadata.get(SERIAL_ID_METADATA_KEY) == _expected_serial_id(201), ( f"VIP targeting should select the split with serialId 201, got flagMetadata={vip_metadata}" ) assert standard_result.get("value") == "standard" - assert standard_metadata.get(SERIAL_ID_METADATA_KEY) == 200, ( + assert standard_metadata.get(SERIAL_ID_METADATA_KEY) == _expected_serial_id(200), ( f"Non-VIP targeting should select the split with serialId 200, got flagMetadata={standard_metadata}" ) @@ -266,6 +275,7 @@ def test_serial_id_stable_across_repeated_evaluations( first_id = _require_flag_metadata(first).get(SERIAL_ID_METADATA_KEY) second_id = _require_flag_metadata(second).get(SERIAL_ID_METADATA_KEY) - assert first_id == second_id == 42, ( - f"Expected stable serial ID 42 across repeated evaluations, got {first_id} then {second_id}" + expected_id = _expected_serial_id(42) + assert first_id == second_id == expected_id, ( + f"Expected stable serial ID {expected_id!r} across repeated evaluations, got {first_id!r} then {second_id!r}" ) From 4e8124157172051f07a66ad86e03dd20887ea122 Mon Sep 17 00:00:00 2001 From: "helen.han" Date: Fri, 16 Oct 2026 16:15:42 +0000 Subject: [PATCH 10/10] Use Remote Config exclusively in serial ID metadata tests Start the provider without injecting the UFC fixture after Remote Config acknowledgement. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- tests/parametric/test_ffe/test_serial_id_metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index bfd599a8b9d..c10feff5803 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -140,7 +140,7 @@ class Test_FFE_Serial_Id_Metadata: def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: """The serial ID of the selected split must appear in flagMetadata.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" result = _ffe_evaluate_with_rc_retry( test_library, @@ -162,7 +162,7 @@ def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test def test_serial_id_zero_is_not_treated_as_missing(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: """Serial ID 0 is a real, valid ID and must survive falsy-value checks.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" result = _ffe_evaluate_with_rc_retry( test_library, @@ -189,7 +189,7 @@ def test_serial_id_absent_when_split_has_no_serial_id( ) -> None: """A split without a serialId must not have one fabricated for it.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" result = _ffe_evaluate_with_rc_retry( test_library, @@ -215,7 +215,7 @@ def test_serial_id_matches_selected_split_under_targeting( ) -> None: """FlagMetadata must reflect whichever split targeting actually selected.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" vip_result = _ffe_evaluate_with_rc_retry( test_library, @@ -256,7 +256,7 @@ def test_serial_id_stable_across_repeated_evaluations( ) -> None: """The same context evaluated twice must yield the same serial ID.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" first = _ffe_evaluate_with_rc_retry( test_library,