From 05b7c5f50a8a672cedc2200fd8a430001f484cd8 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Wed, 30 Sep 2026 12:24:38 -0400 Subject: [PATCH 1/8] Add parametric test for UFC serialId threaded through flagMetadata Every UFC split has a stable per-org serialId used by downstream consumers (APM span enrichment, exposure caching, evaluation-metrics correlation) to identify the exact (allocation, variant) pair that was assigned. Today only the Node server SDK (openfeature-js-client#269) exposes this via flagMetadata["__dd_split_serial_id"]; this test generalizes the contract so every SDK can be validated the same way. Adds a dedicated fixture covering: a plain serialId, serialId=0 (a valid ID that must not be dropped by a truthiness check), a split with no serialId (must not fabricate one), and two allocations with different serialIds selected via targeting. All languages are marked missing_feature: verified directly against the Node.js and Go parametric weblog handlers that neither returns flagMetadata from POST /ffe/evaluate today. The new feature_flags_serial_id_metadata marker uses NOT_REPORTED_ID pending a real feature-parity page. Co-authored-by: helen.han --- manifests/cpp.yml | 1 + manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/php.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + .../test_ffe/serial-id-metadata-flags.json | 97 +++++++ .../test_ffe/test_serial_id_metadata.py | 257 ++++++++++++++++++ utils/_features.py | 8 + 11 files changed, 370 insertions(+) create mode 100644 tests/parametric/test_ffe/serial-id-metadata-flags.json create mode 100644 tests/parametric/test_ffe/test_serial_id_metadata.py diff --git a/manifests/cpp.yml b/manifests/cpp.yml index 8aee9ed717e..fddd915329b 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -189,6 +189,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: missing_feature (Automatic baggage propagation is not implemented) tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (cpp is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=2.1.1' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index f89b1ff2675..c9f1b7de57c 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1183,6 +1183,7 @@ manifest: component_version: <3.55.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=3.54.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/golang.yml b/manifests/golang.yml index e6b95bdd6a1..df303b4643f 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -2149,6 +2149,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (DD_TRACE_PROPAGATION_EXTRACT_FIRST=true is unsupported; dd-trace-go panics when extraction fails) tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev # Easy win for all weblogs and version 2.10.1 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=2.10.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/java.yml b/manifests/java.yml index aae3a4ab270..7465820bd0a 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4242,6 +4242,7 @@ manifest: : v1.65.0 tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0 + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=1.64.2' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index decc3ea9051..5586936a69a 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2658,6 +2658,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: *ref_5_56_0 tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0 + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: # TODO: a lower version might be supported - declaration: missing_feature (dd-trace-js#8343) component_version: <6.19.0 diff --git a/manifests/php.yml b/manifests/php.yml index 5535c09a59c..583007e3228 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1598,6 +1598,7 @@ manifest: : missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=1.23.3' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/python.yml b/manifests/python.yml index 87863447b4c..29f159cb1bc 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2257,6 +2257,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: v2.20.0 tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0 + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=4.9.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/ruby.yml b/manifests/ruby.yml index fb65e50bd5c..79a9995ef79 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2679,6 +2679,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_ffe/test_configuration_sources.py: v2.46.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0 + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: missing_feature tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_extract_invalid: # Easy win for all weblogs and version 2.27.0 - declaration: missing_feature (Missing for 2.x) diff --git a/tests/parametric/test_ffe/serial-id-metadata-flags.json b/tests/parametric/test_ffe/serial-id-metadata-flags.json new file mode 100644 index 00000000000..35bf0ab4c74 --- /dev/null +++ b/tests/parametric/test_ffe/serial-id-metadata-flags.json @@ -0,0 +1,97 @@ +{ + "createdAt": "2024-01-01T00:00:00.000Z", + "format": "SERVER", + "environment": { "name": "Test" }, + "flags": { + "serial-id-basic-flag": { + "key": "serial-id-basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { "key": "on", "value": true }, + "off": { "key": "off", "value": false } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { "variationKey": "on", "shards": [], "serialId": 42 } + ], + "doLog": true + } + ] + }, + "serial-id-zero-flag": { + "key": "serial-id-zero-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { "key": "on", "value": true }, + "off": { "key": "off", "value": false } + }, + "allocations": [ + { + "key": "zero-allocation", + "rules": [], + "splits": [ + { "variationKey": "on", "shards": [], "serialId": 0 } + ], + "doLog": true + } + ] + }, + "serial-id-missing-flag": { + "key": "serial-id-missing-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { "key": "control", "value": "control" }, + "treatment": { "key": "treatment", "value": "treatment" } + }, + "allocations": [ + { + "key": "no-serial-allocation", + "rules": [], + "splits": [ + { "variationKey": "control", "shards": [] } + ], + "doLog": true + } + ] + }, + "serial-id-targeting-flag": { + "key": "serial-id-targeting-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "vip": { "key": "vip", "value": "vip" }, + "standard": { "key": "standard", "value": "standard" } + }, + "allocations": [ + { + "key": "vip-allocation", + "rules": [ + { + "conditions": [ + { "attribute": "segment", "operator": "ONE_OF", "value": ["vip"] } + ] + } + ], + "splits": [ + { "variationKey": "vip", "shards": [], "serialId": 201 } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { "variationKey": "standard", "shards": [], "serialId": 200 } + ], + "doLog": true + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py new file mode 100644 index 00000000000..2c74d124541 --- /dev/null +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -0,0 +1,257 @@ +"""Test that UFC split serial IDs are threaded through OpenFeature flagMetadata. + +Every evaluated (allocation, variant) pair has a stable per-org serial ID +(`Split.serialId` in the UFC wire format). Downstream consumers -- APM span +enrichment, exposure caching, evaluation-metrics correlation -- need that +serial ID without re-deriving it from (flag_key, allocation_key, variant_key). + +The contract under test: the SDK's OpenFeature EvaluationDetails-equivalent +object must expose the selected split's serial ID via `flagMetadata` under +the internal key `__dd_split_serial_id`, matching the value present in the +UFC config pushed over Remote Config. This is the field +`openfeature-js-client#269` introduced for the Node server SDK; this test +generalizes the contract to every parametric-tested SDK. + +This module tests: +1. flagMetadata carries the exact serialId of the selected split. +2. Zero is a valid serial ID and must not be treated as "missing" (falsy-check trap). +3. A split with no serialId must not fabricate one; flagMetadata omits the key or reports null. +4. flagMetadata reflects whichever split targeting actually selected, not always the first one. +5. Repeated evaluations of the same context return the same serial ID. +""" + +import json +import time +from pathlib import Path +from typing import Any + +import pytest + +from utils import features, scenarios +from utils.dd_constants import RemoteConfigApplyState +from utils.docker_fixtures import TestAgentAPI +from tests.parametric.conftest import APMLibrary + +RC_PRODUCT = "FFE_FLAGS" +RC_PATH = f"datadog/2/{RC_PRODUCT}" +FFE_READY_RETRY_ATTEMPTS = 10 +FFE_READY_RETRY_INTERVAL_SECONDS = 0.2 + +SERIAL_ID_METADATA_KEY = "__dd_split_serial_id" + +parametrize = pytest.mark.parametrize + + +def _load_serial_id_metadata_fixture() -> dict[str, Any]: + """Load the UFC fixture file for serial ID metadata tests.""" + fixture_path = Path(__file__).parent / "serial-id-metadata-flags.json" + + if not fixture_path.exists(): + pytest.skip(f"Fixture file not found: {fixture_path}") + + with fixture_path.open() as f: + return json.load(f) + + +UFC_SERIAL_ID_METADATA_DATA = _load_serial_id_metadata_fixture() + +DEFAULT_ENVVARS = { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", +} + + +def _set_and_wait_ffe_rc( + test_agent: TestAgentAPI, ufc_data: dict[str, Any], config_id: str | None = None +) -> dict[str, Any]: + """Set FFE Remote Config and wait for it to be acknowledged.""" + if not config_id: + config_id = str(hash(json.dumps(ufc_data, sort_keys=True))) + + test_agent.set_remote_config(path=f"{RC_PATH}/{config_id}/config", payload=ufc_data) + return test_agent.wait_for_rc_apply_state(RC_PRODUCT, state=RemoteConfigApplyState.ACKNOWLEDGED, clear=True) + + +def _is_ffe_waiting_for_rc(result: dict[str, Any]) -> bool: + provider_state = result.get("providerState") + return result.get("errorCode") == "PROVIDER_NOT_READY" or ( + isinstance(provider_state, dict) and provider_state.get("hasConfig") is False + ) + + +def _ffe_evaluate_with_rc_retry( + test_library: APMLibrary, + *, + flag: str, + variation_type: str, + default_value: bool | str | float | dict[str, Any], + targeting_key: str, + attributes: dict[str, Any] | None = None, +) -> dict[str, Any]: + result = test_library.ffe_evaluate( + flag=flag, + variation_type=variation_type, + default_value=default_value, + targeting_key=targeting_key, + attributes=attributes, + ) + for _ in range(FFE_READY_RETRY_ATTEMPTS - 1): + if not _is_ffe_waiting_for_rc(result): + return result + time.sleep(FFE_READY_RETRY_INTERVAL_SECONDS) + result = test_library.ffe_evaluate( + flag=flag, + variation_type=variation_type, + default_value=default_value, + targeting_key=targeting_key, + attributes=attributes, + ) + + return result + + +@scenarios.parametric +@features.feature_flags_serial_id_metadata +class Test_FFE_Serial_Id_Metadata: + """Test that the selected split's serial ID is threaded through flagMetadata.""" + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + """The serial ID of the selected split must appear in flagMetadata.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-basic-flag", + variation_type="BOOLEAN", + default_value=False, + targeting_key="user-1", + ) + assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" + + flag_metadata = result.get("flagMetadata") or {} + assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == 42, ( + f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == 42 for the selected split, " + f"got flagMetadata={flag_metadata}" + ) + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_zero_is_not_treated_as_missing( + self, test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + """Serial ID 0 is a real, valid ID and must survive falsy-value checks.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-zero-flag", + variation_type="BOOLEAN", + default_value=False, + targeting_key="user-1", + ) + assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" + + flag_metadata = result.get("flagMetadata") or {} + assert SERIAL_ID_METADATA_KEY in flag_metadata, ( + f"flagMetadata is missing '{SERIAL_ID_METADATA_KEY}' entirely; a 0 serial ID must not be " + f"dropped by a truthiness check. flagMetadata={flag_metadata}" + ) + assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == 0, ( + f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == 0, got flagMetadata={flag_metadata}" + ) + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_absent_when_split_has_no_serial_id( + self, test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + """A split without a serialId must not have one fabricated for it.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-missing-flag", + variation_type="STRING", + default_value="default", + targeting_key="user-1", + ) + assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" + + flag_metadata = result.get("flagMetadata") or {} + assert flag_metadata.get(SERIAL_ID_METADATA_KEY) is None, ( + f"Expected no serial ID for a split without one, got " + f"flagMetadata['{SERIAL_ID_METADATA_KEY}']={flag_metadata.get(SERIAL_ID_METADATA_KEY)!r}" + ) + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_matches_selected_split_under_targeting( + self, test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + """flagMetadata must reflect whichever split targeting actually selected.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + vip_result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-targeting-flag", + variation_type="STRING", + default_value="default", + targeting_key="user-vip", + attributes={"segment": "vip"}, + ) + standard_result = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-targeting-flag", + variation_type="STRING", + default_value="default", + targeting_key="user-standard", + attributes={"segment": "standard"}, + ) + + for label, result in (("vip", vip_result), ("standard", standard_result)): + assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data ({label}); result={result}" + + vip_metadata = vip_result.get("flagMetadata") or {} + standard_metadata = standard_result.get("flagMetadata") or {} + + assert vip_result.get("value") == "vip" + assert vip_metadata.get(SERIAL_ID_METADATA_KEY) == 201, ( + f"VIP targeting should select the split with serialId 201, got flagMetadata={vip_metadata}" + ) + + assert standard_result.get("value") == "standard" + assert standard_metadata.get(SERIAL_ID_METADATA_KEY) == 200, ( + f"Non-VIP targeting should select the split with serialId 200, got flagMetadata={standard_metadata}" + ) + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_serial_id_stable_across_repeated_evaluations( + self, test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + """The same context evaluated twice must yield the same serial ID.""" + _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) + assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + + first = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-basic-flag", + variation_type="BOOLEAN", + default_value=False, + targeting_key="user-repeat", + ) + second = _ffe_evaluate_with_rc_retry( + test_library, + flag="serial-id-basic-flag", + variation_type="BOOLEAN", + default_value=False, + targeting_key="user-repeat", + ) + + first_id = (first.get("flagMetadata") or {}).get(SERIAL_ID_METADATA_KEY) + second_id = (second.get("flagMetadata") or {}).get(SERIAL_ID_METADATA_KEY) + assert first_id == second_id == 42, ( + f"Expected stable serial ID 42 across repeated evaluations, got {first_id} then {second_id}" + ) diff --git a/utils/_features.py b/utils/_features.py index c3713a37225..824a5052811 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2718,6 +2718,14 @@ def feature_flags_event_enrichment(test_object): """ return _mark_test_object(test_object, feature_id=551, owner=_Owner.ffe) + @staticmethod + def feature_flags_serial_id_metadata(test_object): + """Feature Flags Serial ID Metadata (UFC split serialId threaded through flagMetadata) + + No feature-parity page yet; using NOT_REPORTED_ID until one is filed. + """ + return _mark_test_object(test_object, feature_id=NOT_REPORTED_ID, owner=_Owner.ffe) + @staticmethod def appsec_extended_data_collection(test_object): """AppSec supports extended data collection including headers and body From 25aeefe436620da06582d1677b82f43ff1576ec7 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Wed, 30 Sep 2026 12:24:41 -0400 Subject: [PATCH 2/8] Fix serial-ID metadata test discovery and validation Exclude the UFC configuration from evaluation-case discovery, disable the new suite for Rust, and require successful evaluation before accepting an absent serial ID. Apply the repository formatter to the new test module. Environment: Datadog workspace Co-authored-by: Leo Romanovsky --- manifests/rust.yml | 1 + tests/parametric/test_ffe/test_dynamic_evaluation.py | 2 +- tests/parametric/test_ffe/test_serial_id_metadata.py | 9 +++++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/manifests/rust.yml b/manifests/rust.yml index d2ba1f539a9..2c1341287d4 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -258,6 +258,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (rust is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature + tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 8aa1b906aaf..35b15215c50 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -41,7 +41,7 @@ def _get_test_case_files() -> list[str]: return [] # Exclude base fixtures that aren't test cases - excluded = {"flags-v1.json", "span-enrichment-flags.json"} + excluded = {"flags-v1.json", "serial-id-metadata-flags.json", "span-enrichment-flags.json"} return [f.name for f in test_data_dir.iterdir() if f.suffix == ".json" and f.name not in excluded] diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 2c74d124541..94cdbffc9ea 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -138,9 +138,7 @@ def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test ) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) - def test_serial_id_zero_is_not_treated_as_missing( - self, test_agent: TestAgentAPI, test_library: APMLibrary - ) -> None: + def test_serial_id_zero_is_not_treated_as_missing(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: """Serial ID 0 is a real, valid ID and must survive falsy-value checks.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" @@ -179,6 +177,9 @@ def test_serial_id_absent_when_split_has_no_serial_id( targeting_key="user-1", ) assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" + assert result.get("errorCode") in {None, ""}, f"FFE evaluation failed; result={result}" + assert result.get("reason") != "ERROR", f"FFE evaluation failed; result={result}" + assert result.get("value") == "control", f"Expected the configured control variation; result={result}" flag_metadata = result.get("flagMetadata") or {} assert flag_metadata.get(SERIAL_ID_METADATA_KEY) is None, ( @@ -190,7 +191,7 @@ def test_serial_id_absent_when_split_has_no_serial_id( def test_serial_id_matches_selected_split_under_targeting( self, test_agent: TestAgentAPI, test_library: APMLibrary ) -> None: - """flagMetadata must reflect whichever split targeting actually selected.""" + """FlagMetadata must reflect whichever split targeting actually selected.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" From 09de711eff519efa5f5518eb7f4c75cd13ed7f9a Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Wed, 30 Sep 2026 12:24:44 -0400 Subject: [PATCH 3/8] Require flagMetadata in serial-ID evaluations Fail evaluations that omit flagMetadata while allowing null or empty metadata when a split has no serial ID. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 Co-authored-by: hhan2024 --- .../test_ffe/test_serial_id_metadata.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 94cdbffc9ea..9c77b0a59a6 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -15,7 +15,8 @@ This module tests: 1. flagMetadata carries the exact serialId of the selected split. 2. Zero is a valid serial ID and must not be treated as "missing" (falsy-check trap). -3. A split with no serialId must not fabricate one; flagMetadata omits the key or reports null. +3. Evaluation details always expose flagMetadata; for a split with no serialId, the map is null/empty + or omits the serial ID key. 4. flagMetadata reflects whichever split targeting actually selected, not always the first one. 5. Repeated evaluations of the same context return the same serial ID. """ @@ -80,6 +81,17 @@ def _is_ffe_waiting_for_rc(result: dict[str, Any]) -> bool: ) +def _require_flag_metadata(result: dict[str, Any]) -> dict[str, Any]: + """Require evaluation details to expose flagMetadata, allowing a null map.""" + assert "flagMetadata" in result, f"FFE evaluation details did not emit flagMetadata; result={result}" + + flag_metadata = result["flagMetadata"] + assert flag_metadata is None or isinstance(flag_metadata, dict), ( + f"Expected flagMetadata to be a map or null, got {flag_metadata!r}" + ) + return flag_metadata or {} + + def _ffe_evaluate_with_rc_retry( test_library: APMLibrary, *, @@ -131,7 +143,7 @@ def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test ) assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" - flag_metadata = result.get("flagMetadata") or {} + flag_metadata = _require_flag_metadata(result) assert flag_metadata.get(SERIAL_ID_METADATA_KEY) == 42, ( f"Expected flagMetadata['{SERIAL_ID_METADATA_KEY}'] == 42 for the selected split, " f"got flagMetadata={flag_metadata}" @@ -152,7 +164,7 @@ def test_serial_id_zero_is_not_treated_as_missing(self, test_agent: TestAgentAPI ) assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data; result={result}" - flag_metadata = result.get("flagMetadata") or {} + flag_metadata = _require_flag_metadata(result) assert SERIAL_ID_METADATA_KEY in flag_metadata, ( f"flagMetadata is missing '{SERIAL_ID_METADATA_KEY}' entirely; a 0 serial ID must not be " f"dropped by a truthiness check. flagMetadata={flag_metadata}" @@ -181,7 +193,7 @@ def test_serial_id_absent_when_split_has_no_serial_id( assert result.get("reason") != "ERROR", f"FFE evaluation failed; result={result}" assert result.get("value") == "control", f"Expected the configured control variation; result={result}" - flag_metadata = result.get("flagMetadata") or {} + flag_metadata = _require_flag_metadata(result) assert flag_metadata.get(SERIAL_ID_METADATA_KEY) is None, ( f"Expected no serial ID for a split without one, got " f"flagMetadata['{SERIAL_ID_METADATA_KEY}']={flag_metadata.get(SERIAL_ID_METADATA_KEY)!r}" @@ -215,8 +227,8 @@ def test_serial_id_matches_selected_split_under_targeting( for label, result in (("vip", vip_result), ("standard", standard_result)): assert not _is_ffe_waiting_for_rc(result), f"FFE provider did not load RC data ({label}); result={result}" - vip_metadata = vip_result.get("flagMetadata") or {} - standard_metadata = standard_result.get("flagMetadata") or {} + vip_metadata = _require_flag_metadata(vip_result) + standard_metadata = _require_flag_metadata(standard_result) assert vip_result.get("value") == "vip" assert vip_metadata.get(SERIAL_ID_METADATA_KEY) == 201, ( @@ -251,8 +263,8 @@ def test_serial_id_stable_across_repeated_evaluations( targeting_key="user-repeat", ) - first_id = (first.get("flagMetadata") or {}).get(SERIAL_ID_METADATA_KEY) - second_id = (second.get("flagMetadata") or {}).get(SERIAL_ID_METADATA_KEY) + first_id = _require_flag_metadata(first).get(SERIAL_ID_METADATA_KEY) + second_id = _require_flag_metadata(second).get(SERIAL_ID_METADATA_KEY) assert first_id == second_id == 42, ( f"Expected stable serial ID 42 across repeated evaluations, got {first_id} then {second_id}" ) From 6dbea1dc639ae594656a859fdb13e46ac0044409 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Wed, 30 Sep 2026 12:24:45 -0400 Subject: [PATCH 4/8] Require missing serial ID to be omitted Tighten the missing-serial-ID test so an SDK cannot serialize an absent UFC serial ID as a null metadata entry. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 Co-authored-by: hhan2024 --- tests/parametric/test_ffe/test_serial_id_metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 9c77b0a59a6..536094775d1 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -15,8 +15,8 @@ This module tests: 1. flagMetadata carries the exact serialId of the selected split. 2. Zero is a valid serial ID and must not be treated as "missing" (falsy-check trap). -3. Evaluation details always expose flagMetadata; for a split with no serialId, the map is null/empty - or omits the serial ID key. +3. Evaluation details always expose flagMetadata; for a split with no serialId, the map must not + contain the serial ID key. 4. flagMetadata reflects whichever split targeting actually selected, not always the first one. 5. Repeated evaluations of the same context return the same serial ID. """ @@ -194,9 +194,9 @@ def test_serial_id_absent_when_split_has_no_serial_id( assert result.get("value") == "control", f"Expected the configured control variation; result={result}" flag_metadata = _require_flag_metadata(result) - assert flag_metadata.get(SERIAL_ID_METADATA_KEY) is None, ( - f"Expected no serial ID for a split without one, got " - f"flagMetadata['{SERIAL_ID_METADATA_KEY}']={flag_metadata.get(SERIAL_ID_METADATA_KEY)!r}" + assert SERIAL_ID_METADATA_KEY not in flag_metadata, ( + f"Expected flagMetadata to omit '{SERIAL_ID_METADATA_KEY}' for a split without a serial ID, " + f"got flagMetadata={flag_metadata}" ) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) From e33e31b7167f7db7e8a540af7967d5d19127ca46 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Wed, 30 Sep 2026 12:24:47 -0400 Subject: [PATCH 5/8] Enable span enrichment in serial ID tests Exercise serial-ID flag metadata under the required span-enrichment configuration for positive, zero, and missing UFC serial IDs. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 Co-authored-by: hhan2024 --- tests/parametric/test_ffe/test_serial_id_metadata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 536094775d1..8bcea2a48e6 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -5,10 +5,10 @@ enrichment, exposure caching, evaluation-metrics correlation -- need that serial ID without re-deriving it from (flag_key, allocation_key, variant_key). -The contract under test: the SDK's OpenFeature EvaluationDetails-equivalent -object must expose the selected split's serial ID via `flagMetadata` under -the internal key `__dd_split_serial_id`, matching the value present in the -UFC config pushed over Remote Config. This is the field +The contract under test: when span enrichment is enabled, the SDK's OpenFeature +EvaluationDetails-equivalent object must expose the selected split's serial ID +via `flagMetadata` under the internal key `__dd_split_serial_id`, matching the +value present in the UFC config pushed over Remote Config. This is the field `openfeature-js-client#269` introduced for the Node server SDK; this test generalizes the contract to every parametric-tested SDK. @@ -58,6 +58,7 @@ def _load_serial_id_metadata_fixture() -> dict[str, Any]: DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": "true", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", } From 161f7f3b7d8fc4bccaf27967bc671474c6642a91 Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Wed, 30 Sep 2026 12:24:49 -0400 Subject: [PATCH 6/8] Use approved pytest facade in serial ID tests Fix the import-policy failure when the PR is tested against current main. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 Co-authored-by: hhan2024 --- tests/parametric/test_ffe/test_serial_id_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 8bcea2a48e6..c893c067479 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -26,7 +26,7 @@ from pathlib import Path from typing import Any -import pytest +from utils import pytest from utils import features, scenarios from utils.dd_constants import RemoteConfigApplyState From 43527046566833b34cee0ad7d658d24f4ccd8a8f Mon Sep 17 00:00:00 2001 From: hhan2024 Date: Thu, 15 Oct 2026 13:00:15 -0400 Subject: [PATCH 7/8] Assign feature parity ID to serial metadata Use the registered Feature Parity entry for FFE serial ID metadata. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- utils/_features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/_features.py b/utils/_features.py index 824a5052811..69a5e411a0e 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2722,9 +2722,9 @@ def feature_flags_event_enrichment(test_object): def feature_flags_serial_id_metadata(test_object): """Feature Flags Serial ID Metadata (UFC split serialId threaded through flagMetadata) - No feature-parity page yet; using NOT_REPORTED_ID until one is filed. + https://feature-parity.us1.prod.dog/#/?feature=648 """ - return _mark_test_object(test_object, feature_id=NOT_REPORTED_ID, owner=_Owner.ffe) + return _mark_test_object(test_object, feature_id=648, owner=_Owner.ffe) @staticmethod def appsec_extended_data_collection(test_object): From 4c1c0107af49bf80680f2f3dd57e7c9267459526 Mon Sep 17 00:00:00 2001 From: "helen.han" Date: Fri, 16 Oct 2026 16:14:34 +0000 Subject: [PATCH 8/8] Use Remote Config exclusively in serial ID metadata tests Start the provider without injecting the UFC fixture after Remote Config acknowledgement. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- tests/parametric/test_ffe/test_serial_id_metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index c893c067479..8e8a9e14d45 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -133,7 +133,7 @@ class Test_FFE_Serial_Id_Metadata: def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: """The serial ID of the selected split must appear in flagMetadata.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" result = _ffe_evaluate_with_rc_retry( test_library, @@ -154,7 +154,7 @@ def test_serial_id_present_in_flag_metadata(self, test_agent: TestAgentAPI, test def test_serial_id_zero_is_not_treated_as_missing(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: """Serial ID 0 is a real, valid ID and must survive falsy-value checks.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" result = _ffe_evaluate_with_rc_retry( test_library, @@ -180,7 +180,7 @@ def test_serial_id_absent_when_split_has_no_serial_id( ) -> None: """A split without a serialId must not have one fabricated for it.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" result = _ffe_evaluate_with_rc_retry( test_library, @@ -206,7 +206,7 @@ def test_serial_id_matches_selected_split_under_targeting( ) -> None: """FlagMetadata must reflect whichever split targeting actually selected.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" vip_result = _ffe_evaluate_with_rc_retry( test_library, @@ -247,7 +247,7 @@ def test_serial_id_stable_across_repeated_evaluations( ) -> None: """The same context evaluated twice must yield the same serial ID.""" _set_and_wait_ffe_rc(test_agent, UFC_SERIAL_ID_METADATA_DATA) - assert test_library.ffe_start(UFC_SERIAL_ID_METADATA_DATA), "Failed to start FFE provider" + assert test_library.ffe_start(), "Failed to start FFE provider" first = _ffe_evaluate_with_rc_retry( test_library,