From 7b4a853215424ac6638067983b95a92f1f7c3ec1 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Mon, 19 Oct 2026 19:19:00 -0400 Subject: [PATCH 01/12] test(ffe): repair consented bounds and cover PHP IPC size fallback Port FFL-3407 coverage onto current main and exclude the PHP native IPC regression from other SDKs. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 Co-authored-by: Vickie Boettcher --- manifests/c.yml | 1 + manifests/cpp_httpd.yml | 1 + manifests/cpp_kong.yml | 1 + manifests/cpp_nginx.yml | 1 + manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + manifests/rust.yml | 1 + tests/ffe/test_flag_eval_evp.py | 79 ++++++++++++++++++++++++++------- 12 files changed, 75 insertions(+), 15 deletions(-) diff --git a/manifests/c.yml b/manifests/c.yml index 416336889bb..b79f39c0363 100644 --- a/manifests/c.yml +++ b/manifests/c.yml @@ -4,6 +4,7 @@ manifest: tests/ai_guard/: irrelevant (dd-trace-c does not implement AI Guard) tests/appsec/: irrelevant (dd-trace-c does not implement AppSec or IAST) tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/integrations/test_dbm.py: incomplete_test_app (The Perl Mojolicious workload does not provide database endpoints) tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION: missing_feature tests/parametric/otel_env_vars/test_otel_exporter_otlp_headers.py::Test_OTEL_EXPORTER_OTLP_HEADERS: missing_feature diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 47030f0bcc9..61dd9a90324 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -57,6 +57,7 @@ manifest: tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/: missing_feature (Endpoint not implemented) diff --git a/manifests/cpp_kong.yml b/manifests/cpp_kong.yml index fc369f3647c..2d59e7cb35b 100644 --- a/manifests/cpp_kong.yml +++ b/manifests/cpp_kong.yml @@ -10,6 +10,7 @@ manifest: tests/debugger/: irrelevant tests/ffe/: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integrations/: missing_feature (Endpoints not implemented) tests/otel/: irrelevant (library does not implement OpenTelemetry) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 30648f6498b..e378866eb6c 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -290,6 +290,7 @@ manifest: tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_kafka.py::Test_Kafka: missing_feature diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index e21345f359b..ed5f8ea9375 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -852,6 +852,7 @@ manifest: - declaration: missing_feature (EX-3416) excluded_weblog: [poc, uds] tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: v3.44.0 tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_kafka.py::Test_Kafka: v2.0.0-prerelease diff --git a/manifests/golang.yml b/manifests/golang.yml index 2ccae0ece76..943100328ab 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1766,6 +1766,7 @@ manifest: tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: v2.11.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed: v2.11.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed: v2.11.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.10.0 + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: v2.8.0 tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Metric_Parse_Error_Invalid_Regex: irrelevant (Go validates regex at config load time) tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Nested_Attributes_Ignored: irrelevant (FFL-1980) diff --git a/manifests/java.yml b/manifests/java.yml index 70c07421c3f..3118f086ade 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3493,6 +3493,7 @@ manifest: tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Caching_Serial_Id_Disappears: missing_feature (EX-3415) tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Serial_Id: missing_feature (EX-3415) tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 11846c06b7f..7a4f935f07d 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1920,6 +1920,7 @@ manifest: - weblog_declaration: "*": incomplete_test_app express5: v7.0.0-pre + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": incomplete_test_app diff --git a/manifests/python.yml b/manifests/python.yml index fefe9079dfc..807cacbd266 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1567,6 +1567,7 @@ manifest: tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Runtime_Default: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: v4.7.0 tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Nested_Attributes_Ignored: irrelevant (FFL-1980) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 20f6ac2dc5d..c29303d1741 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2153,6 +2153,7 @@ manifest: tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Caching_Serial_Id_Disappears: missing_feature (EX-3417) tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Serial_Id: missing_feature (EX-3417) tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/rust.yml b/manifests/rust.yml index dc1a7ced743..ed88859e78a 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -26,6 +26,7 @@ manifest: tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/: incomplete_test_app (crossed integration endpoints are not implemented) diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 551d47ce40b..fc0df929964 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -401,10 +401,13 @@ class Test_FFE_EVP_Flagevaluation_Context_Bounds: def setup_ffe_evp_flagevaluation_context_bounds(self) -> None: config_id = "ffe-evp-context-bounds" self.flag_key = "evp-context-bounds-flag" - self.oversized_field = "field_010_oversized" - rc.tracer_rc_state.reset().set_config(f"{RC_PATH}/{config_id}/config", make_ufc_fixture(self.flag_key)).apply() + self.oversized_field = "field_010" + rc.tracer_rc_state.reset().set_config( + f"{RC_PATH}/{config_id}/config", make_ufc_fixture(self.flag_key, observe_full_evaluation_data=True) + ).apply() attributes: JSON = {f"field_{index:03d}": f"value-{index}" for index in range(300)} + # Replace an early value so omission cannot pass just by stopping at 256 fields. attributes[self.oversized_field] = "x" * 300 self.r = evaluate_flag(self.flag_key, targeting_key="evp-context-user", attributes=attributes) @@ -415,27 +418,16 @@ def test_ffe_evp_flagevaluation_context_bounds(self) -> None: events = find_evp_flagevaluation_events(self.flag_key) assert events, f"Expected EVP flagevaluation event for flag {self.flag_key}" - full_context_events = 0 for _, event in events: assert_event_contract(event, self.flag_key) context = event.get("context") - if context is None: - continue - - assert isinstance(context, dict), "context must be an object when present" + assert isinstance(context, dict), "consented evaluation must retain context" evaluation_context = context.get("evaluation") - if evaluation_context is None: - continue - - full_context_events += 1 assert isinstance(evaluation_context, dict), "context.evaluation must be an object" + assert evaluation_context.get("field_000") == "value-0", "valid context fields must survive pruning" assert len(evaluation_context) <= 256, f"context.evaluation has too many fields: {len(evaluation_context)}" assert self.oversized_field not in evaluation_context, "oversized string context field must be omitted" - if full_context_events == 0: - for _, event in events: - assert "context" not in event, f"degraded event should omit context: {event}" - @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation @@ -622,6 +614,63 @@ def test_ffe_evp_flagevaluation_degradation(self) -> None: assert object_key(event.get("allocation"), "allocation") == "default-allocation" +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_evp_flagevaluation +@slow +class Test_FFE_EVP_Flagevaluation_Oversized_Identity: + """Preserve PHP flag results and counts across native IPC size degradation.""" + + def setup_ffe_evp_flagevaluation_oversized_identity(self) -> None: + config_id = "ffe-evp-oversized-identity" + self.flag_key = "evp-oversized-identity-flag" + self.control_targeting_key = "evp-size-control-user" + rc.tracer_rc_state.reset().set_config( + f"{RC_PATH}/{config_id}/config", make_ufc_fixture(self.flag_key, observe_full_evaluation_data=True) + ).apply() + + # PHP's native IPC path has a 5 MiB event limit; other SDKs are excluded in manifests. + # The normal successor also verifies that oversize fallback leaves IPC usable. + oversized_targeting_key = "x" * (5 * 1024 * 1024 + 1) + self.responses = [ + evaluate_flag( + self.flag_key, + targeting_key=targeting_key, + attributes={"email": "evp-size-canary@example.invalid"}, + ) + for targeting_key in (oversized_targeting_key, self.control_targeting_key) + ] + + def test_ffe_evp_flagevaluation_oversized_identity(self) -> None: + for response in self.responses: + assert response.status_code == 200, f"Flag evaluation failed: {response.text}" + assert json.loads(response.text).get("value") == "on-value", ( + "telemetry limits must not change the flag result" + ) + + events = find_evp_flagevaluation_events(self.flag_key) + assert_total_evaluation_count(events, 2, self.flag_key) + + degraded_events = [(batch, event) for batch, event in events if "targeting_key" not in event] + assert_total_evaluation_count(degraded_events, 1, self.flag_key) + for _, event in degraded_events: + assert "context" not in event, "size-degraded events must omit all event context" + + control_events = [ + (batch, event) for batch, event in events if event.get("targeting_key") == self.control_targeting_key + ] + assert_total_evaluation_count(control_events, 1, self.flag_key) + for _, event in control_events: + assert event.get("context", {}).get("evaluation", {}).get("email") == "evp-size-canary@example.invalid" + + for batch, event in events: + assert_batch_context(batch) + assert_event_contract(event, self.flag_key) + assert object_key(event.get("variant"), "variant") == "on" + assert object_key(event.get("allocation"), "allocation") == "default-allocation" + assert "error" not in event + assert "runtime_default_used" not in event + + @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation @slow From 23bbb8815190e0771ddc831a68f405595f7676e6 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Mon, 19 Oct 2026 19:19:02 -0400 Subject: [PATCH 02/12] test(ffe): isolate overload accounting from native cap enforcement Separate context count and value bounds, validate reported capacity drops in an isolated scenario, and add contract regression coverage. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- .github/workflows/run-end-to-end.yml | 3 + docs/understand/weblogs/end-to-end_weblog.md | 7 + manifests/php.yml | 2 +- manifests/python.yml | 2 +- tests/ffe/README.md | 46 ++++- tests/ffe/test_flag_eval_evp.py | 90 ++++++--- tests/ffe/test_tests.py | 186 ++++++++++++++++++ tests/ffe/utils/evp.py | 34 ++++ .../test_minimal_number_of_scenarios.py | 4 + utils/_context/_scenarios/__init__.py | 42 ++-- 10 files changed, 371 insertions(+), 45 deletions(-) diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 18a202bf6a0..955858bda88 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -305,6 +305,9 @@ jobs: - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS + - name: Run FEATURE_FLAGGING_EVP_STRESS scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVP_STRESS"') + run: ./run.sh FEATURE_FLAGGING_EVP_STRESS - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT diff --git a/docs/understand/weblogs/end-to-end_weblog.md b/docs/understand/weblogs/end-to-end_weblog.md index 967040e6cdf..eac7a749c48 100644 --- a/docs/understand/weblogs/end-to-end_weblog.md +++ b/docs/understand/weblogs/end-to-end_weblog.md @@ -1143,6 +1143,13 @@ accept a JSON body with these fields: - `targetingKey`: the evaluation subject key. - `attributes`: flat scalar targeting attributes. +For the high-cardinality FFE workload, the endpoint also accepts a non-empty +`targetingKeys` array. Evaluate the same flag once per key through the public SDK +API, using the same attributes and default value. The response includes `count` +(the number of evaluations performed) and the final evaluation's `value`. +Do not replace the loop with a direct telemetry submission or change SDK flush +timing. When the array is absent or empty, evaluate the single `targetingKey`. + The response must be JSON and include at least `value` and `reason`. Error responses should also include `errorCode` and `errorMessage`. diff --git a/manifests/php.yml b/manifests/php.yml index 4d4f1a2f563..5c7f31bff48 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1260,7 +1260,7 @@ manifest: tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Serial_Id: missing_feature (EX-3425) tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Burst_Aggregation: bug (FFL-2676) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation: bug (FFL-2676) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Accounting: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Aggregation: bug (FFL-2676) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: diff --git a/manifests/python.yml b/manifests/python.yml index 807cacbd266..a7dc9ed6d72 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1552,7 +1552,6 @@ manifest: tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Burst_Aggregation: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Context_Bounds: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Count: missing_feature (FFL-2446) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Egress_Agentless_Direct: - weblog_declaration: "*": missing_feature (Agentless EVP enablement is validated only on flask-poc) @@ -1562,6 +1561,7 @@ manifest: "*": missing_feature (Agentless EVP enablement is validated only on flask-poc) flask-poc: v4.16.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Egress_Datadog_Agent: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Accounting: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Aggregation: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Load_Aggregation: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: missing_feature (FFL-2446) diff --git a/tests/ffe/README.md b/tests/ffe/README.md index be43490db3c..48d02fab2b5 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -20,18 +20,60 @@ This directory contains system tests for the Feature Flags & Experimentation (FF ## Test-contract self-tests -The self-tests in `testing_tests/` validate the FFE assertions using synthetic +The self-tests in `test_tests.py` validate the FFE assertions using synthetic captures. They do not run SDKs or contact an intake. With the Python 3.12 runner environment installed, run them explicitly: ```bash -./run.sh FEATURE_FLAGGING_CONTRACT_TESTS tests/ffe/testing_tests +./run.sh FEATURE_FLAGGING_CONTRACT_TESTS tests/ffe/test_tests.py ``` This no-container scenario has no CI workflow or scenario-group membership. Its tests do not run in `TEST_THE_TEST`, `DEFAULT`, or the FFE end-to-end scenarios. Framework mock-backend and lifecycle tests stay in `tests/test_the_test/`. +## Bounds and overload coverage + +The consented context tests exercise field count and string length separately. +The count case accepts any valid retained subset (SDK map iteration order is not +part of the contract). The value case stays below the field-count cap, so that +cap cannot hide a missing string-length check. Missing evaluation context and +truncated, rather than omitted, oversized strings fail. + +`Test_FFE_EVP_Flagevaluation_Oversized_Identity` is PHP-native IPC coverage. +It sends a targeting key over 5 MiB, then a normal evaluation through the public +weblog endpoint. Counts and customer results must survive; the oversized event +loses identity/context and the successor retains consented context. Existing +manifests exclude this case from other SDKs. PHP release activation remains a +separate task; use `-F` to validate an unreleased implementation locally. + +The high-cardinality workload runs in its own scenario: + +```sh +./run.sh FEATURE_FLAGGING_EVP_STRESS --library php -F tests/ffe/test_flag_eval_evp.py +``` + +Do not add other evaluation workloads to this scenario: drop counters are +process-wide, not flag-scoped. It checks that delivered evaluation counts plus +reported capacity drops equal 12,000 attempted evaluations. It rejects missing +accounting, total loss, serialization/delivery failures, and invalid counters. +Degradation counts are not drops: degraded evaluations are already included in +the delivered count. This is an overload-accounting test, not a lossless-delivery +or throughput guarantee. Normal count and aggregation tests still require +lossless delivery. + +The accepted drop counters are `flagevaluation.rows.dropped` (SDK submission) +and `flagevaluation.evaluations.dropped` (native writer), in `tracers` or `general` +telemetry namespaces. Only `pre_queue_overflow`, `queue_overflow`, `degraded_cap`, +and `payload_limit` reasons count as capacity drops. A new SDK counter convention +needs explicit coverage here, not an inferred allowance for missing evaluations. + +The external workload does not guarantee degradation: SDK flush windows can +split it before the per-flag cap is reached. Exact cap-boundary proof belongs in +SDK/native tests with synchronous enqueueing. PHP's pinned libdatadog has +`coalescer_degrades_after_per_flag_cap` in `libdd-ffe`, which verifies 10,000 full +rows plus 50 degraded evaluations. No production timing override is used. + --- # Eval Metrics Implementation Guide diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index fc0df929964..e042e0284ae 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -7,6 +7,7 @@ from tests.ffe.utils.evp import ( + assert_flagevaluation_accounting, assert_agentless_evp_intake_request, assert_agentless_evp_topology, feature_flagging_evp_egress, @@ -28,8 +29,7 @@ RC_PRODUCT = "FFE_FLAGS" RC_PATH = f"datadog/2/{RC_PRODUCT}" EVP_FLAGEVALUATIONS_PATH = "/api/v2/flagevaluation" -EVP_FULL_TIER_PER_FLAG_CAP = 10_000 -EVP_DEGRADATION_OVERFLOW_EVALS = 2_000 +EVP_HIGH_CARDINALITY_EVALS = 12_000 # Fixed input/output vector for the PII-protection tests. Every SDK's unit tests # should assert against the same input to prove byte-identical hashing across SDKs. @@ -401,16 +401,12 @@ class Test_FFE_EVP_Flagevaluation_Context_Bounds: def setup_ffe_evp_flagevaluation_context_bounds(self) -> None: config_id = "ffe-evp-context-bounds" self.flag_key = "evp-context-bounds-flag" - self.oversized_field = "field_010" rc.tracer_rc_state.reset().set_config( f"{RC_PATH}/{config_id}/config", make_ufc_fixture(self.flag_key, observe_full_evaluation_data=True) ).apply() - attributes: JSON = {f"field_{index:03d}": f"value-{index}" for index in range(300)} - # Replace an early value so omission cannot pass just by stopping at 256 fields. - attributes[self.oversized_field] = "x" * 300 - - self.r = evaluate_flag(self.flag_key, targeting_key="evp-context-user", attributes=attributes) + self.attributes: JSON = {f"field_{index:03d}": f"value-{index}" for index in range(300)} + self.r = evaluate_flag(self.flag_key, targeting_key="evp-context-user", attributes=self.attributes) def test_ffe_evp_flagevaluation_context_bounds(self) -> None: assert self.r.status_code == 200, f"Flag evaluation failed: {self.r.text}" @@ -424,8 +420,40 @@ def test_ffe_evp_flagevaluation_context_bounds(self) -> None: assert isinstance(context, dict), "consented evaluation must retain context" evaluation_context = context.get("evaluation") assert isinstance(evaluation_context, dict), "context.evaluation must be an object" - assert evaluation_context.get("field_000") == "value-0", "valid context fields must survive pruning" + assert evaluation_context, "valid context fields must survive pruning" assert len(evaluation_context) <= 256, f"context.evaluation has too many fields: {len(evaluation_context)}" + # SDK maps need not preserve input order or choose the same retained subset. + for key, value in evaluation_context.items(): + assert key in self.attributes, f"unexpected context field: {key}" + assert value == self.attributes[key], f"context field {key} changed during pruning" + + def setup_ffe_evp_flagevaluation_context_value_bounds(self) -> None: + self.value_flag_key = "evp-context-value-bounds-flag" + self.oversized_field = "oversized" + rc.tracer_rc_state.reset().set_config( + f"{RC_PATH}/ffe-evp-context-value-bounds/config", + make_ufc_fixture(self.value_flag_key, observe_full_evaluation_data=True), + ).apply() + + # Stay below the field-count limit so it cannot hide a missing value-length check. + self.value_response = evaluate_flag( + self.value_flag_key, + targeting_key="evp-context-value-user", + attributes={"valid": "retained", self.oversized_field: "x" * 300}, + ) + + def test_ffe_evp_flagevaluation_context_value_bounds(self) -> None: + assert self.value_response.status_code == 200, f"Flag evaluation failed: {self.value_response.text}" + events = find_evp_flagevaluation_events(self.value_flag_key) + assert events, f"Expected EVP flagevaluation event for flag {self.value_flag_key}" + + for _, event in events: + assert_event_contract(event, self.value_flag_key) + context = event.get("context") + assert isinstance(context, dict), "consented evaluation must retain context" + evaluation_context = context.get("evaluation") + assert isinstance(evaluation_context, dict), "context.evaluation must be an object" + assert evaluation_context.get("valid") == "retained", "valid context fields must survive pruning" assert self.oversized_field not in evaluation_context, "oversized string context field must be omitted" @@ -572,16 +600,16 @@ def test_ffe_evp_flagevaluation_high_cardinality_aggregation(self) -> None: assert_total_evaluation_count(events, self.eval_count, self.flag_key) -@scenarios.feature_flagging_and_experimentation +@scenarios.feature_flagging_evp_stress @features.feature_flags_evp_flagevaluation @scenario_crash -class Test_FFE_EVP_Flagevaluation_Degradation: - """Test degraded EVP shape after the production per-flag full-tier cap is exceeded.""" +class Test_FFE_EVP_Flagevaluation_High_Cardinality_Accounting: + """Account for an isolated workload without assuming a single aggregation window.""" - def setup_ffe_evp_flagevaluation_degradation(self) -> None: - config_id = "ffe-evp-degradation" - self.flag_key = "evp-degradation-flag" - self.eval_count = EVP_FULL_TIER_PER_FLAG_CAP + EVP_DEGRADATION_OVERFLOW_EVALS + def setup_ffe_evp_flagevaluation_high_cardinality_accounting(self) -> None: + config_id = "ffe-evp-high-cardinality-accounting" + self.flag_key = "evp-high-cardinality-accounting-flag" + self.eval_count = EVP_HIGH_CARDINALITY_EVALS rc.tracer_rc_state.reset().set_config(f"{RC_PATH}/{config_id}/config", make_ufc_fixture(self.flag_key)).apply() targeting_keys = [f"evp-degradation-user-{index}" for index in range(self.eval_count)] @@ -594,24 +622,40 @@ def setup_ffe_evp_flagevaluation_degradation(self) -> None: ) ] - def test_ffe_evp_flagevaluation_degradation(self) -> None: + def test_ffe_evp_flagevaluation_high_cardinality_accounting(self) -> None: for index, response in enumerate(self.responses): assert response.status_code == 200, f"Request {index + 1} failed: {response.text}" + result = json.loads(response.text) + assert result.get("value") == "on-value", "overload must not change the flag result" + assert result.get("count") == self.eval_count, "weblog must execute the whole workload" events = find_evp_flagevaluation_events(self.flag_key) assert events, f"Expected EVP flagevaluation events for flag {self.flag_key}" - degraded_events = [event for _, event in events if "context" not in event and "targeting_key" not in event] - assert degraded_events, f"Expected degraded EVP flagevaluation buckets for flag {self.flag_key}" assert_no_duplicate_visible_events(events) - assert_total_evaluation_count(events, self.eval_count, self.flag_key) + # These counters are process-wide, not flag-scoped: this class must remain + # the only evaluation workload in FEATURE_FLAGGING_EVP_STRESS. + drop_series = [ + series + for namespace in ("tracers", "general") + for metric in ("flagevaluation.rows.dropped", "flagevaluation.evaluations.dropped") + for series in interfaces.library.get_telemetry_metric_series(namespace, metric) + ] + assert_flagevaluation_accounting(sum_evaluation_count(events), self.eval_count, drop_series) - for event in degraded_events: + # A slow run may span many flushes and never degrade. Native tests prove + # the exact cap; every delivered row must still have a valid tier shape. + for _, event in events: assert_event_contract(event, self.flag_key) - assert "context" not in event, f"degraded event must omit context: {event}" - assert "targeting_key" not in event, f"degraded event must omit targeting_key: {event}" + if "targeting_key" not in event: + assert "context" not in event, "degraded events must omit all event context" + else: + assert isinstance(event["targeting_key"], str), "full-tier targeting_key must be a string" + assert event["targeting_key"], "full-tier targeting_key must be non-empty" assert object_key(event.get("variant"), "variant") == "on" assert object_key(event.get("allocation"), "allocation") == "default-allocation" + assert "error" not in event + assert "runtime_default_used" not in event @scenarios.feature_flagging_and_experimentation diff --git a/tests/ffe/test_tests.py b/tests/ffe/test_tests.py index 78aa81f9156..37d65723163 100644 --- a/tests/ffe/test_tests.py +++ b/tests/ffe/test_tests.py @@ -15,6 +15,7 @@ assert_agentless_evp_topology, assert_direct_evp_runtime_evidence, assert_direct_evp_shutdown_evidence, + assert_flagevaluation_accounting, assert_no_evp_proxy_requests, expected_evp_origin, ) @@ -36,6 +37,191 @@ ) +@pytest.mark.parametrize( + ("delivered", "dropped", "reason", "valid"), + [ + (12_000, None, "queue_overflow", True), + (11_852, 148.0, "queue_overflow", True), + (11_950, 50, "degraded_cap", True), + (11_999, None, "queue_overflow", False), + (12_000, 1, "queue_overflow", False), + (0, 12_000, "queue_overflow", False), + (11_999, 1, "serialization_error", False), + (11_999, 1, "delivery_failure", False), + (11_999, 1.5, "queue_overflow", False), + (12_001, -1, "queue_overflow", False), + (11_999, True, "queue_overflow", False), + (11_999, float("inf"), "queue_overflow", False), + ], +) +@scenarios.feature_flagging_contract_tests +@features.not_reported +def test_flagevaluation_isolated_accounting(delivered: int, dropped: float | None, reason: str, *, valid: bool) -> None: + series = [] if dropped is None else [{"type": "count", "tags": [f"reason:{reason}"], "points": [[1, dropped]]}] + if valid: + assert_flagevaluation_accounting(delivered, 12_000, series) + else: + with pytest.raises(AssertionError): + assert_flagevaluation_accounting(delivered, 12_000, series) + + +@pytest.mark.parametrize( + "series", + [ + {"type": "gauge", "tags": ["reason:queue_overflow"], "points": [[1, 1]]}, + {"type": "count", "tags": [], "points": [[1, 1]]}, + {"type": "count", "tags": ["reason:queue_overflow", "reason:degraded_cap"], "points": [[1, 1]]}, + {"type": "count", "tags": ["reason:queue_overflow"], "points": []}, + ], +) +@scenarios.feature_flagging_contract_tests +@features.not_reported +def test_flagevaluation_accounting_rejects_invalid_telemetry(series: dict[str, Any]) -> None: + with pytest.raises(AssertionError): + assert_flagevaluation_accounting(11_999, 12_000, [series]) + + +def _bounded_context_event(flag_key: str) -> dict[str, Any]: + return { + "flag": {"key": flag_key}, + "variant": {"key": "on"}, + "allocation": {"key": "default-allocation"}, + "timestamp": 1, + "first_evaluation": 1, + "last_evaluation": 1, + "evaluation_count": 1, + } + + +@pytest.mark.parametrize( + ("case", "valid"), + [ + ("first subset", True), + ("last subset", True), + ("missing context", False), + ("empty evaluation", False), + ("too many fields", False), + ("changed value", False), + ("unexpected field", False), + ], +) +@scenarios.feature_flagging_contract_tests +@features.not_reported +def test_flagevaluation_context_count_contract(monkeypatch: pytest.MonkeyPatch, case: str, *, valid: bool) -> None: + check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() + check.flag_key = "bounds-contract" + check.r = MagicMock(status_code=200) + check.attributes = {f"field_{index:03d}": f"value-{index}" for index in range(300)} + retained = dict(list(check.attributes.items())[:256]) + if case == "last subset": + retained = dict(list(check.attributes.items())[-256:]) + elif case == "empty evaluation": + retained = {} + elif case == "too many fields": + retained = check.attributes + elif case == "changed value": + retained["field_000"] = "changed" + elif case == "unexpected field": + retained = {"not-supplied": "value"} + event = _bounded_context_event(check.flag_key) + if case != "missing context": + event["context"] = {"evaluation": retained} + monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: [({}, event)]) + if valid: + check.test_ffe_evp_flagevaluation_context_bounds() + else: + with pytest.raises(AssertionError): + check.test_ffe_evp_flagevaluation_context_bounds() + + +@pytest.mark.parametrize("case", ["valid", "missing context", "service only", "lost sibling", "long", "truncated"]) +@scenarios.feature_flagging_contract_tests +@features.not_reported +def test_flagevaluation_context_value_contract(monkeypatch: pytest.MonkeyPatch, case: str) -> None: + check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() + check.value_flag_key = "value-bounds-contract" + check.oversized_field = "oversized" + check.value_response = MagicMock(status_code=200) + event = _bounded_context_event(check.value_flag_key) + event["context"] = {"evaluation": {"valid": "retained"}} + if case == "missing context": + del event["context"] + elif case == "service only": + event["context"] = {"dd": {"service": "weblog"}} + elif case == "lost sibling": + event["context"]["evaluation"] = {} + elif case in ("long", "truncated"): + event["context"]["evaluation"]["oversized"] = "x" * (300 if case == "long" else 256) + monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: [({}, event)]) + if case == "valid": + check.test_ffe_evp_flagevaluation_context_value_bounds() + else: + with pytest.raises(AssertionError): + check.test_ffe_evp_flagevaluation_context_value_bounds() + + +@pytest.mark.parametrize("kind", ["count", "value"]) +@scenarios.feature_flagging_contract_tests +@features.not_reported +def test_flagevaluation_bounds_fixtures_enable_consent(monkeypatch: pytest.MonkeyPatch, kind: str) -> None: + state = MagicMock() + evaluate = MagicMock() + monkeypatch.setattr(flag_eval_evp_tests.rc, "tracer_rc_state", state) + monkeypatch.setattr(flag_eval_evp_tests, "evaluate_flag", evaluate) + check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() + if kind == "count": + check.setup_ffe_evp_flagevaluation_context_bounds() + else: + check.setup_ffe_evp_flagevaluation_context_value_bounds() + fixture = state.reset.return_value.set_config.call_args.args[1] + assert fixture["observeFullEvaluationData"] is True + attributes = evaluate.call_args.kwargs["attributes"] + if kind == "count": + assert len(attributes) > 256 + assert all(len(value) <= 256 for value in attributes.values()) + else: + assert len(attributes) < 256 + assert len(attributes["oversized"]) > 256 + assert attributes["valid"] == "retained" + + +@pytest.mark.parametrize( + "case", + ["valid", "dropped count", "retained identity", "retained context", "lost control", "lost variant", "wrong result"], +) +@scenarios.feature_flagging_contract_tests +@features.not_reported +def test_flagevaluation_php_oversized_identity_contract(monkeypatch: pytest.MonkeyPatch, case: str) -> None: + check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Oversized_Identity() + check.flag_key = "oversized-contract" + check.control_targeting_key = "control" + check.responses = [MagicMock(status_code=200, text='{"value":"on-value"}')] * 2 + batch = {"context": {"service": "weblog"}} + degraded = _bounded_context_event(check.flag_key) + control = _bounded_context_event(check.flag_key) + control["targeting_key"] = check.control_targeting_key + control["context"] = {"evaluation": {"email": "evp-size-canary@example.invalid"}} + rows = [(batch, degraded), (batch, control)] + if case == "dropped count": + rows.pop(0) + elif case == "retained identity": + degraded["targeting_key"] = "must-not-survive" + elif case == "retained context": + degraded["context"] = {} + elif case == "lost control": + del control["context"] + elif case == "lost variant": + del degraded["variant"] + elif case == "wrong result": + check.responses = [MagicMock(status_code=200, text='{"value":"default"}')] + monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: rows) + if case == "valid": + check.test_ffe_evp_flagevaluation_oversized_identity() + else: + with pytest.raises(AssertionError): + check.test_ffe_evp_flagevaluation_oversized_identity() + + @pytest.mark.parametrize("has_capture", [True, False]) @scenarios.feature_flagging_contract_tests @features.not_reported diff --git a/tests/ffe/utils/evp.py b/tests/ffe/utils/evp.py index 970f6091c2b..1c2c60372ee 100644 --- a/tests/ffe/utils/evp.py +++ b/tests/ffe/utils/evp.py @@ -3,6 +3,7 @@ from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime +from math import isfinite from pathlib import Path from typing import Any, Literal @@ -52,6 +53,39 @@ EVPRoute = Literal["agent", "sidecar", "direct"] +def assert_flagevaluation_accounting(delivered: int, attempted: int, drop_series: Iterable[dict[str, Any]]) -> None: + """Account for isolated stress traffic; never count degradation as lost evaluations.""" + dropped = 0 + capacity_reasons = { + "reason:pre_queue_overflow", + "reason:queue_overflow", + "reason:degraded_cap", + "reason:payload_limit", + } + for series in drop_series: + assert series.get("type") == "count", "drop telemetry must be a delta count" + reasons = {tag for tag in series.get("tags", []) if tag.startswith("reason:")} + assert len(reasons) == 1, f"expected one drop reason: {reasons}" + assert reasons <= capacity_reasons, f"unexpected drop reason: {reasons}" + points = series.get("points") + assert isinstance(points, list), "drop metric points must be a list" + assert points, "drop metric must contain points" + for point in points: + assert isinstance(point, list), "invalid drop metric point" + assert len(point) == 2, "invalid drop metric point" + count = point[1] + assert type(count) in (int, float), "invalid drop count type" + assert isfinite(count), "drop count must be finite" + assert count >= 0, "drop count must be nonnegative" + assert count == int(count), "drop count must be integral" + dropped += int(count) + assert delivered > 0, "stress workload must deliver evaluations, not only report drops" + assert delivered + dropped == attempted, ( + f"Expected delivered + reported drops == {attempted}, got {delivered} + {dropped}. " + "Missing telemetry is not evidence of an accounted drop." + ) + + def _is_expected_pid1_executable(library_name: str, executable: str, expected: set[str]) -> bool: """Accept ordinary language launchers without accepting wrappers by prefix.""" if executable in expected: diff --git a/tests/test_the_test/test_minimal_number_of_scenarios.py b/tests/test_the_test/test_minimal_number_of_scenarios.py index 06cf0bbe1ba..b734dda1be6 100644 --- a/tests/test_the_test/test_minimal_number_of_scenarios.py +++ b/tests/test_the_test/test_minimal_number_of_scenarios.py @@ -42,6 +42,10 @@ ("APPSEC_RUNTIME_ACTIVATION", "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE"): "TODO", ("APPSEC_RUNTIME_ACTIVATION", "REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING"): "TODO", ("APPSEC_RUNTIME_ACTIVATION", "TRACING_CONFIG_NONDEFAULT_4"): "TODO", + ( + "FEATURE_FLAGGING_AND_EXPERIMENTATION", + "FEATURE_FLAGGING_EVP_STRESS", + ): "Process-wide FFE drop counters cannot be attributed to the stress workload if other evaluations run", ("REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD", "APPSEC_REQUEST_BLOCKING"): "TODO", ("REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES", "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE"): "TODO", ("TRACING_CONFIG_EMPTY", "TRACE_PROPAGATION_STYLE_DEFAULT"): "TODO", diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index efe42703db9..09d6c35a15f 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -63,6 +63,20 @@ } +_FFE_WEBLOG_ENV: dict[str, str | None] = { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", + # Python set_provider() blocks waiting for RC, which arrives only after boot. + # Fail initialization fast; the provider recovers when the test installs RC. + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": "100", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", + "DD_METRICS_OTEL_ENABLED": "true", + "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://agent:4318/v1/metrics", + "OTEL_METRIC_EXPORT_INTERVAL": "1000", +} + + class _Scenarios: todo = Scenario("TODO", doc="scenario that skips tests not yet executed", github_workflow=None) test_the_test = TestTheTestScenario("TEST_THE_TEST", doc="Small scenario that check system-tests internals") @@ -885,28 +899,20 @@ class _Scenarios: rc_api_enabled=True, # Allow final EVP batches to reach the backend after the weblog flushes and stops. agent_interface_timeout=15, - weblog_env={ - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", - # set_provider() in Python blocks until we receive RC - # configuration for feature flags. But it is only sent - # after weblog sucessfully boots and tests start - # executing. Unfortunately, Python's OpenFeature SDK does - # not have "set provider and don't wait," so we reduce the - # timeout here, so that the provider initialization fails - # fast, weblog boots, and provider recovers when we set RC - # configuration later. - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": "100", - "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", - "DD_METRICS_OTEL_ENABLED": "true", - "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf", - "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://agent:4318/v1/metrics", - "OTEL_METRIC_EXPORT_INTERVAL": "1000", - }, + weblog_env=_FFE_WEBLOG_ENV.copy(), doc="", scenario_groups=[scenario_groups.ffe], ) + feature_flagging_evp_stress = DdTraceEndToEndScenario( + "FEATURE_FLAGGING_EVP_STRESS", + rc_api_enabled=True, + agent_interface_timeout=15, + weblog_env=_FFE_WEBLOG_ENV.copy(), + doc="Isolated high-cardinality EVP accounting: delivered evaluations plus reported capacity drops.", + scenario_groups=[scenario_groups.ffe], + ) + feature_flagging_and_experimentation_agentless = FeatureFlaggingAgentlessEndToEndScenario( "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS" ) From 134624e0b9f218af3833250764f7eb970852c3e5 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Tue, 20 Oct 2026 16:46:26 +0000 Subject: [PATCH 03/12] test(ffe): distinguish in-limit context from overflow pruning Require complete context at the field limit and preserve results, identity, and counts when oversized contexts are pruned or omitted. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- tests/ffe/README.md | 11 +++-- tests/ffe/test_flag_eval_evp.py | 53 ++++++++++++++++---- tests/ffe/test_tests.py | 88 +++++++++++++++++++++++++++++++-- 3 files changed, 133 insertions(+), 19 deletions(-) diff --git a/tests/ffe/README.md b/tests/ffe/README.md index 48d02fab2b5..15c30800cc6 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -35,10 +35,13 @@ Framework mock-backend and lifecycle tests stay in `tests/test_the_test/`. ## Bounds and overload coverage The consented context tests exercise field count and string length separately. -The count case accepts any valid retained subset (SDK map iteration order is not -part of the contract). The value case stays below the field-count cap, so that -cap cannot hide a missing string-length check. Missing evaluation context and -truncated, rather than omitted, oversized strings fail. +At the 256-field limit, every valid field must survive unchanged. Above the +limit, SDKs may retain any valid subset or omit evaluation context entirely to +bound capture work; map iteration order is not part of the contract. Both cases +require the correct flag result, consented identity, and exact evaluation count. +The value case stays below the field-count cap, so that cap cannot hide a missing +string-length check. Missing in-limit context and truncated, rather than omitted, +oversized strings fail. `Test_FFE_EVP_Flagevaluation_Oversized_Identity` is PHP-native IPC coverage. It sends a targeting key over 5 MiB, then a normal evaluation through the public diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index e042e0284ae..760655fde4c 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -398,6 +398,24 @@ def test_ffe_evp_flagevaluation_count(self) -> None: class Test_FFE_EVP_Flagevaluation_Context_Bounds: """Test that EVP evaluation context is bounded before it reaches payloads.""" + def setup_ffe_evp_flagevaluation_context_within_bounds(self) -> None: + self.within_flag_key = "evp-context-within-bounds-flag" + rc.tracer_rc_state.reset().set_config( + f"{RC_PATH}/ffe-evp-context-within-bounds/config", + make_ufc_fixture(self.within_flag_key, observe_full_evaluation_data=True), + ).apply() + self.within_attributes: JSON = {f"field_{index:03d}": f"value-{index}" for index in range(256)} + self.within_response = evaluate_flag( + self.within_flag_key, targeting_key="evp-context-user", attributes=self.within_attributes + ) + + def test_ffe_evp_flagevaluation_context_within_bounds(self) -> None: + events = self._assert_context_evaluation(self.within_response, self.within_flag_key) + for _, event in events: + context = event.get("context") + assert isinstance(context, dict), "in-limit consented evaluation must retain context" + assert context.get("evaluation") == self.within_attributes, "in-limit context fields must survive unchanged" + def setup_ffe_evp_flagevaluation_context_bounds(self) -> None: config_id = "ffe-evp-context-bounds" self.flag_key = "evp-context-bounds-flag" @@ -409,24 +427,39 @@ def setup_ffe_evp_flagevaluation_context_bounds(self) -> None: self.r = evaluate_flag(self.flag_key, targeting_key="evp-context-user", attributes=self.attributes) def test_ffe_evp_flagevaluation_context_bounds(self) -> None: - assert self.r.status_code == 200, f"Flag evaluation failed: {self.r.text}" - - events = find_evp_flagevaluation_events(self.flag_key) - assert events, f"Expected EVP flagevaluation event for flag {self.flag_key}" - + events = self._assert_context_evaluation(self.r, self.flag_key) for _, event in events: - assert_event_contract(event, self.flag_key) - context = event.get("context") - assert isinstance(context, dict), "consented evaluation must retain context" - evaluation_context = context.get("evaluation") + # SDKs may omit an over-limit map entirely to bound capture work. + # The separate in-limit case prevents unconditional context loss from passing. + if "context" not in event: + continue + context = event["context"] + assert isinstance(context, dict), "context must be an object when present" + if "evaluation" not in context: + continue + evaluation_context = context["evaluation"] assert isinstance(evaluation_context, dict), "context.evaluation must be an object" - assert evaluation_context, "valid context fields must survive pruning" assert len(evaluation_context) <= 256, f"context.evaluation has too many fields: {len(evaluation_context)}" # SDK maps need not preserve input order or choose the same retained subset. for key, value in evaluation_context.items(): assert key in self.attributes, f"unexpected context field: {key}" assert value == self.attributes[key], f"context field {key} changed during pruning" + @staticmethod + def _assert_context_evaluation(response: HttpResponse, flag_key: str) -> list[tuple[JSON, JSON]]: + assert response.status_code == 200, f"Flag evaluation failed: {response.text}" + assert json.loads(response.text).get("value") == "on-value", "context limits must not change the flag result" + events = find_evp_flagevaluation_events(flag_key) + assert_total_evaluation_count(events, 1, flag_key) + for _, event in events: + assert_event_contract(event, flag_key) + assert event.get("targeting_key") == "evp-context-user", "context limits must preserve consented identity" + assert object_key(event.get("variant"), "variant") == "on" + assert object_key(event.get("allocation"), "allocation") == "default-allocation" + assert "error" not in event + assert "runtime_default_used" not in event + return events + def setup_ffe_evp_flagevaluation_context_value_bounds(self) -> None: self.value_flag_key = "evp-context-value-bounds-flag" self.oversized_field = "oversized" diff --git a/tests/ffe/test_tests.py b/tests/ffe/test_tests.py index 37d65723163..166f69f47a5 100644 --- a/tests/ffe/test_tests.py +++ b/tests/ffe/test_tests.py @@ -98,11 +98,24 @@ def _bounded_context_event(flag_key: str) -> dict[str, Any]: [ ("first subset", True), ("last subset", True), - ("missing context", False), - ("empty evaluation", False), + ("missing context", True), + ("service only", True), + ("empty evaluation", True), + ("null context", False), + ("null evaluation", False), + ("invalid evaluation", False), ("too many fields", False), ("changed value", False), ("unexpected field", False), + ("missing identity", False), + ("wrong identity", False), + ("wrong count", False), + ("no event", False), + ("wrong result", False), + ("missing variant", False), + ("missing allocation", False), + ("runtime default", False), + ("error", False), ], ) @scenarios.feature_flagging_contract_tests @@ -110,7 +123,7 @@ def _bounded_context_event(flag_key: str) -> dict[str, Any]: def test_flagevaluation_context_count_contract(monkeypatch: pytest.MonkeyPatch, case: str, *, valid: bool) -> None: check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() check.flag_key = "bounds-contract" - check.r = MagicMock(status_code=200) + check.r = MagicMock(status_code=200, text='{"value":"on-value"}') check.attributes = {f"field_{index:03d}": f"value-{index}" for index in range(300)} retained = dict(list(check.attributes.items())[:256]) if case == "last subset": @@ -124,9 +137,35 @@ def test_flagevaluation_context_count_contract(monkeypatch: pytest.MonkeyPatch, elif case == "unexpected field": retained = {"not-supplied": "value"} event = _bounded_context_event(check.flag_key) + event["targeting_key"] = "evp-context-user" if case != "missing context": event["context"] = {"evaluation": retained} - monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: [({}, event)]) + if case == "service only": + event["context"] = {"dd": {"service": "weblog"}} + elif case == "null context": + event["context"] = None + elif case == "null evaluation": + event["context"]["evaluation"] = None + elif case == "invalid evaluation": + event["context"]["evaluation"] = [] + elif case == "missing identity": + del event["targeting_key"] + elif case == "wrong identity": + event["targeting_key"] = "different-user" + elif case == "wrong count": + event["evaluation_count"] = 2 + elif case == "wrong result": + check.r.text = '{"value":"default"}' + elif case == "missing variant": + del event["variant"] + elif case == "missing allocation": + del event["allocation"] + elif case == "runtime default": + event["runtime_default_used"] = True + elif case == "error": + event["error"] = {"message": "failed"} + events: list[tuple[dict[str, Any], dict[str, Any]]] = [] if case == "no event" else [({}, event)] + monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: events) if valid: check.test_ffe_evp_flagevaluation_context_bounds() else: @@ -134,6 +173,40 @@ def test_flagevaluation_context_count_contract(monkeypatch: pytest.MonkeyPatch, check.test_ffe_evp_flagevaluation_context_bounds() +@pytest.mark.parametrize( + "case", + ["valid", "missing context", "service only", "empty evaluation", "lost field", "changed value", "extra field"], +) +@scenarios.feature_flagging_contract_tests +@features.not_reported +def test_flagevaluation_context_within_bounds_contract(monkeypatch: pytest.MonkeyPatch, case: str) -> None: + check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() + check.within_flag_key = "within-bounds-contract" + check.within_response = MagicMock(status_code=200, text='{"value":"on-value"}') + check.within_attributes = {f"field_{index:03d}": f"value-{index}" for index in range(256)} + event = _bounded_context_event(check.within_flag_key) + event["targeting_key"] = "evp-context-user" + event["context"] = {"evaluation": dict(check.within_attributes)} + if case == "missing context": + del event["context"] + elif case == "service only": + event["context"] = {"dd": {"service": "weblog"}} + elif case == "empty evaluation": + event["context"]["evaluation"] = {} + elif case == "lost field": + del event["context"]["evaluation"]["field_000"] + elif case == "changed value": + event["context"]["evaluation"]["field_000"] = "changed" + elif case == "extra field": + event["context"]["evaluation"]["extra"] = "not supplied" + monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: [({}, event)]) + if case == "valid": + check.test_ffe_evp_flagevaluation_context_within_bounds() + else: + with pytest.raises(AssertionError): + check.test_ffe_evp_flagevaluation_context_within_bounds() + + @pytest.mark.parametrize("case", ["valid", "missing context", "service only", "lost sibling", "long", "truncated"]) @scenarios.feature_flagging_contract_tests @features.not_reported @@ -160,7 +233,7 @@ def test_flagevaluation_context_value_contract(monkeypatch: pytest.MonkeyPatch, check.test_ffe_evp_flagevaluation_context_value_bounds() -@pytest.mark.parametrize("kind", ["count", "value"]) +@pytest.mark.parametrize("kind", ["count", "within", "value"]) @scenarios.feature_flagging_contract_tests @features.not_reported def test_flagevaluation_bounds_fixtures_enable_consent(monkeypatch: pytest.MonkeyPatch, kind: str) -> None: @@ -171,6 +244,8 @@ def test_flagevaluation_bounds_fixtures_enable_consent(monkeypatch: pytest.Monke check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() if kind == "count": check.setup_ffe_evp_flagevaluation_context_bounds() + elif kind == "within": + check.setup_ffe_evp_flagevaluation_context_within_bounds() else: check.setup_ffe_evp_flagevaluation_context_value_bounds() fixture = state.reset.return_value.set_config.call_args.args[1] @@ -179,6 +254,9 @@ def test_flagevaluation_bounds_fixtures_enable_consent(monkeypatch: pytest.Monke if kind == "count": assert len(attributes) > 256 assert all(len(value) <= 256 for value in attributes.values()) + elif kind == "within": + assert len(attributes) == 256 + assert all(len(value) <= 256 for value in attributes.values()) else: assert len(attributes) < 256 assert len(attributes["oversized"]) > 256 From 2a6bc21f8550983436d54012f54498ee332620f8 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 22 Oct 2026 21:36:26 +0000 Subject: [PATCH 04/12] test(ffe): move oversized identity coverage to PHP Keep the shared suite focused on context bounds and overload accounting; cover native IPC degradation through the PHP API-to-wire tests. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- manifests/c.yml | 1 - manifests/cpp_httpd.yml | 1 - manifests/cpp_kong.yml | 1 - manifests/cpp_nginx.yml | 1 - manifests/dotnet.yml | 1 - manifests/golang.yml | 1 - manifests/java.yml | 1 - manifests/nodejs.yml | 1 - manifests/python.yml | 1 - manifests/ruby.yml | 1 - manifests/rust.yml | 1 - tests/ffe/README.md | 7 ---- tests/ffe/test_flag_eval_evp.py | 57 --------------------------- tests/test_the_test/test_ffe_tests.py | 36 ----------------- 14 files changed, 111 deletions(-) diff --git a/manifests/c.yml b/manifests/c.yml index b79f39c0363..416336889bb 100644 --- a/manifests/c.yml +++ b/manifests/c.yml @@ -4,7 +4,6 @@ manifest: tests/ai_guard/: irrelevant (dd-trace-c does not implement AI Guard) tests/appsec/: irrelevant (dd-trace-c does not implement AppSec or IAST) tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/integrations/test_dbm.py: incomplete_test_app (The Perl Mojolicious workload does not provide database endpoints) tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION: missing_feature tests/parametric/otel_env_vars/test_otel_exporter_otlp_headers.py::Test_OTEL_EXPORTER_OTLP_HEADERS: missing_feature diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 61dd9a90324..47030f0bcc9 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -57,7 +57,6 @@ manifest: tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/: missing_feature (Endpoint not implemented) diff --git a/manifests/cpp_kong.yml b/manifests/cpp_kong.yml index 2d59e7cb35b..fc369f3647c 100644 --- a/manifests/cpp_kong.yml +++ b/manifests/cpp_kong.yml @@ -10,7 +10,6 @@ manifest: tests/debugger/: irrelevant tests/ffe/: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integrations/: missing_feature (Endpoints not implemented) tests/otel/: irrelevant (library does not implement OpenTelemetry) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index e378866eb6c..30648f6498b 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -290,7 +290,6 @@ manifest: tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_kafka.py::Test_Kafka: missing_feature diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 10e0082a4c1..294c20704ef 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -852,7 +852,6 @@ manifest: - declaration: missing_feature (EX-3416) excluded_weblog: [poc, uds] tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: v3.44.0 tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_kafka.py::Test_Kafka: v2.0.0-prerelease diff --git a/manifests/golang.yml b/manifests/golang.yml index 322d7ada148..d4b4e0e1934 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1766,7 +1766,6 @@ manifest: tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: v2.11.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed: v2.11.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed: v2.11.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.10.0 - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: v2.8.0 tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Metric_Parse_Error_Invalid_Regex: irrelevant (Go validates regex at config load time) tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Nested_Attributes_Ignored: irrelevant (FFL-1980) diff --git a/manifests/java.yml b/manifests/java.yml index 841ccaef3c2..6f7fcc2d46f 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3496,7 +3496,6 @@ manifest: tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Caching_Serial_Id_Disappears: missing_feature (EX-3415) tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Serial_Id: missing_feature (EX-3415) tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 99b43458d79..94b6334410c 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1920,7 +1920,6 @@ manifest: - weblog_declaration: "*": incomplete_test_app express5: v7.0.0-pre - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": incomplete_test_app diff --git a/manifests/python.yml b/manifests/python.yml index 54c5f272090..dbe1dbde15b 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2270,7 +2270,6 @@ manifest: tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed: missing_feature (FFL-2446) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Runtime_Default: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: v4.7.0 tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Nested_Attributes_Ignored: irrelevant (FFL-1980) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index a2a1b2ce408..5a7cf90a6ae 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2153,7 +2153,6 @@ manifest: tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Caching_Serial_Id_Disappears: missing_feature (EX-3417) tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Serial_Id: missing_feature (EX-3417) tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/rust.yml b/manifests/rust.yml index e700ba32792..67215ed811e 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -26,7 +26,6 @@ manifest: tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Oversized_Identity: irrelevant (PHP native IPC size limit regression) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/: incomplete_test_app (crossed integration endpoints are not implemented) diff --git a/tests/ffe/README.md b/tests/ffe/README.md index 77729b79013..6567cdf41af 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -40,13 +40,6 @@ The value case stays below the field-count cap, so that cap cannot hide a missin string-length check. Missing in-limit context and truncated, rather than omitted, oversized strings fail. -`Test_FFE_EVP_Flagevaluation_Oversized_Identity` is PHP-native IPC coverage. -It sends a targeting key over 5 MiB, then a normal evaluation through the public -weblog endpoint. Counts and customer results must survive; the oversized event -loses identity/context and the successor retains consented context. Existing -manifests exclude this case from other SDKs. PHP release activation remains a -separate task; use `-F` to validate an unreleased implementation locally. - The high-cardinality workload runs in its own scenario: ```sh diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 96ba37a64fd..cfcfa1365c7 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -692,63 +692,6 @@ def test_ffe_evp_flagevaluation_high_cardinality_accounting(self) -> None: assert "runtime_default_used" not in event -@scenarios.feature_flagging_and_experimentation -@features.feature_flags_evp_flagevaluation -@slow -class Test_FFE_EVP_Flagevaluation_Oversized_Identity: - """Preserve PHP flag results and counts across native IPC size degradation.""" - - def setup_ffe_evp_flagevaluation_oversized_identity(self) -> None: - config_id = "ffe-evp-oversized-identity" - self.flag_key = "evp-oversized-identity-flag" - self.control_targeting_key = "evp-size-control-user" - rc.tracer_rc_state.reset().set_config( - f"{RC_PATH}/{config_id}/config", make_ufc_fixture(self.flag_key, observe_full_evaluation_data=True) - ).apply() - - # PHP's native IPC path has a 5 MiB event limit; other SDKs are excluded in manifests. - # The normal successor also verifies that oversize fallback leaves IPC usable. - oversized_targeting_key = "x" * (5 * 1024 * 1024 + 1) - self.responses = [ - evaluate_flag( - self.flag_key, - targeting_key=targeting_key, - attributes={"email": "evp-size-canary@example.invalid"}, - ) - for targeting_key in (oversized_targeting_key, self.control_targeting_key) - ] - - def test_ffe_evp_flagevaluation_oversized_identity(self) -> None: - for response in self.responses: - assert response.status_code == 200, f"Flag evaluation failed: {response.text}" - assert json.loads(response.text).get("value") == "on-value", ( - "telemetry limits must not change the flag result" - ) - - events = find_evp_flagevaluation_events(self.flag_key) - assert_total_evaluation_count(events, 2, self.flag_key) - - degraded_events = [(batch, event) for batch, event in events if "targeting_key" not in event] - assert_total_evaluation_count(degraded_events, 1, self.flag_key) - for _, event in degraded_events: - assert "context" not in event, "size-degraded events must omit all event context" - - control_events = [ - (batch, event) for batch, event in events if event.get("targeting_key") == self.control_targeting_key - ] - assert_total_evaluation_count(control_events, 1, self.flag_key) - for _, event in control_events: - assert event.get("context", {}).get("evaluation", {}).get("email") == "evp-size-canary@example.invalid" - - for batch, event in events: - assert_batch_context(batch) - assert_event_contract(event, self.flag_key) - assert object_key(event.get("variant"), "variant") == "on" - assert object_key(event.get("allocation"), "allocation") == "default-allocation" - assert "error" not in event - assert "runtime_default_used" not in event - - @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation @slow diff --git a/tests/test_the_test/test_ffe_tests.py b/tests/test_the_test/test_ffe_tests.py index 2db530e6412..890ecc35e36 100644 --- a/tests/test_the_test/test_ffe_tests.py +++ b/tests/test_the_test/test_ffe_tests.py @@ -319,42 +319,6 @@ def test_flagevaluation_bounds_fixtures_enable_consent(monkeypatch: pytest.Monke assert attributes["valid"] == "retained" -@pytest.mark.parametrize( - "case", - ["valid", "dropped count", "retained identity", "retained context", "lost control", "lost variant", "wrong result"], -) -@scenarios.test_the_test -def test_flagevaluation_php_oversized_identity_contract(monkeypatch: pytest.MonkeyPatch, case: str) -> None: - check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Oversized_Identity() - check.flag_key = "oversized-contract" - check.control_targeting_key = "control" - check.responses = [MagicMock(status_code=200, text='{"value":"on-value"}')] * 2 - batch = {"context": {"service": "weblog"}} - degraded = _bounded_context_event(check.flag_key) - control = _bounded_context_event(check.flag_key) - control["targeting_key"] = check.control_targeting_key - control["context"] = {"evaluation": {"email": "evp-size-canary@example.invalid"}} - rows = [(batch, degraded), (batch, control)] - if case == "dropped count": - rows.pop(0) - elif case == "retained identity": - degraded["targeting_key"] = "must-not-survive" - elif case == "retained context": - degraded["context"] = {} - elif case == "lost control": - del control["context"] - elif case == "lost variant": - del degraded["variant"] - elif case == "wrong result": - check.responses = [MagicMock(status_code=200, text='{"value":"default"}')] - monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: rows) - if case == "valid": - check.test_ffe_evp_flagevaluation_oversized_identity() - else: - with pytest.raises(AssertionError): - check.test_ffe_evp_flagevaluation_oversized_identity() - - @pytest.mark.parametrize("has_capture", [True, False]) @scenarios.test_the_test def test_flagevaluation_egress_validates_captured_data_without_waiting( From bb1a1b468e606988b4e4278938ce8279180f9788 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 22 Oct 2026 22:10:58 +0000 Subject: [PATCH 05/12] test(ffe): clarify high-cardinality accounting scope Describe flush timing without claiming exact-cap coverage in SDK tests. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- tests/ffe/test_flag_eval_evp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index cfcfa1365c7..b13b63c245c 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -677,8 +677,8 @@ def test_ffe_evp_flagevaluation_high_cardinality_accounting(self) -> None: ] assert_flagevaluation_accounting(sum_evaluation_count(events), self.eval_count, drop_series) - # A slow run may span many flushes and never degrade. Native tests prove - # the exact cap; every delivered row must still have a valid tier shape. + # A run may span multiple flushes and never degrade. + # Every delivered row must still have a valid tier shape. for _, event in events: assert_event_contract(event, self.flag_key) if "targeting_key" not in event: From 42df470a9c35b1d0e25cb49463fb6e1f2d8b382d Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 22 Oct 2026 22:20:53 +0000 Subject: [PATCH 06/12] docs(ffe): shorten bounds and accounting guidance Keep the shared contract, run command, and isolation requirement; link to implementation details. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- tests/ffe/README.md | 49 ++++++++++----------------------------------- 1 file changed, 11 insertions(+), 38 deletions(-) diff --git a/tests/ffe/README.md b/tests/ffe/README.md index 6567cdf41af..19bd965617c 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -31,49 +31,22 @@ Framework mock-backend and lifecycle tests stay in `tests/test_the_test/`. ## Bounds and overload coverage -The consented context tests exercise field count and string length separately. -At the 256-field limit, every valid field must survive unchanged. Above the -limit, SDKs may retain any valid subset or omit evaluation context entirely to -bound capture work; map iteration order is not part of the contract. Both cases -require the correct flag result, consented identity, and exact evaluation count. -The value case stays below the field-count cap, so that cap cannot hide a missing -string-length check. Missing in-limit context and truncated, rather than omitted, -oversized strings fail. +With full-data consent, the context tests require all 256 valid attributes to +survive unchanged at the field-count limit. Over-limit contexts may retain an +unchanged subset of at most 256 attributes or be omitted entirely. A separate +string-length test requires oversized values to be omitted while valid fields survive. -The high-cardinality workload runs in its own scenario: +Run the high-cardinality accounting test in its dedicated scenario: ```sh -./run.sh FEATURE_FLAGGING_EVP_STRESS --library php -F tests/ffe/test_flag_eval_evp.py +./run.sh FEATURE_FLAGGING_EVP_STRESS --library -F tests/ffe/test_flag_eval_evp.py ``` -Do not add other evaluation workloads to this scenario: drop counters are -process-wide, not flag-scoped. It checks that delivered evaluation counts plus -reported capacity drops equal 12,000 attempted evaluations. It rejects missing -accounting, total loss, serialization/delivery failures, and invalid counters. -Degradation counts are not drops: degraded evaluations are already included in -the delivered count. This is an overload-accounting test, not a lossless-delivery -or throughput guarantee. Normal count and aggregation tests still require -lossless delivery. - -The accepted drop counters are `flagevaluation.rows.dropped` (SDK submission) -and `flagevaluation.evaluations.dropped` (native writer), in `tracers` or `general` -telemetry namespaces. Only `pre_queue_overflow`, `queue_overflow`, `degraded_cap`, -and `payload_limit` reasons count as capacity drops. A new SDK counter convention -needs explicit coverage here, not an inferred allowance for missing evaluations. - -The external workload does not guarantee degradation: SDK flush windows can -split it before the per-flag cap is reached. Exact cap-boundary proof belongs in -SDK/native tests with synchronous enqueueing. PHP's pinned libdatadog has -`coalescer_degrades_after_per_flag_cap` in `libdd-ffe`, which verifies 10,000 full -rows plus 50 degraded evaluations. No production timing override is used. - -This replaces the two-window degradation workload from -[PR #8133](https://github.com/DataDog/system-tests/pull/8133), retaining its short -numeric targeting keys to stay within weblog request-body limits. Doubling the -workload handles one flush boundary, but does not guarantee degradation when a -slow request spans more windows. The isolated accounting test therefore keeps -12,000 evaluations and validates both full and degraded rows without requiring -degradation to occur. +Keep this scenario limited to one evaluation workload: its drop counters are +process-wide. Delivered evaluation counts (including degraded delivery) plus +accepted reported drops must equal attempted evaluations, and some delivery is +required. See the [accounting helper](utils/evp.py) for accepted counters and validation. +Periodic flushes may prevent degradation, so the test does not require it to occur. --- From 30ad97d6af54f04d6286dd5453ba5ada4aabdf8c Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 22 Oct 2026 22:24:56 +0000 Subject: [PATCH 07/12] test(ffe): simplify stress assertion self-tests Check workload properties and cover each malformed capture once while retaining valid flush layouts. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- tests/test_the_test/test_ffe_tests.py | 31 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/test_the_test/test_ffe_tests.py b/tests/test_the_test/test_ffe_tests.py index 890ecc35e36..69b1df6e7ab 100644 --- a/tests/test_the_test/test_ffe_tests.py +++ b/tests/test_the_test/test_ffe_tests.py @@ -103,14 +103,25 @@ def test_flagevaluation_stress_request_uses_compact_unique_keys(monkeypatch: pyt post.assert_called_once() assert post.call_args.args == ("/ffe",) payload = post.call_args.kwargs["json"] - assert payload["targetingKeys"] == [str(index) for index in range(12_000)] - assert len(set(payload["targetingKeys"])) == check.eval_count == 12_000 - assert payload["targetingKey"] == "0" + targeting_keys = payload["targetingKeys"] + assert len(targeting_keys) == len(set(targeting_keys)) == check.eval_count == 12_000 + assert all(isinstance(key, str) and key for key in targeting_keys) + assert payload["targetingKey"] in targeting_keys assert len(json.dumps(payload).encode("utf-8")) < 256 * 1024 -@pytest.mark.parametrize("windows", [(12_000,), (6_000, 6_000), (3_000, 3_000, 3_000, 3_000)]) -@pytest.mark.parametrize("mutation", ["none", "missing count", "duplicate", "invalid tier shape"]) +@pytest.mark.parametrize( + ("windows", "mutation"), + [ + ((12_000,), "none"), + ((6_000, 6_000), "none"), + ((3_000, 3_000, 3_000, 3_000), "none"), + ((12_000,), "missing count"), + ((12_000,), "duplicate"), + ((12_000,), "invalid full tier"), + ((12_000,), "invalid degraded tier"), + ], +) @scenarios.test_the_test def test_flagevaluation_stress_contract_across_flush_windows( monkeypatch: pytest.MonkeyPatch, windows: tuple[int, ...], mutation: str @@ -138,12 +149,10 @@ def test_flagevaluation_stress_contract_across_flush_windows( rows.pop(0) elif mutation == "duplicate": rows[1] = rows[0] - elif mutation == "invalid tier shape": - event = rows[-1][1] - if "targeting_key" in event: - event["targeting_key"] = "" - else: - event["context"] = {} + elif mutation == "invalid full tier": + rows[0][1]["targeting_key"] = "" + elif mutation == "invalid degraded tier": + rows[-1][1]["context"] = {} monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: rows) monkeypatch.setattr(interfaces.library, "get_telemetry_metric_series", MagicMock(return_value=[])) if mutation == "none": From 5dc5703791b41f461b6a599a400886ec7852b44d Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 22 Oct 2026 22:31:33 +0000 Subject: [PATCH 08/12] test(ffe): inherit PHP accounting skip from file manifest Remove the redundant class-level missing-feature declaration. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- manifests/php.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/manifests/php.yml b/manifests/php.yml index 90cd1ef29db..4f9c8eafdb8 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1280,7 +1280,6 @@ manifest: excluded_weblog: [apache-mod-8.0-zts, apache-mod-7.1-zts, apache-mod-8.0, apache-mod-7.4, php-fpm-7.2, php-fpm-8.1, apache-mod-7.0, apache-mod-7.0-zts, php-fpm-7.0, php-fpm-8.0, php-fpm-8.2, apache-mod-7.2-zts, apache-mod-8.1-zts, apache-mod-7.1, php-fpm-8.5, apache-mod-8.2-zts, php-fpm-7.3, php-fpm-7.4, apache-mod-7.2, apache-mod-7.4-zts, apache-mod-8.2, apache-mod-7.3, apache-mod-8.1, php-fpm-7.1, apache-mod-7.3-zts] tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Burst_Aggregation: bug (FFL-2676) - tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Accounting: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Aggregation: bug (FFL-2676) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: From 5682d59da7dd98320127a5985b380edc73d82cec Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 22 Oct 2026 23:27:26 +0000 Subject: [PATCH 09/12] fix(ffe): preserve large numeric attributes in Java weblog Pass Jackson Long attributes through OpenFeature numeric context instead of replacing them with null. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- .../featureflag/FeatureFlagEvaluatorController.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/featureflag/FeatureFlagEvaluatorController.java b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/featureflag/FeatureFlagEvaluatorController.java index 16a2e6b8875..023360ae5d1 100644 --- a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/featureflag/FeatureFlagEvaluatorController.java +++ b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/featureflag/FeatureFlagEvaluatorController.java @@ -123,8 +123,9 @@ private static EvaluationContext context(final EvaluateRequest request, final St context.add(key, (Boolean) value); } else if (value instanceof Integer) { context.add(key, (Integer) value); - } else if (value instanceof Double) { - context.add(key, (Double) value); + } else if (value instanceof Number) { + // Jackson uses Long for JSON integers outside the Integer range. + context.add(key, ((Number) value).doubleValue()); } else if (value instanceof String) { context.add(key, (String) value); } else if (value instanceof Map) { From 7ead0b2401307ee22ca82ebd2ee860499e3f72ff Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 22 Oct 2026 23:42:19 +0000 Subject: [PATCH 10/12] fix(ffe): preserve JSON context types in .NET weblog Build typed OpenFeature values instead of stringifying numeric, boolean, and structured evaluation attributes. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- .../FeatureFlagEvaluatorController.cs | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs b/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs index 183cd1bcf9f..9247290a2bb 100644 --- a/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs +++ b/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs @@ -7,6 +7,7 @@ using OpenFeature.Model; using System; using System.Collections.Generic; +using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Tasks; @@ -139,14 +140,12 @@ private static EvaluationContext CreateContext(EvaluateRequest request, string t { foreach (var attr in request.Attributes) { - // System.Text.Json deserializes to JsonElement, not string + // Preserve JSON types when building the OpenFeature context. var value = attr.Value switch { - JsonElement jsonElement => jsonElement.ValueKind == JsonValueKind.String - ? jsonElement.GetString() - : jsonElement.ToString(), - string s => s, - _ => attr.Value?.ToString() + JsonElement jsonElement => JsonElementToValue(jsonElement), + string s => new Value(s), + _ => new Value(attr.Value?.ToString()) }; builder.Set(attr.Key, value); } @@ -154,6 +153,21 @@ private static EvaluationContext CreateContext(EvaluateRequest request, string t return builder.Build(); } + private static Value JsonElementToValue(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => new Value(element.GetString()), + JsonValueKind.Number => new Value(element.GetDouble()), + JsonValueKind.True => new Value(true), + JsonValueKind.False => new Value(false), + JsonValueKind.Array => new Value(element.EnumerateArray().Select(JsonElementToValue).ToList()), + JsonValueKind.Object => new Value(new Structure(element.EnumerateObject() + .ToDictionary(property => property.Name, property => JsonElementToValue(property.Value)))), + _ => new Value() + }; + } + public class EvaluateRequest { [JsonPropertyName("flag")] From 3747f03b854d9b56bcc91b046f1e59b6058a2c3d Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Fri, 23 Oct 2026 17:49:23 +0000 Subject: [PATCH 11/12] test(ffe): remove added test-class self-tests Remove the FFE self-tests added by this PR while preserving the pre-existing coverage. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- tests/test_the_test/test_ffe_tests.py | 294 +------------------------- 1 file changed, 1 insertion(+), 293 deletions(-) diff --git a/tests/test_the_test/test_ffe_tests.py b/tests/test_the_test/test_ffe_tests.py index 605381b4f9e..71a67fc439e 100644 --- a/tests/test_the_test/test_ffe_tests.py +++ b/tests/test_the_test/test_ffe_tests.py @@ -1,7 +1,6 @@ -"""Unit coverage for the shared Feature Flags EVP test contract.""" +"""Opt-in unit coverage for the shared Feature Flags EVP test contract.""" from copy import deepcopy -import json from typing import Any, Literal from unittest.mock import MagicMock @@ -16,7 +15,6 @@ assert_agentless_evp_topology, assert_direct_evp_runtime_evidence, assert_direct_evp_shutdown_evidence, - assert_flagevaluation_accounting, assert_no_evp_proxy_requests, expected_evp_origin, ) @@ -38,296 +36,6 @@ ) -@pytest.mark.parametrize( - ("delivered", "dropped", "reason", "valid"), - [ - (12_000, None, "queue_overflow", True), - (11_852, 148.0, "queue_overflow", True), - (11_950, 50, "degraded_cap", True), - (11_999, None, "queue_overflow", False), - (12_000, 1, "queue_overflow", False), - (0, 12_000, "queue_overflow", False), - (11_999, 1, "serialization_error", False), - (11_999, 1, "delivery_failure", False), - (11_999, 1.5, "queue_overflow", False), - (12_001, -1, "queue_overflow", False), - (11_999, True, "queue_overflow", False), - (11_999, float("inf"), "queue_overflow", False), - ], -) -@scenarios.test_the_test -def test_flagevaluation_isolated_accounting(delivered: int, dropped: float | None, reason: str, *, valid: bool) -> None: - series = [] if dropped is None else [{"type": "count", "tags": [f"reason:{reason}"], "points": [[1, dropped]]}] - if valid: - assert_flagevaluation_accounting(delivered, 12_000, series) - else: - with pytest.raises(AssertionError): - assert_flagevaluation_accounting(delivered, 12_000, series) - - -@pytest.mark.parametrize( - "series", - [ - {"type": "gauge", "tags": ["reason:queue_overflow"], "points": [[1, 1]]}, - {"type": "count", "tags": [], "points": [[1, 1]]}, - {"type": "count", "tags": ["reason:queue_overflow", "reason:degraded_cap"], "points": [[1, 1]]}, - {"type": "count", "tags": ["reason:queue_overflow"], "points": []}, - ], -) -@scenarios.test_the_test -def test_flagevaluation_accounting_rejects_invalid_telemetry(series: dict[str, Any]) -> None: - with pytest.raises(AssertionError): - assert_flagevaluation_accounting(11_999, 12_000, [series]) - - -def _bounded_context_event(flag_key: str) -> dict[str, Any]: - return { - "flag": {"key": flag_key}, - "variant": {"key": "on"}, - "allocation": {"key": "default-allocation"}, - "timestamp": 1, - "first_evaluation": 1, - "last_evaluation": 1, - "evaluation_count": 1, - } - - -@scenarios.test_the_test -def test_flagevaluation_stress_request_uses_compact_unique_keys(monkeypatch: pytest.MonkeyPatch) -> None: - post = MagicMock() - monkeypatch.setattr(flag_eval_evp_tests.rc, "tracer_rc_state", MagicMock()) - monkeypatch.setattr(flag_eval_evp_tests.weblog, "post", post) - check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_High_Cardinality_Accounting() - check.setup_ffe_evp_flagevaluation_high_cardinality_accounting() - - post.assert_called_once() - assert post.call_args.args == ("/ffe",) - payload = post.call_args.kwargs["json"] - targeting_keys = payload["targetingKeys"] - assert len(targeting_keys) == len(set(targeting_keys)) == check.eval_count == 12_000 - assert all(isinstance(key, str) and key for key in targeting_keys) - assert payload["targetingKey"] in targeting_keys - assert len(json.dumps(payload).encode("utf-8")) < 256 * 1024 - - -@pytest.mark.parametrize( - ("windows", "mutation"), - [ - ((12_000,), "none"), - ((6_000, 6_000), "none"), - ((3_000, 3_000, 3_000, 3_000), "none"), - ((12_000,), "missing count"), - ((12_000,), "duplicate"), - ((12_000,), "invalid full tier"), - ((12_000,), "invalid degraded tier"), - ], -) -@scenarios.test_the_test -def test_flagevaluation_stress_contract_across_flush_windows( - monkeypatch: pytest.MonkeyPatch, windows: tuple[int, ...], mutation: str -) -> None: - check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_High_Cardinality_Accounting() - check.flag_key = "stress-window-contract" - check.eval_count = 12_000 - check.responses = [MagicMock(status_code=200, text='{"value":"on-value","count":12000}')] - rows = [] - next_key = 0 - for count in windows: - batch = {"context": {"service": "weblog"}} - # Model native per-window coalescing; this exercises the external assertion, - # not the SDK cap itself. Split windows can legitimately contain no degradation. - for _ in range(min(count, 10_000)): - event = _bounded_context_event(check.flag_key) - event["targeting_key"] = str(next_key) - next_key += 1 - rows.append((batch, event)) - if count > 10_000: - event = _bounded_context_event(check.flag_key) - event["evaluation_count"] = count - 10_000 - rows.append((batch, event)) - if mutation == "missing count": - rows.pop(0) - elif mutation == "duplicate": - rows[1] = rows[0] - elif mutation == "invalid full tier": - rows[0][1]["targeting_key"] = "" - elif mutation == "invalid degraded tier": - rows[-1][1]["context"] = {} - monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: rows) - monkeypatch.setattr(interfaces.library, "get_telemetry_metric_series", MagicMock(return_value=[])) - if mutation == "none": - check.test_ffe_evp_flagevaluation_high_cardinality_accounting() - else: - with pytest.raises(AssertionError): - check.test_ffe_evp_flagevaluation_high_cardinality_accounting() - - -@pytest.mark.parametrize( - ("case", "valid"), - [ - ("first subset", True), - ("last subset", True), - ("missing context", True), - ("service only", True), - ("empty evaluation", True), - ("null context", False), - ("null evaluation", False), - ("invalid evaluation", False), - ("too many fields", False), - ("changed value", False), - ("unexpected field", False), - ("missing identity", False), - ("wrong identity", False), - ("wrong count", False), - ("no event", False), - ("wrong result", False), - ("missing variant", False), - ("missing allocation", False), - ("runtime default", False), - ("error", False), - ], -) -@scenarios.test_the_test -def test_flagevaluation_context_count_contract(monkeypatch: pytest.MonkeyPatch, case: str, *, valid: bool) -> None: - check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() - check.flag_key = "bounds-contract" - check.r = MagicMock(status_code=200, text='{"value":"on-value"}') - check.attributes = {f"field_{index:03d}": f"value-{index}" for index in range(300)} - retained = dict(list(check.attributes.items())[:256]) - if case == "last subset": - retained = dict(list(check.attributes.items())[-256:]) - elif case == "empty evaluation": - retained = {} - elif case == "too many fields": - retained = check.attributes - elif case == "changed value": - retained["field_000"] = "changed" - elif case == "unexpected field": - retained = {"not-supplied": "value"} - event = _bounded_context_event(check.flag_key) - event["targeting_key"] = "evp-context-user" - if case != "missing context": - event["context"] = {"evaluation": retained} - if case == "service only": - event["context"] = {"dd": {"service": "weblog"}} - elif case == "null context": - event["context"] = None - elif case == "null evaluation": - event["context"]["evaluation"] = None - elif case == "invalid evaluation": - event["context"]["evaluation"] = [] - elif case == "missing identity": - del event["targeting_key"] - elif case == "wrong identity": - event["targeting_key"] = "different-user" - elif case == "wrong count": - event["evaluation_count"] = 2 - elif case == "wrong result": - check.r.text = '{"value":"default"}' - elif case == "missing variant": - del event["variant"] - elif case == "missing allocation": - del event["allocation"] - elif case == "runtime default": - event["runtime_default_used"] = True - elif case == "error": - event["error"] = {"message": "failed"} - events: list[tuple[dict[str, Any], dict[str, Any]]] = [] if case == "no event" else [({}, event)] - monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: events) - if valid: - check.test_ffe_evp_flagevaluation_context_bounds() - else: - with pytest.raises(AssertionError): - check.test_ffe_evp_flagevaluation_context_bounds() - - -@pytest.mark.parametrize( - "case", - ["valid", "missing context", "service only", "empty evaluation", "lost field", "changed value", "extra field"], -) -@scenarios.test_the_test -def test_flagevaluation_context_within_bounds_contract(monkeypatch: pytest.MonkeyPatch, case: str) -> None: - check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() - check.within_flag_key = "within-bounds-contract" - check.within_response = MagicMock(status_code=200, text='{"value":"on-value"}') - check.within_attributes = {f"field_{index:03d}": f"value-{index}" for index in range(256)} - event = _bounded_context_event(check.within_flag_key) - event["targeting_key"] = "evp-context-user" - event["context"] = {"evaluation": dict(check.within_attributes)} - if case == "missing context": - del event["context"] - elif case == "service only": - event["context"] = {"dd": {"service": "weblog"}} - elif case == "empty evaluation": - event["context"]["evaluation"] = {} - elif case == "lost field": - del event["context"]["evaluation"]["field_000"] - elif case == "changed value": - event["context"]["evaluation"]["field_000"] = "changed" - elif case == "extra field": - event["context"]["evaluation"]["extra"] = "not supplied" - monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: [({}, event)]) - if case == "valid": - check.test_ffe_evp_flagevaluation_context_within_bounds() - else: - with pytest.raises(AssertionError): - check.test_ffe_evp_flagevaluation_context_within_bounds() - - -@pytest.mark.parametrize("case", ["valid", "missing context", "service only", "lost sibling", "long", "truncated"]) -@scenarios.test_the_test -def test_flagevaluation_context_value_contract(monkeypatch: pytest.MonkeyPatch, case: str) -> None: - check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() - check.value_flag_key = "value-bounds-contract" - check.oversized_field = "oversized" - check.value_response = MagicMock(status_code=200) - event = _bounded_context_event(check.value_flag_key) - event["context"] = {"evaluation": {"valid": "retained"}} - if case == "missing context": - del event["context"] - elif case == "service only": - event["context"] = {"dd": {"service": "weblog"}} - elif case == "lost sibling": - event["context"]["evaluation"] = {} - elif case in ("long", "truncated"): - event["context"]["evaluation"]["oversized"] = "x" * (300 if case == "long" else 256) - monkeypatch.setattr(flag_eval_evp_tests, "find_evp_flagevaluation_events", lambda _: [({}, event)]) - if case == "valid": - check.test_ffe_evp_flagevaluation_context_value_bounds() - else: - with pytest.raises(AssertionError): - check.test_ffe_evp_flagevaluation_context_value_bounds() - - -@pytest.mark.parametrize("kind", ["count", "within", "value"]) -@scenarios.test_the_test -def test_flagevaluation_bounds_fixtures_enable_consent(monkeypatch: pytest.MonkeyPatch, kind: str) -> None: - state = MagicMock() - evaluate = MagicMock() - monkeypatch.setattr(flag_eval_evp_tests.rc, "tracer_rc_state", state) - monkeypatch.setattr(flag_eval_evp_tests, "evaluate_flag", evaluate) - check = flag_eval_evp_tests.Test_FFE_EVP_Flagevaluation_Context_Bounds() - if kind == "count": - check.setup_ffe_evp_flagevaluation_context_bounds() - elif kind == "within": - check.setup_ffe_evp_flagevaluation_context_within_bounds() - else: - check.setup_ffe_evp_flagevaluation_context_value_bounds() - fixture = state.reset.return_value.set_config.call_args.args[1] - assert fixture["observeFullEvaluationData"] is True - attributes = evaluate.call_args.kwargs["attributes"] - if kind == "count": - assert len(attributes) > 256 - assert all(len(value) <= 256 for value in attributes.values()) - elif kind == "within": - assert len(attributes) == 256 - assert all(len(value) <= 256 for value in attributes.values()) - else: - assert len(attributes) < 256 - assert len(attributes["oversized"]) > 256 - assert attributes["valid"] == "retained" - - @pytest.mark.parametrize("has_capture", [True, False]) @scenarios.test_the_test def test_flagevaluation_egress_validates_captured_data_without_waiting( From 9b6ae1cbd822fcba0a2cf10da318188dbfcd11be Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Mon, 26 Oct 2026 19:54:06 +0000 Subject: [PATCH 12/12] test(ffe): address accounting and context review feedback Preserve longer capture waits while allowing telemetry to flush, gate Go drop accounting at 2.11.0, and apply shared outcome checks to oversized values. Clarify counter units and batch semantics. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- docs/understand/weblogs/end-to-end_weblog.md | 2 +- manifests/golang.yml | 1 + tests/ffe/README.md | 7 +++++-- tests/ffe/test_flag_eval_evp.py | 18 ++++++++++-------- utils/_context/_scenarios/feature_flagging.py | 8 ++++++++ 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/understand/weblogs/end-to-end_weblog.md b/docs/understand/weblogs/end-to-end_weblog.md index 05ef4ef1c7a..8b4baa42233 100644 --- a/docs/understand/weblogs/end-to-end_weblog.md +++ b/docs/understand/weblogs/end-to-end_weblog.md @@ -1156,7 +1156,7 @@ accept a JSON body with these fields: For the high-cardinality FFE workload, the endpoint also accepts a non-empty `targetingKeys` array. Evaluate the same flag once per key through the public SDK API, using the same attributes and default value. The response includes `count` -(the number of evaluations performed) and the final evaluation's `value`. +(the number of targeting keys in the request) and the final evaluation's `value`. Do not replace the loop with a direct telemetry submission or change SDK flush timing. When the array is absent or empty, evaluate the single `targetingKey`. diff --git a/manifests/golang.yml b/manifests/golang.yml index 565e95b90ec..4f0d3c37ee5 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1767,6 +1767,7 @@ manifest: tests/ffe/test_flag_eval_evp.py: v2.10.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Egress_Agentless_Direct: v2.12.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Egress_Agentless_Sidecar: v2.12.0-dev + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Accounting: v2.11.0 tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: v2.11.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed: v2.11.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed: v2.11.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.10.0 diff --git a/tests/ffe/README.md b/tests/ffe/README.md index f47164b40e2..ea26b4f63df 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -130,14 +130,17 @@ string-length test requires oversized values to be omitted while valid fields su Run the high-cardinality accounting test in its dedicated scenario: ```sh -./run.sh FEATURE_FLAGGING_EVP_STRESS --library -F tests/ffe/test_flag_eval_evp.py +./run.sh FEATURE_FLAGGING_EVP_STRESS --library ``` Keep this scenario limited to one evaluation workload: its drop counters are process-wide. Delivered evaluation counts (including degraded delivery) plus accepted reported drops must equal attempted evaluations, and some delivery is required. See the [accounting helper](utils/evp.py) for accepted counters and validation. -Periodic flushes may prevent degradation, so the test does not require it to occur. +Periodic flushes may prevent degradation, so the test validates degraded rows when +present but does not require them. Deterministic degradation-threshold checks belong +in SDK-local tests. Drop counters must report lost evaluations, including those +represented by aggregated rows. --- diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index b13b63c245c..1f2fa7522ff 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -446,14 +446,16 @@ def test_ffe_evp_flagevaluation_context_bounds(self) -> None: assert value == self.attributes[key], f"context field {key} changed during pruning" @staticmethod - def _assert_context_evaluation(response: HttpResponse, flag_key: str) -> list[tuple[JSON, JSON]]: + def _assert_context_evaluation( + response: HttpResponse, flag_key: str, targeting_key: str = "evp-context-user" + ) -> list[tuple[JSON, JSON]]: assert response.status_code == 200, f"Flag evaluation failed: {response.text}" assert json.loads(response.text).get("value") == "on-value", "context limits must not change the flag result" events = find_evp_flagevaluation_events(flag_key) assert_total_evaluation_count(events, 1, flag_key) for _, event in events: assert_event_contract(event, flag_key) - assert event.get("targeting_key") == "evp-context-user", "context limits must preserve consented identity" + assert event.get("targeting_key") == targeting_key, "context limits must preserve consented identity" assert object_key(event.get("variant"), "variant") == "on" assert object_key(event.get("allocation"), "allocation") == "default-allocation" assert "error" not in event @@ -476,12 +478,10 @@ def setup_ffe_evp_flagevaluation_context_value_bounds(self) -> None: ) def test_ffe_evp_flagevaluation_context_value_bounds(self) -> None: - assert self.value_response.status_code == 200, f"Flag evaluation failed: {self.value_response.text}" - events = find_evp_flagevaluation_events(self.value_flag_key) - assert events, f"Expected EVP flagevaluation event for flag {self.value_flag_key}" - + events = self._assert_context_evaluation( + self.value_response, self.value_flag_key, targeting_key="evp-context-value-user" + ) for _, event in events: - assert_event_contract(event, self.value_flag_key) context = event.get("context") assert isinstance(context, dict), "consented evaluation must retain context" evaluation_context = context.get("evaluation") @@ -661,7 +661,7 @@ def test_ffe_evp_flagevaluation_high_cardinality_accounting(self) -> None: assert response.status_code == 200, f"Request {index + 1} failed: {response.text}" result = json.loads(response.text) assert result.get("value") == "on-value", "overload must not change the flag result" - assert result.get("count") == self.eval_count, "weblog must execute the whole workload" + assert result.get("count") == self.eval_count, "weblog must accept the whole targetingKeys batch" events = find_evp_flagevaluation_events(self.flag_key) assert events, f"Expected EVP flagevaluation events for flag {self.flag_key}" @@ -669,6 +669,8 @@ def test_ffe_evp_flagevaluation_high_cardinality_accounting(self) -> None: assert_no_duplicate_visible_events(events) # These counters are process-wide, not flag-scoped: this class must remain # the only evaluation workload in FEATURE_FLAGGING_EVP_STRESS. + # Tracers report rows.dropped; libdatadog's sidecar reports evaluations.dropped. + # Both count lost evaluations; current emitters do not report the same loss under both names. drop_series = [ series for namespace in ("tracers", "general") diff --git a/utils/_context/_scenarios/feature_flagging.py b/utils/_context/_scenarios/feature_flagging.py index 5cb2e4ce7c7..711d7402497 100644 --- a/utils/_context/_scenarios/feature_flagging.py +++ b/utils/_context/_scenarios/feature_flagging.py @@ -1,3 +1,5 @@ +import pytest + from ._core import DdTraceEndToEndScenario, ScenarioGroup, scenario_groups # Shared by the ordinary FFE scenario, the isolated EVP stress scenario, and both @@ -80,3 +82,9 @@ def __init__(self) -> None: doc="Isolated high-cardinality EVP accounting: delivered evaluations plus reported capacity drops.", scenario_groups=[scenario_groups.ffe], ) + + def configure(self, config: pytest.Config) -> None: + super().configure(config) + # Allow an EVP flush (Go: 10 s) plus the telemetry heartbeat (2 s) before stopping. + # Preserve longer language-specific waits. + self.library_interface_timeout = max(self.library_interface_timeout, 15)