From 63a8323d38c719f70547ebf02103c55c2c36bd65 Mon Sep 17 00:00:00 2001 From: Tyler Potter Date: Fri, 23 Oct 2026 12:07:23 -0600 Subject: [PATCH 1/3] test: honor FFE evaluation case platform filters Apply canonical _skip and _include annotations before Remote Config setup and refresh the checked-in corpus snapshot from ffe-system-test-data#56. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- .../test_ffe/ffe-system-test-data/SOURCE.md | 2 +- .../test-case-disabled-flag.json | 18 + .../test-case-empty-flag.json | 18 + ...t-case-experimentation-opt-out-do-log.json | 22 ++ ...t-case-experimentation-opt-out-future.json | 146 +++++++++ .../test-case-experimentation-opt-out.json | 96 ++++-- .../test-case-split-features.json | 109 +++++++ .../ffe-system-test-data/ufc-config.json | 308 +++++++++++++++++- .../test_ffe/test_dynamic_evaluation.py | 40 +++ 9 files changed, 724 insertions(+), 35 deletions(-) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md index 7a06e3fc2be..8b1456b5fb3 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -3,7 +3,7 @@ These files are copied from the canonical FFE fixture repository. Canonical source: https://github.com/DataDog/ffe-system-test-data -Source commit: e5a64a8be81e0f435dbb5c950c06d45d4969b505 +Source commit: 5534027e0adca4bfd532b909af48ff2d4d166b69 Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in ffe-system-test-data first, then refresh this snapshot. diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json index 208e5c96943..2278ff33f64 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json @@ -1,5 +1,11 @@ [ { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], "attributes": { "country": "US", "email": "alice@mycompany.com" @@ -14,6 +20,12 @@ "variationType": "INTEGER" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], "attributes": { "country": "Canada", "email": "bob@example.com" @@ -28,6 +40,12 @@ "variationType": "INTEGER" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], "attributes": { "age": 50 }, diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json index 18ee5381fdc..f2805230ef3 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json @@ -1,5 +1,11 @@ [ { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], "attributes": { "country": "US", "email": "alice@mycompany.com" @@ -14,6 +20,12 @@ "variationType": "STRING" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], "attributes": { "country": "Canada", "email": "bob@example.com" @@ -28,6 +40,12 @@ "variationType": "STRING" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], "attributes": { "age": 50 }, diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json new file mode 100644 index 00000000000..0d2e8d5b2f5 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json @@ -0,0 +1,22 @@ +[ + { + "description": "ordinary allocation remains eligible when activation equals cutoff", + "flag": "no-log-flag", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": true, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": true, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": true, "reason": "STATIC" } + } + }, + { + "description": "ordinary allocation remains eligible when activation is after cutoff", + "flag": "no-log-flag", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T11:59:59Z" }, + "result": { "value": true, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": true, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": true, "reason": "STATIC" } + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json new file mode 100644 index 00000000000..ea33f207067 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json @@ -0,0 +1,146 @@ +[ + { + "description": "older allocation continues", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T12:00:00.000001Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "description": "equal activation is excluded", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T12:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "newer allocation is excluded", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T11:59:59.999999Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "equal instant with negative offset", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T08:00:00-04:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "equal instant with positive offset", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T14:00:00+02:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "future cutoff is not an evaluation schedule", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "description": "skip newer targeted allocation and continue older randomization", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 3, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "description": "targeted allocation remains eligible before cutoff", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-21T12:00:00Z" }, + "result": { "value": 3, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 3, "reason": "TARGETING_MATCH" } + } + }, + { + "description": "skip both allocations", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-01T12:00:00Z" }, + "result": { "value": 3, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "fractional activation equals cutoff", + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "description": "allocation is one microsecond older", + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123001Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" } + } + }, + { + "description": "allocation is one microsecond newer", + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.122999Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "description": "missing activation is not treated as older", + "flag": "kill-switch", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "alice", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": true, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": false, "reason": "DEFAULT" } + } + }, + { + "description": "null activation is not treated as older", + "flag": "numeric_flag", "variationType": "NUMERIC", "defaultValue": 0, "targetingKey": "alice", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 3.1415926, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "activation is not startAt or document createdAt", + "flag": "start-and-end-date-test", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "current", "reason": "DEFAULT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "DEFAULT" } + } + }, + { + "description": "old activation does not override endAt or startAt", + "flag": "start-and-end-date-test", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-21T12:00:00Z" }, + "result": { "value": "current", "reason": "DEFAULT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": "current", "reason": "DEFAULT" } + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json index 8120c5fb73d..69ca8ccaf01 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json @@ -4,15 +4,23 @@ "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, "result": { "value": "default", "reason": "ERROR", "errorCode": "FLAG_NOT_FOUND" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports GENERAL when the targeting key is missing." + } + ], "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": null, "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": {}, "reason": "ERROR", "errorCode": "TARGETING_KEY_MISSING" }, "capabilities": { - "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } } }, { @@ -20,7 +28,8 @@ "attributes": { "country": "UK" }, "result": { "value": "red", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "red", "reason": "TARGETING_MATCH" } + "experimentationOptOutImmediate": { "value": "red", "reason": "TARGETING_MATCH" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "red", "reason": "TARGETING_MATCH" } } }, { @@ -28,7 +37,8 @@ "attributes": { "country": "UK", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": "red", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "red", "reason": "TARGETING_MATCH" } } }, { @@ -36,7 +46,8 @@ "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": "green", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "green", "reason": "TARGETING_MATCH" } + "experimentationOptOutImmediate": { "value": "green", "reason": "TARGETING_MATCH" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "green", "reason": "TARGETING_MATCH" } } }, { @@ -44,7 +55,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } } }, { @@ -52,7 +64,8 @@ "attributes": {}, "result": { "value": 2, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 2, "reason": "SPLIT" } + "experimentationOptOutImmediate": { "value": 2, "reason": "SPLIT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 2, "reason": "SPLIT" } } }, { @@ -60,7 +73,8 @@ "attributes": {}, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 1, "reason": "SPLIT" } + "experimentationOptOutImmediate": { "value": 1, "reason": "SPLIT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -68,7 +82,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": 2, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 2, "reason": "SPLIT" } } }, { @@ -76,7 +91,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -84,7 +100,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": 3.1415926, "reason": "STATIC" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } } }, { @@ -92,7 +109,8 @@ "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": true, "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": false, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": false, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": false, "reason": "DEFAULT" } } }, { @@ -100,7 +118,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123456789Z" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -108,7 +127,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T08:00:00-04:00" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -116,7 +136,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -124,7 +145,8 @@ "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -132,7 +154,8 @@ "attributes": { "dd_experimentation_opt_out_at": "" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -140,7 +163,8 @@ "attributes": { "dd_experimentation_opt_out_at": true }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -148,7 +172,8 @@ "attributes": { "dd_experimentation_opt_out_at": null }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -156,7 +181,8 @@ "attributes": { "dd_experimentation_opt_out_at": 1792411200 }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -164,7 +190,8 @@ "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "not-a-timestamp" }, "result": { "value": "green", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -172,7 +199,8 @@ "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": false }, "result": { "value": "green", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -180,7 +208,8 @@ "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": null }, "result": { "value": "green", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -188,23 +217,38 @@ "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": {}, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": {}, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": { }, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { + "_skip": [ + { + "platform": "python", + "reason": "Python rejects object opt-out attributes before flag evaluation." + } + ], "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", "attributes": { "dd_experimentation_opt_out_at": { "timestamp": "2026-10-19T12:00:00Z" } }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { + "_skip": [ + { + "platform": "python", + "reason": "Python rejects array opt-out attributes before flag evaluation." + } + ], "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", "attributes": { "dd_experimentation_opt_out_at": ["2026-10-19T12:00:00Z"] }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json new file mode 100644 index 00000000000..05b3348f630 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json @@ -0,0 +1,109 @@ +[ + { + "flag": "split-features-destinations", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 7, + "doLog": true, + "features": { + "HOOK": { + "holdout.key": "holdout-q3", + "holdout.assignment_group": "status_quo", + "shared": true, + "empty": "" + }, + "EXPOSURE": { + "holdout.assignment_group": "status_quo", + "exposure.only": false, + "shared": true + }, + "EVALUATION": { + "evaluation.only": 1.25, + "shared": true, + "zero": 0 + } + } + } + }, + { + "flag": "split-features-first-valid-key", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 7, + "doLog": true, + "features": { + "HOOK": { + "invalid-first": "accepted", + "mixed": "kept" + }, + "EXPOSURE": { + "first-wins": "first", + "empty-destinations": false + }, + "EVALUATION": {} + } + } + }, + { + "flag": "split-features-selected-allocation", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": { + "tier": "vip" + }, + "result": { + "value": true, + "reason": "TARGETING_MATCH", + "serialId": 0, + "doLog": true, + "features": { + "HOOK": { + "selected": "vip" + }, + "EXPOSURE": { + "selected": "vip" + }, + "EVALUATION": { + "selected": "vip" + } + } + } + }, + { + "flag": "split-features-selected-allocation", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": { + "tier": "standard" + }, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 7, + "doLog": false, + "features": { + "HOOK": { + "selected": "fallback" + }, + "EXPOSURE": { + "selected": "fallback" + }, + "EVALUATION": { + "selected": "fallback" + } + } + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json index a436b889b15..677e719c1b5 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json @@ -65,7 +65,8 @@ "shards": [] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": null } ] }, @@ -1016,7 +1017,8 @@ } ], "endAt": "2002-10-31T09:00:00.594Z", - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2000-01-01T00:00:00Z" }, { "key": "future-versions", @@ -1027,7 +1029,8 @@ } ], "startAt": "2052-10-31T09:00:00.594Z", - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-01T12:00:00Z" }, { "key": "current-versions", @@ -1039,7 +1042,8 @@ ], "startAt": "2022-10-31T09:00:00.594Z", "endAt": "2050-10-31T09:00:00.594Z", - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-20T12:00:00Z" } ] }, @@ -1278,7 +1282,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-01T12:00:00Z" }, { "key": "rollout", @@ -1384,7 +1389,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-20T12:00:00Z" }, { "key": "50/50 split", @@ -1421,7 +1427,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-01T12:00:00Z" } ] }, @@ -1644,7 +1651,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-19T12:00:00.123Z" }, { "key": "allocation-default", @@ -5341,6 +5349,7 @@ "allocations": [ { "key": "no-log-allocation", + "randomizationActivatedAt": "2026-10-19T12:00:00Z", "rules": [], "splits": [ { @@ -5402,6 +5411,289 @@ "doLog": true } ] + }, + "split-features-destinations": { + "key": "split-features-destinations", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + } + }, + "allocations": [ + { + "key": "allocation", + "doLog": true, + "splits": [ + { + "variationKey": "on", + "serialId": 7, + "shards": [], + "features": [ + { + "key": "holdout.key", + "value": "holdout-q3", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK", + "EXPOSURE" + ] + }, + { + "key": "exposure.only", + "value": false, + "destinations": [ + "EXPOSURE" + ] + }, + { + "key": "evaluation.only", + "value": 1.25, + "destinations": [ + "EVALUATION" + ] + }, + { + "key": "shared", + "value": true, + "destinations": [ + "HOOK", + "EXPOSURE", + "EVALUATION", + "HOOK", + "FUTURE" + ] + }, + { + "key": "empty", + "value": "", + "destinations": [ + "HOOK" + ] + }, + { + "key": "zero", + "value": 0, + "destinations": [ + "EVALUATION" + ] + }, + { + "key": "unknown.only", + "value": true, + "destinations": [ + "FUTURE", + "hook" + ] + } + ] + } + ] + } + ] + }, + "split-features-first-valid-key": { + "key": "split-features-first-valid-key", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + } + }, + "allocations": [ + { + "key": "allocation", + "doLog": true, + "splits": [ + { + "variationKey": "on", + "serialId": 7, + "shards": [], + "features": [ + { + "key": "invalid-first", + "value": {}, + "destinations": [ + "HOOK" + ] + }, + { + "key": "invalid-first", + "value": "accepted", + "destinations": [ + "HOOK" + ] + }, + { + "key": "invalid-first", + "value": "later", + "destinations": [ + "EXPOSURE" + ] + }, + { + "key": "first-wins", + "value": "first", + "destinations": [ + "EXPOSURE" + ] + }, + { + "key": "first-wins", + "value": "later", + "destinations": [ + "HOOK", + "EVALUATION" + ] + }, + { + "key": "unknown-first", + "value": "first", + "destinations": [ + "FUTURE" + ] + }, + { + "key": "unknown-first", + "value": "later", + "destinations": [ + "HOOK" + ] + }, + { + "key": "empty-destinations", + "value": "invalid", + "destinations": [] + }, + { + "key": "empty-destinations", + "value": false, + "destinations": [ + "EXPOSURE" + ] + }, + null, + { + "key": "array", + "value": [], + "destinations": [ + "HOOK" + ] + }, + { + "key": "null", + "value": null, + "destinations": [ + "HOOK" + ] + }, + { + "key": "", + "value": true, + "destinations": [ + "HOOK" + ] + }, + { + "key": "mixed", + "value": "kept", + "destinations": [ + "HOOK", + null, + 42, + "" + ] + }, + { + "key": "bad-destinations", + "value": true, + "destinations": "HOOK" + } + ] + } + ] + } + ] + }, + "split-features-selected-allocation": { + "key": "split-features-selected-allocation", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + } + }, + "allocations": [ + { + "key": "vip", + "doLog": true, + "rules": [ + { + "conditions": [ + { + "attribute": "tier", + "operator": "ONE_OF", + "value": [ + "vip" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "on", + "serialId": 0, + "shards": [], + "features": [ + { + "key": "selected", + "value": "vip", + "destinations": [ + "HOOK", + "EXPOSURE", + "EVALUATION" + ] + } + ] + } + ] + }, + { + "key": "allocation", + "doLog": false, + "splits": [ + { + "variationKey": "on", + "serialId": 7, + "shards": [], + "features": [ + { + "key": "selected", + "value": "fallback", + "destinations": [ + "HOOK", + "EXPOSURE", + "EVALUATION" + ] + } + ] + } + ] + } + ] } } } diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 642a73d56d0..9fc8d2e92de 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -7,6 +7,7 @@ from typing import Any from utils import ( + context, features, scenarios, ) @@ -23,6 +24,19 @@ parametrize = pytest.mark.parametrize +CORPUS_PLATFORM_BY_LIBRARY = { + "cpp": "cpp", + "dotnet": "dotnet", + "golang": "golang", + "java": "java", + "nodejs": "nodejs", + "php": "php", + "python": "python", + "ruby": "ruby", + "rust": "rust", + "web": "web", +} + # Load the UFC fixture file at module level def _load_ufc_fixture() -> dict[str, Any]: @@ -41,6 +55,22 @@ def _get_test_case_files() -> list[str]: return case_files +def _filter_test_cases_for_platform(test_cases: list[dict[str, Any]], platform: str) -> list[dict[str, Any]]: + """Apply the canonical corpus _skip/_include selection rules.""" + applicable_cases = [] + for test_case in test_cases: + if "_skip" in test_case and "_include" in test_case: + raise ValueError("Evaluation cases cannot define both _skip and _include") + skipped_platforms = {entry["platform"] for entry in test_case.get("_skip", [])} + included_platforms = {entry["platform"] for entry in test_case.get("_include", [])} + if platform in skipped_platforms: + continue + if included_platforms and platform not in included_platforms: + continue + applicable_cases.append(test_case) + return applicable_cases + + # Load fixture at module level for reuse across tests UFC_FIXTURE_DATA = _load_ufc_fixture() ALL_TEST_CASE_FILES = _get_test_case_files() @@ -162,6 +192,16 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file}" assert test_cases, f"No evaluation cases in {test_case_file}" + try: + platform = CORPUS_PLATFORM_BY_LIBRARY[context.library.name] + except KeyError as error: + raise ValueError( + f"No canonical FFE corpus platform for system-tests library {context.library.name!r}" + ) from error + test_cases = _filter_test_cases_for_platform(test_cases, platform) + if not test_cases: + pytest.skip(f"No cases in {test_case_file} apply to platform {platform}") + # Set up UFC Remote Config and wait for it to be applied _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) From 2a108d4a68d84f75434926f398c0a05242c75948 Mon Sep 17 00:00:00 2001 From: Tyler Potter Date: Fri, 23 Oct 2026 12:17:33 -0600 Subject: [PATCH 2/3] refactor: use canonical system-tests platform names Use the existing system-tests library identity directly for FFE corpus case selection instead of maintaining a redundant identity mapping. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- .../test_ffe/test_dynamic_evaluation.py | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 9fc8d2e92de..e2aaf50d10b 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -24,19 +24,6 @@ parametrize = pytest.mark.parametrize -CORPUS_PLATFORM_BY_LIBRARY = { - "cpp": "cpp", - "dotnet": "dotnet", - "golang": "golang", - "java": "java", - "nodejs": "nodejs", - "php": "php", - "python": "python", - "ruby": "ruby", - "rust": "rust", - "web": "web", -} - # Load the UFC fixture file at module level def _load_ufc_fixture() -> dict[str, Any]: @@ -192,12 +179,7 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file}" assert test_cases, f"No evaluation cases in {test_case_file}" - try: - platform = CORPUS_PLATFORM_BY_LIBRARY[context.library.name] - except KeyError as error: - raise ValueError( - f"No canonical FFE corpus platform for system-tests library {context.library.name!r}" - ) from error + platform = context.library.name test_cases = _filter_test_cases_for_platform(test_cases, platform) if not test_cases: pytest.skip(f"No cases in {test_case_file} apply to platform {platform}") From ebc0b78187621f6c3eb58f5bf56a75df7dab7f6b Mon Sep 17 00:00:00 2001 From: "tyler.potter" Date: Fri, 23 Oct 2026 18:46:18 +0000 Subject: [PATCH 3/3] test(ffe): refresh corpus provenance --- tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md index 8b1456b5fb3..50d004210d7 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -3,7 +3,7 @@ These files are copied from the canonical FFE fixture repository. Canonical source: https://github.com/DataDog/ffe-system-test-data -Source commit: 5534027e0adca4bfd532b909af48ff2d4d166b69 +Source commit: caa6d1ac3e5fbda0ce57ef3ffff3772901f3f466 Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in ffe-system-test-data first, then refresh this snapshot.