Skip to content

test: honor FFE evaluation case platform skip/include filters #8159

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
leoromanovsky merged 3 commits into from
Oct 26, 2026
Merged

test: honor FFE evaluation case platform skip/include filters #8159

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
63a8323
test: honor FFE evaluation case platform filters
typotter Oct 23, 2026
2a108d4
refactor: use canonical system-tests platform names
typotter Oct 23, 2026
ebc0b78
test(ffe): refresh corpus provenance
typotter Oct 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
These files are copied from the canonical FFE fixture repository.

Canonical source: https://github.com/DataDog/ffe-system-test-data
Source commit: e5a64a8be81e0f435dbb5c950c06d45d4969b505
Source commit: caa6d1ac3e5fbda0ce57ef3ffff3772901f3f466

Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in
ffe-system-test-data first, then refresh this snapshot.
Expand Down
18 changes: 18 additions & 0 deletions tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
[
{
"_skip": [
{
"platform": "php",
"reason": "PHP reports PARSE_ERROR for disabled flags."
}
],
"attributes": {
"country": "US",
"email": "alice@mycompany.com"
Expand All @@ -14,6 +20,12 @@
"variationType": "INTEGER"
},
{
"_skip": [
{
"platform": "php",
"reason": "PHP reports PARSE_ERROR for disabled flags."
}
],
"attributes": {
"country": "Canada",
"email": "bob@example.com"
Expand All @@ -28,6 +40,12 @@
"variationType": "INTEGER"
},
{
"_skip": [
{
"platform": "php",
"reason": "PHP reports PARSE_ERROR for disabled flags."
}
],
"attributes": {
"age": 50
},
Expand Down
18 changes: 18 additions & 0 deletions tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
[
{
"_skip": [
{
"platform": "php",
"reason": "PHP reports PARSE_ERROR for empty flags."
}
],
"attributes": {
"country": "US",
"email": "alice@mycompany.com"
Expand All @@ -14,6 +20,12 @@
"variationType": "STRING"
},
{
"_skip": [
{
"platform": "php",
"reason": "PHP reports PARSE_ERROR for empty flags."
}
],
"attributes": {
"country": "Canada",
"email": "bob@example.com"
Expand All @@ -28,6 +40,12 @@
"variationType": "STRING"
},
{
"_skip": [
{
"platform": "php",
"reason": "PHP reports PARSE_ERROR for empty flags."
}
],
"attributes": {
"age": 50
},
Expand Down
22 changes: 22 additions & 0 deletions ...t_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[
{
"description": "ordinary allocation remains eligible when activation equals cutoff",
"flag": "no-log-flag", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "subject",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" },
"result": { "value": true, "reason": "STATIC" },
"capabilities": {
"experimentationOptOutImmediate": { "value": true, "reason": "STATIC" },
"experimentationOptOutFutureExperimentsOnly": { "value": true, "reason": "STATIC" }
}
},
{
"description": "ordinary allocation remains eligible when activation is after cutoff",
"flag": "no-log-flag", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "subject",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-19T11:59:59Z" },
"result": { "value": true, "reason": "STATIC" },
"capabilities": {
"experimentationOptOutImmediate": { "value": true, "reason": "STATIC" },
"experimentationOptOutFutureExperimentsOnly": { "value": true, "reason": "STATIC" }
}
}
]
146 changes: 146 additions & 0 deletions ...t_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
[
{
"description": "older allocation continues",
"flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-01T12:00:00.000001Z" },
"result": { "value": 1, "reason": "SPLIT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" }
}
},
{
"description": "equal activation is excluded",
"flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-01T12:00:00Z" },
"result": { "value": 1, "reason": "SPLIT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" }
}
},
{
"description": "newer allocation is excluded",
"flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-01T11:59:59.999999Z" },
"result": { "value": 1, "reason": "SPLIT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" }
}
},
{
"description": "equal instant with negative offset",
"flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-01T08:00:00-04:00" },
"result": { "value": 1, "reason": "SPLIT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" }
}
},
{
"description": "equal instant with positive offset",
"flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-01T14:00:00+02:00" },
"result": { "value": 1, "reason": "SPLIT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" }
}
},
{
"description": "future cutoff is not an evaluation schedule",
"flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5",
"attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" },
"result": { "value": 1, "reason": "SPLIT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" }
}
},
{
"description": "skip newer targeted allocation and continue older randomization",
"flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5",
"attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" },
"result": { "value": 3, "reason": "TARGETING_MATCH" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" }
}
},
{
"description": "targeted allocation remains eligible before cutoff",
"flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5",
"attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-21T12:00:00Z" },
"result": { "value": 3, "reason": "TARGETING_MATCH" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 3, "reason": "TARGETING_MATCH" }
}
},
{
"description": "skip both allocations",
"flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5",
"attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-01T12:00:00Z" },
"result": { "value": 3, "reason": "TARGETING_MATCH" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" }
}
},
{
"description": "fractional activation equals cutoff",
"flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123Z" },
"result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" }
}
},
{
"description": "allocation is one microsecond older",
"flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123001Z" },
"result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }
}
},
{
"description": "allocation is one microsecond newer",
"flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.122999Z" },
"result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" }
}
},
{
"description": "missing activation is not treated as older",
"flag": "kill-switch", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "alice",
"attributes": { "country": "US", "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" },
"result": { "value": true, "reason": "TARGETING_MATCH" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": false, "reason": "DEFAULT" }
}
},
{
"description": "null activation is not treated as older",
"flag": "numeric_flag", "variationType": "NUMERIC", "defaultValue": 0, "targetingKey": "alice",
"attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" },
"result": { "value": 3.1415926, "reason": "STATIC" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" }
}
},
{
"description": "activation is not startAt or document createdAt",
"flag": "start-and-end-date-test", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" },
"result": { "value": "current", "reason": "DEFAULT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "DEFAULT" }
}
},
{
"description": "old activation does not override endAt or startAt",
"flag": "start-and-end-date-test", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject",
"attributes": { "dd_experimentation_opt_out_at": "2026-10-21T12:00:00Z" },
"result": { "value": "current", "reason": "DEFAULT" },
"capabilities": {
"experimentationOptOutFutureExperimentsOnly": { "value": "current", "reason": "DEFAULT" }
}
}
]
Loading
Loading