From e40fc66e9cb6764ba008583937d682cc024c6c85 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 21 Oct 2026 16:56:20 +0000 Subject: [PATCH 1/7] test(ffe): consume canonical fixture snapshot Replace local fixture copies with the shared corpus, preserve extracted metadata and span inputs, and add a reviewed snapshot update workflow. Environment: Datadog workspace --- .github/CODEOWNERS | 4 + ...elf.update-ffe-fixtures.create-pr.sts.yaml | 14 + .github/workflows/update-ffe-fixtures.yml | 87 + tests/ffe/README.md | 31 +- tests/ffe/test_flag_eval_evp.py | 2 +- .../test_ffe/ffe-system-test-data/SOURCE.md | 12 + .../test-case-boolean-false-assignment.json | 35 +- .../test-case-boolean-one-of-matches.json | 176 +- .../test-case-comparator-operator-flag.json | 63 +- .../test-case-disabled-flag.json | 41 +- .../test-case-empty-flag.json | 41 +- .../test-case-empty-string-variation.json | 35 +- .../test-case-experimentation-opt-out.json | 210 ++ .../test-case-falsy-value-assignments.json | 35 +- .../test-case-flag-with-empty-string.json | 22 +- .../test-case-integer-flag.json | 265 ++- .../test-case-invalid-condition-operands.json | 59 + .../test-case-invalid-regex-isolation.json | 17 + ...t-case-invalid-shard-bounds-isolation.json | 40 + .../test-case-kill-switch-flag.json | 292 +-- .../test-case-malformed-flag-isolation.json | 27 + .../test-case-microsecond-date-flag.json | 39 + ...t-case-missing-split-shards-isolation.json | 27 + .../test-case-new-user-onboarding-flag.json | 316 +-- .../test-case-no-allocations-flag.json | 49 +- .../test-case-null-operator-flag.json | 63 +- .../test-case-null-shard-range-isolation.json | 15 + .../test-case-null-targeting-key.json | 40 + .../test-case-numeric-flag.json | 41 +- .../test-case-numeric-one-of-default.json | 13 + .../test-case-numeric-one-of.json | 77 +- .../test-case-of-7-empty-targeting-key.json | 11 +- .../test-case-regex-flag.json | 122 + .../test-case-semver-comparison-flag.json | 169 ++ ...e-semver-invalid-comparand-categories.json | 120 + ...se-semver-invalid-comparand-waterfall.json | 17 + .../test-case-semver-precedence-flag.json | 170 ++ .../test-case-semver-validation-flag.json | 255 ++ .../test-case-semver-version-parts.json | 118 + .../test-case-serial-id-metadata.json | 97 + .../test-case-span-enrichment-inputs.json | 67 + .../test-case-start-and-end-date-flag.json | 41 +- .../test-case-unknown-fields-tolerance.json | 16 + .../test-case-unknown-operator-isolation.json | 31 + .../test-case-variant-type-mismatch.json | 27 + .../test-flag-that-does-not-exist.json | 46 + .../test-json-config-flag.json | 76 +- .../test-no-allocations-flag.json | 49 +- .../test-special-characters.json | 44 +- .../test-string-with-special-characters.json | 732 +++--- .../ufc-config.json} | 2113 ++++++++++++++++- .../test_ffe/serial-id-metadata-flags.json | 97 - .../test_ffe/span-enrichment-flags.json | 146 -- .../test_ffe/test-case-regex-flag.json | 53 - .../test-flag-that-does-not-exist.json | 40 - .../test_ffe/test_configuration_sources.py | 2 +- .../test_ffe/test_dynamic_evaluation.py | 44 +- .../test_ffe/test_serial_id_metadata.py | 15 +- .../test_ffe/test_span_enrichment.py | 21 +- tests/parametric/test_ffe/utils.py | 14 + .../test_the_test/test_update_ffe_fixtures.py | 145 ++ .../_test_clients/_test_client_parametric.py | 4 +- utils/mocked_backend/ffe.py | 2 +- utils/scripts/update_ffe_fixtures.py | 206 ++ 64 files changed, 5761 insertions(+), 1537 deletions(-) create mode 100644 .github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml create mode 100644 .github/workflows/update-ffe-fixtures.yml create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-boolean-false-assignment.json (74%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-boolean-one-of-matches.json (72%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-comparator-operator-flag.json (68%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-disabled-flag.json (59%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-empty-flag.json (63%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-empty-string-variation.json (75%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-falsy-value-assignments.json (73%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-flag-with-empty-string.json (72%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-integer-flag.json (66%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-kill-switch-flag.json (67%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-new-user-onboarding-flag.json (69%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-no-allocations-flag.json (68%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-null-operator-flag.json (67%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-numeric-flag.json (52%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-numeric-one-of.json (70%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-of-7-empty-targeting-key.json (73%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-start-and-end-date-flag.json (65%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-json-config-flag.json (62%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-no-allocations-flag.json (70%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-special-characters.json (77%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-string-with-special-characters.json (77%) rename tests/parametric/test_ffe/{flags-v1.json => ffe-system-test-data/ufc-config.json} (60%) delete mode 100644 tests/parametric/test_ffe/serial-id-metadata-flags.json delete mode 100644 tests/parametric/test_ffe/span-enrichment-flags.json delete mode 100644 tests/parametric/test_ffe/test-case-regex-flag.json delete mode 100644 tests/parametric/test_ffe/test-flag-that-does-not-exist.json create mode 100644 tests/test_the_test/test_update_ffe_fixtures.py create mode 100644 utils/scripts/update_ffe_fixtures.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 277408f8d82..641b874b47c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,6 +12,9 @@ /tests/ @DataDog/system-tests-reviewers # FFE +/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core +/.github/workflows/update-ffe-fixtures.yml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core +/utils/scripts/update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/docker_fixtures/_mock_ffe_agentless_backend.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/build/docker/dotnet/parametric/Endpoints/FfeTestApi.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core @@ -23,6 +26,7 @@ /utils/build/docker/php/common/ffe.php @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /tests/ffe/ @DataDog/feature-flagging-and-experimentation-sdk /tests/test_the_test/test_mock_ffe_agentless_backend.py @DataDog/feature-flagging-and-experimentation-sdk +/tests/test_the_test/test_update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk /tests/parametric/test_ffe/ @DataDog/feature-flagging-and-experimentation-sdk # SDK cap diff --git a/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml b/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml new file mode 100644 index 00000000000..7aeba37ded2 --- /dev/null +++ b/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml @@ -0,0 +1,14 @@ +--- +issuer: https://token.actions.githubusercontent.com + +subject: repo:DataDog/system-tests:ref:refs/heads/main + +claim_pattern: + event_name: (workflow_dispatch|schedule) + ref: refs/heads/main + ref_protected: "true" + job_workflow_ref: DataDog/system-tests/\.github/workflows/update-ffe-fixtures\.yml@refs/heads/main + +permissions: + contents: write + pull_requests: write diff --git a/.github/workflows/update-ffe-fixtures.yml b/.github/workflows/update-ffe-fixtures.yml new file mode 100644 index 00000000000..778db406a4b --- /dev/null +++ b/.github/workflows/update-ffe-fixtures.yml @@ -0,0 +1,87 @@ +--- +name: Update FFE fixtures + +on: + workflow_dispatch: + inputs: + fixture_ref: + description: Branch, tag, or commit from DataDog/ffe-system-test-data + required: true + default: main + type: string + schedule: + - cron: "0 0 * * 0" + +concurrency: + group: update-ffe-fixtures + cancel-in-progress: false + +jobs: + update-ffe-fixtures: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + persist-credentials: false + + - name: Update fixture snapshot + id: fixtures + env: + FFE_FIXTURE_REF: ${{ inputs.fixture_ref || 'main' }} + run: python3 utils/scripts/update_ffe_fixtures.py --ref "$FFE_FIXTURE_REF" + + - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4 + if: steps.fixtures.outputs.changed == 'true' + id: octo-sts + with: + scope: DataDog/system-tests + policy: self.update-ffe-fixtures.create-pr + + - name: Check for an open fixture update PR + if: steps.fixtures.outputs.changed == 'true' + id: existing-pr + env: + GH_TOKEN: ${{ steps.octo-sts.outputs.token }} + run: | + count=$(gh pr list --repo "$GITHUB_REPOSITORY" \ + --head actions/update-ffe-fixtures --state open \ + --json headRefName,headRepositoryOwner,isCrossRepository \ + --jq '[.[] | select( + .headRefName == "actions/update-ffe-fixtures" and + .headRepositoryOwner.login == "DataDog" and + (.isCrossRepository | not) + )] | length') + echo "count=$count" >> "$GITHUB_OUTPUT" + + - name: Create Pull Request + if: >- + steps.fixtures.outputs.changed == 'true' && steps.existing-pr.outputs.count == '0' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.octo-sts.outputs.token }} + branch: actions/update-ffe-fixtures + base: main + commit-message: "test(ffe): update canonical fixtures" + title: "test(ffe): update canonical fixtures" + draft: true + sign-commits: true + add-paths: tests/parametric/test_ffe/ffe-system-test-data + body: | + ## Motivation + + Keep SDK behavior consistent with the shared FFE evaluator contract. + + ## Changes + + Refresh the checked-in snapshot to [${{ steps.fixtures.outputs.source_commit }}](https://github.com/DataDog/ffe-system-test-data/commit/${{ steps.fixtures.outputs.source_commit }}), containing ${{ steps.fixtures.outputs.fixture_count }} evaluation cases. + + Ordinary test runs use the snapshot without network access. Resolve newly exposed evaluator failures before merging; correct shared expectations upstream only when they are wrong. Scheduled updates leave an existing update PR untouched to preserve review and fixes. + + ## Workflow + + 1. Keep this PR as a draft while validating the updated fixtures against the supported SDKs. + 2. Follow the repository's [review workflow](https://github.com/DataDog/system-tests/blob/main/.github/pull_request_template.md). diff --git a/tests/ffe/README.md b/tests/ffe/README.md index be43490db3c..f1205b765d5 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -12,6 +12,35 @@ This directory contains system tests for the Feature Flags & Experimentation (FF | `test_flag_eval_metrics.py` | Evaluation metrics (OTel counter) | | `test_flag_eval_evp.py` | Server-side EVP flagevaluation payloads, aggregation, and bounds | +## Canonical parametric fixtures + +The parametric evaluation suite reads the checked-in snapshot under +`tests/parametric/test_ffe/ffe-system-test-data`. Its upstream source and exact +commit are recorded in `SOURCE.md` inside that directory. + +Change shared evaluator cases in +[`DataDog/ffe-system-test-data`](https://github.com/DataDog/ffe-system-test-data) +first, then refresh the snapshot: + +```bash +python3 utils/scripts/update_ffe_fixtures.py --ref +``` + +The updater imports `ufc-config.json` and `evaluation-cases/*.json` only. Ordinary +test runs use those checked-in files without submodules or network access. A +weekly/manual workflow opens a signed draft PR when their contents change and +leaves any existing fixture update PR untouched. Review and resolve newly exposed +SDK failures before merging an update. + +Dynamic evaluation checks each case's value, reason, and explicit error code. +Cases with optional `capabilities` still run their baseline `result`; enabling +capability-specific provider options requires separate harness support. +Remote Configuration tests and the mocked agentless backend read the same UFC. +Serial-ID metadata and span-enrichment tests select their flag definitions from +the shared corpus using their dedicated evaluation-case files. Their integration +harnesses retain trace construction and metadata assertions; no separate local +configuration copies are needed. + ## Running FFE Tests ```bash @@ -197,7 +226,7 @@ The tests use `interfaces.agent.get_metrics()` to retrieve metrics from the agen ### 1. Test returns `parse_error` instead of expected reason - Check UFC fixture format (especially `totalShards` placement) -- Verify the fixture matches `flags-v1.json` format in dd-trace-py +- Verify the fixture matches the canonical `ffe-system-test-data/ufc-config.json` format ### 2. Test returns `static` instead of `split` - Need multiple variations with different shard ranges diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index bcf93e46a17..5e53cb1b8a8 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -242,7 +242,7 @@ def assert_no_duplicate_visible_events(events: list[tuple[JSON, JSON]]) -> None: class FlagevaluationEgressContract: """One flag-evaluation contract inherited by each supported topology adapter.""" - # Agentless scenarios preload flags-v1.json before the weblog starts. Use the + # Agentless scenarios preload the canonical UFC before the weblog starts. Use the # same known fixture key in every topology; the Agent scenario installs an # equivalent fixture through Remote Config below. flag_key = "empty-targeting-key-flag" diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md new file mode 100644 index 00000000000..7a06e3fc2be --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -0,0 +1,12 @@ +# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: e5a64a8be81e0f435dbb5c950c06d45d4969b505 + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in +ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed +draft test PR only when the server evaluator fixture contents change. diff --git a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json similarity index 74% rename from tests/parametric/test_ffe/test-case-boolean-false-assignment.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json index cdd2b0a513d..ab718f5bee1 100644 --- a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json @@ -1,38 +1,41 @@ [ { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "alice", "attributes": { "should_disable_feature": true }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": false - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "bob", "attributes": { "should_disable_feature": false }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "charlie", "attributes": { "unknown_attribute": "value" }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "DEFAULT", "value": true - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json similarity index 72% rename from tests/parametric/test_ffe/test-case-boolean-one-of-matches.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json index f616614d936..392ec97ec0f 100644 --- a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json @@ -1,192 +1,208 @@ [ { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "one_of_flag": "True" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "not_matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "not_matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "haley", "attributes": { "not_matches_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "haley", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "ivy", "attributes": { "not_one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "ivy", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "julia", "attributes": { "not_one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "julia", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "kim", "attributes": { "not_one_of_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "kim", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "lucas", "attributes": { "not_one_of_flag": "true" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "lucas", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "mike", "attributes": { "not_one_of_flag": "false" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "mike", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "nicole", "attributes": { "null_flag": "null" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 5 - } + }, + "targetingKey": "nicole", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "owen", "attributes": { "null_flag": null }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "owen", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "pete", "attributes": {}, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "pete", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-comparator-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json index 2d94f30eb30..e4daa2de7c2 100644 --- a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "small" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "medium" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "size": 25 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "large" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-disabled-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json similarity index 59% rename from tests/parametric/test_ffe/test-case-disabled-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json index 0da79189ade..208e5c96943 100644 --- a/tests/parametric/test_ffe/test-case-disabled-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-empty-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json similarity index 63% rename from tests/parametric/test_ffe/test-case-empty-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json index 52100b1fe47..18ee5381fdc 100644 --- a/tests/parametric/test_ffe/test-case-empty-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-empty-string-variation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json similarity index 75% rename from tests/parametric/test_ffe/test-case-empty-string-variation.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json index 1af431f3db4..f1fa3994af2 100644 --- a/tests/parametric/test_ffe/test-case-empty-string-variation.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json @@ -1,38 +1,41 @@ [ { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "empty_user", "attributes": { "content_type": "minimal" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "empty_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "full_user", "attributes": { "content_type": "full" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "detailed_content" - } + }, + "targetingKey": "full_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "default_user", "attributes": { "content_type": "unknown" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "default_user", + "variationType": "STRING" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json new file mode 100644 index 00000000000..8120c5fb73d --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json @@ -0,0 +1,210 @@ +[ + { + "flag": "unknown-opt-out-flag", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": "default", "reason": "ERROR", "errorCode": "FLAG_NOT_FOUND" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": null, + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": {}, "reason": "ERROR", "errorCode": "TARGETING_KEY_MISSING" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "19", + "attributes": { "country": "UK" }, + "result": { "value": "red", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "red", "reason": "TARGETING_MATCH" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "19", + "attributes": { "country": "UK", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "red", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "DEFAULT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "green", "reason": "TARGETING_MATCH" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "1", + "attributes": {}, + "result": { "value": 2, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 2, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": {}, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 1, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "1", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 2, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "numeric_flag", "variationType": "NUMERIC", "defaultValue": 0, "targetingKey": "alice", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 3.1415926, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "kill-switch", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "alice", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": true, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": false, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123456789Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T08:00:00-04:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": true }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": null }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": 1792411200 }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": false }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": null }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": {}, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": { "timestamp": "2026-10-19T12:00:00Z" } }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": ["2026-10-19T12:00:00Z"] }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + } +] diff --git a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json similarity index 73% rename from tests/parametric/test_ffe/test-case-falsy-value-assignments.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json index 4ad259620d8..6a8cee51bcc 100644 --- a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json @@ -1,38 +1,41 @@ [ { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "zero_user", "attributes": { "plan_tier": "free" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 0 - } + }, + "targetingKey": "zero_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "premium_user", "attributes": { "plan_tier": "premium" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 100 - } + }, + "targetingKey": "premium_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "unknown_user", "attributes": { "plan_tier": "trial" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "DEFAULT", "value": 999 - } + }, + "targetingKey": "unknown_user", + "variationType": "INTEGER" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json similarity index 72% rename from tests/parametric/test_ffe/test-case-flag-with-empty-string.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json index 0bf562efa35..6a5d9cd62fd 100644 --- a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json @@ -1,24 +1,26 @@ [ { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": {}, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "SPLIT", "value": "non_empty" - } + }, + "targetingKey": "bob", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-integer-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json similarity index 66% rename from tests/parametric/test_ffe/test-case-integer-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json index 5e930885b44..8ff11aec1fc 100644 --- a/tests/parametric/test_ffe/test-case-integer-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json @@ -1,244 +1,267 @@ [ { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "debra", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "1", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "1", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "2", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "2", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "3", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "3", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "4", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "4", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "5", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "5", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "6", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "6", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "7", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "7", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "8", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "8", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "9", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "9", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "10", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "10", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "11", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "11", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "12", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "12", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "13", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "13", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "14", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "14", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "15", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "15", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "16", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "16", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "17", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "17", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "18", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "18", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "19", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "19", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json new file mode 100644 index 00000000000..be226381455 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json @@ -0,0 +1,59 @@ +[ + { + "description": "GT requires a numeric configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "age": 42 + }, + "defaultValue": "default", + "flag": "invalid-gt-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-gt", + "variationType": "STRING" + }, + { + "description": "ONE_OF requires an array configured operand. A flag with a scalar operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "role": "admin" + }, + "defaultValue": "default", + "flag": "invalid-one-of-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-one-of", + "variationType": "STRING" + }, + { + "description": "IS_NULL requires a boolean configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "deleted_at": null + }, + "defaultValue": "default", + "flag": "invalid-is-null-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-is-null", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same configuration must still evaluate after malformed condition operands are isolated.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json new file mode 100644 index 00000000000..3ad596cbf06 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json @@ -0,0 +1,17 @@ +[ + { + "description": "The malformed flag uses the invalid regex *@datadoghq.com before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": { + "email": "harish@datadoghq.com" + }, + "defaultValue": "default", + "flag": "invalid-regex-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "harish", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json new file mode 100644 index 00000000000..ca8553f099c --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json @@ -0,0 +1,40 @@ +[ + { + "description": "The malformed flag has totalShards above the unsigned 32-bit range. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "overflow-total-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "The malformed flag has a negative shard range start. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "negative-shard-range-start-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after invalid shard-bound flags are skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-kill-switch-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-kill-switch-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json index 8f34a1bc3af..3af700dc711 100644 --- a/tests/parametric/test_ffe/test-case-kill-switch-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json @@ -1,290 +1,314 @@ [ { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "barbara", "attributes": { - "email": "barbara@example.com", - "country": "canada" + "country": "canada", + "email": "barbara@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "barbara", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "charlie", "attributes": { "age": 40 }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "debra", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "1", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "1", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "2", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 50 + "age": 50, + "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "3", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "4", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "5", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "6", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "7", "attributes": { - "country": "US", - "age": 12 + "age": 12, + "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "7", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "8", "attributes": { - "country": "Italy", - "age": 60 + "age": 60, + "country": "Italy" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "8", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "9", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "10", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "10", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "11", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "11", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "12", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "13", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "14", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "14", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "15", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "16", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "17", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "18", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "19", + "variationType": "BOOLEAN" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json new file mode 100644 index 00000000000..41672357c23 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has allocations encoded as a string instead of an array. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "malformed-allocations-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the malformed flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json new file mode 100644 index 00000000000..3dba337ad02 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json @@ -0,0 +1,39 @@ +[ + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "country": "US" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "1.0.0" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "charlie", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json new file mode 100644 index 00000000000..d6dfeb180ea --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has a split that omits required shards. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "missing-split-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the missing-shards flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json similarity index 69% rename from tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json index 8ed3e2a024f..bb08523de4e 100644 --- a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json @@ -1,318 +1,344 @@ [ { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "green" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "debra", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "purple" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "id": "override-id", - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com", + "id": "override-id" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "Zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "Zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "1", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "1", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "2", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 33 + "age": 33, + "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "3", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "4", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "5", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "6", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "7", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "7", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "8", "attributes": { "country": "Italy" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "8", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "9", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "10", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "10", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "11", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "11", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "12", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "13", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "14", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "14", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "15", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "16", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "17", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "18", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "19", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json index 132c39db32a..7d985b8efd4 100644 --- a/tests/parametric/test_ffe/test-case-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "hello": "world" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "hello": "world" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-case-null-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-null-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json index 09e5d78dacd..8063ff8fee3 100644 --- a/tests/parametric/test_ffe/test-case-null-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "charlie", "attributes": { "size": null }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json new file mode 100644 index 00000000000..15f4af5a77d --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json @@ -0,0 +1,15 @@ +[ + { + "description": "The malformed flag has a null shard range before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "null-shard-range-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json new file mode 100644 index 00000000000..a2226d1b21e --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json @@ -0,0 +1,40 @@ +[ + { + "attributes": {}, + "defaultValue": 42, + "flag": "numeric_flag", + "result": { + "reason": "STATIC", + "value": 3.1415926 + }, + "targetingKey": null, + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "France" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "ERROR", + "value": "default" + }, + "targetingKey": null, + "variationType": "STRING" + }, + { + "attributes": { + "country": "France", + "email": "null@mycompany.com" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "TARGETING_MATCH", + "value": "green" + }, + "targetingKey": null, + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json similarity index 52% rename from tests/parametric/test_ffe/test-case-numeric-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json index 757f0f70e55..cc52dcdff4d 100644 --- a/tests/parametric/test_ffe/test-case-numeric-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "alice", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "bob", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json new file mode 100644 index 00000000000..d8a7e8ba092 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json @@ -0,0 +1,13 @@ +[ + { + "attributes": {}, + "defaultValue": 0, + "flag": "numeric-one-of", + "result": { + "reason": "DEFAULT", + "value": 0 + }, + "targetingKey": "race-default", + "variationType": "INTEGER" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-one-of.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json similarity index 70% rename from tests/parametric/test_ffe/test-case-numeric-one-of.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json index 9eaccbc477c..bfa957335d3 100644 --- a/tests/parametric/test_ffe/test-case-numeric-one-of.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json @@ -1,86 +1,93 @@ [ { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "number": 2 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "number": 3 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "number": 4 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "number": "1" }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "number": 123456789 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json similarity index 73% rename from tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json index 03f03ab1ecb..e99a207b6da 100644 --- a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json @@ -1,12 +1,13 @@ [ { - "flag": "empty-targeting-key-flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "", "attributes": {}, + "defaultValue": "default", + "flag": "empty-targeting-key-flag", "result": { + "reason": "STATIC", "value": "on-value" - } + }, + "targetingKey": "", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json new file mode 100644 index 00000000000..b01f6f1e5ca --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json @@ -0,0 +1,122 @@ +[ + { + "attributes": { + "comma_regex_case": "xxaaay" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "comma-quantifier" + }, + "targetingKey": "comma-quantifier", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "prefix exact suffix" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "literal-whitespace" + }, + "targetingKey": "literal-whitespace", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "exact" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "trimmed-whitespace-does-not-match", + "variationType": "STRING" + }, + { + "attributes": { + "email": "user.name+tag@capture.example" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "capturing-groups" + }, + "targetingKey": "capture", + "variationType": "STRING" + }, + { + "attributes": { + "email": "llega mañana temprano" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "unicode-word-boundary" + }, + "targetingKey": "unicode", + "variationType": "STRING" + }, + { + "attributes": { + "email": "alice@example.com", + "version": "1.15.0" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "partial-example" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "email": "bob@test.com", + "version": "0.20.1" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "test" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "charlie", + "variationType": "STRING" + }, + { + "attributes": { + "email": "derek@gmail.com", + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "derek", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json new file mode 100644 index 00000000000..bc6fe8b92bc --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json @@ -0,0 +1,169 @@ +[ + { + "attributes": { + "app_version": "1.2.3-alpha.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "1.2.3-alpha.5" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.3.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "carol", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.9.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "dave", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "eve", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.2" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "frank", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.0" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "grace", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "henry", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "mia", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "future" + }, + "targetingKey": "iris", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "5.6.7" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "jane", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "not-a-semver" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "kyle", + "variationType": "STRING" + }, + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "liam", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json new file mode 100644 index 00000000000..4f00961d9f9 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json @@ -0,0 +1,120 @@ +[ + { + "description": "A non-parseable SemVer comparand (\"not-a-version\") rejects the flag at parse time with a PARSE_ERROR rather than matching or coercing.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-syntax-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-syntax", + "variationType": "STRING" + }, + { + "description": "A SemVer comparand with repeated hyphens inside its prerelease identifiers (\"1.2.3----R-S.12.9.1--.12+meta\") is valid and matches normally.", + "attributes": { + "app_version": "1.2.3----R-S.12.9.1--.12+meta" + }, + "defaultValue": "unknown", + "flag": "semver-valid-comparand-hyphenated-prerelease-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "matched" + }, + "targetingKey": "valid-hyphenated-prerelease", + "variationType": "STRING" + }, + { + "description": "A SemVer comparand with consecutive dots in its prerelease identifiers (\"1.0.0-alpha..1\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.0.0-alpha.1" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-consecutive-dots-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-consecutive-dots", + "variationType": "STRING" + }, + { + "description": "A Ruby-style dot-suffixed SemVer comparand (\"1.2.3.DEV\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-dot-suffix-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-dot-suffix", + "variationType": "STRING" + }, + { + "description": "A v-prefixed SemVer comparand (\"v1.2.3\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-vprefix-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-vprefix", + "variationType": "STRING" + }, + { + "description": "A leading-zero core component in the SemVer comparand (\"01.2.3\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-leading-zero-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-leading-zero", + "variationType": "STRING" + }, + { + "description": "A non-string (JSON number) SemVer comparand rejects the flag at parse time with a PARSE_ERROR rather than coercing the number to a string.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-non-string-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-non-string", + "variationType": "STRING" + }, + { + "description": "The rejected invalid-comparand flags must not poison the rest of the configuration: a neighboring valid SemVer flag still evaluates normally.", + "attributes": { + "app_version": "2.3.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "after-invalid-categories", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json new file mode 100644 index 00000000000..cbddbb87d03 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json @@ -0,0 +1,17 @@ +[ + { + "description": "An invalid configured SemVer comparand in a non-last allocation must reject the whole flag at parse time, even though a valid default allocation follows it in the waterfall. An implementation that skips the invalid allocation and falls through to the default would return \"fallback\" with a STATIC reason instead of an error.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-waterfall-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "waterfall", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json new file mode 100644 index 00000000000..ef027a06660 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json @@ -0,0 +1,170 @@ +[ + { + "description": "Multi-digit numeric prerelease identifiers compare by numeric value, not lexicographically: 1.0.0-beta.2 must precede 1.0.0-beta.11. An implementation that string-compares identifiers would treat \"2\" as greater than \"11\" and fail to match.", + "attributes": { + "app_version": "10.0.0-beta.2" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "numeric-prerelease" + }, + "targetingKey": "numeric-prerelease", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for equality: 1.0.0+linux and 1.0.0+darwin have equal precedence, so SEMVER_EQ matches.", + "attributes": { + "app_version": "20.0.0+linux" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-equal-precedence" + }, + "targetingKey": "build-equal-precedence", + "variationType": "STRING" + }, + { + "description": "Dotted/multi-segment build metadata is ignored for equality: 4.0.0+exp.sha.5114f85 equals 4.0.0.", + "attributes": { + "app_version": "30.0.0+exp.sha.5114f85" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-dotted-eq" + }, + "targetingKey": "build-dotted-eq", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_LTE: 4.0.0+build.42 <= 4.0.0 is true (match).", + "attributes": { + "app_version": "40.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-lte" + }, + "targetingKey": "build-lte", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_GTE: 4.0.0+build.42 >= 4.0.0 is true (match).", + "attributes": { + "app_version": "50.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-gte" + }, + "targetingKey": "build-gte", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_NEQ: 4.0.0+build.42 != 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "60.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-neq", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_LT: 4.0.0+build.42 < 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "70.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-lt", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_GT: 4.0.0+build.42 > 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "80.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-gt", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: 1.0.0-alpha precedes 1.0.0-alpha.1 (fewer identifiers first), so SEMVER_LT matches.", + "attributes": { + "app_version": "90.0.0-alpha" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-before-alpha-1" + }, + "targetingKey": "alpha-before-alpha-1", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: numeric identifiers have lower precedence than alphanumeric (1.0.0-alpha.1 precedes 1.0.0-alpha.beta), so SEMVER_LT matches.", + "attributes": { + "app_version": "91.0.0-alpha.1" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-1-before-alpha-beta" + }, + "targetingKey": "alpha-1-before-alpha-beta", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: identifiers are compared lexicographically (1.0.0-alpha.beta precedes 1.0.0-beta), so SEMVER_LT matches.", + "attributes": { + "app_version": "92.0.0-alpha.beta" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-beta-before-beta" + }, + "targetingKey": "alpha-beta-before-beta", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: a prerelease version precedes the corresponding release (1.0.0-beta precedes 1.0.0), so SEMVER_LT matches.", + "attributes": { + "app_version": "93.0.0-beta" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta-before-release" + }, + "targetingKey": "beta-before-release", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json new file mode 100644 index 00000000000..9b4bfea3710 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json @@ -0,0 +1,255 @@ +[ + { + "description": "The maximum safe integer value is valid in every core version component.", + "attributes": { + "app_version": "9007199254740991.9007199254740991.9007199254740991" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "maximum" + }, + "targetingKey": "max-core", + "variationType": "STRING" + }, + { + "description": "Semver does not accept a v prefix.", + "attributes": { + "app_version": "v1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "v-prefix", + "variationType": "STRING" + }, + { + "description": "Core numeric components cannot have leading zeros.", + "attributes": { + "app_version": "01.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "core-leading-zero", + "variationType": "STRING" + }, + { + "description": "Numeric prerelease identifiers cannot have leading zeros.", + "attributes": { + "app_version": "1.2.3-01" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "prerelease-leading-zero", + "variationType": "STRING" + }, + { + "description": "Prerelease identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3-alpha..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-prerelease-identifier", + "variationType": "STRING" + }, + { + "description": "Build metadata identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3+build..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-build-identifier", + "variationType": "STRING" + }, + { + "description": "Prerelease and build identifiers are ASCII-only.", + "attributes": { + "app_version": "1.2.3-α" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "non-ascii-prerelease", + "variationType": "STRING" + }, + { + "description": "An empty string is not a valid SemVer version.", + "attributes": { + "app_version": "" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-string", + "variationType": "STRING" + }, + { + "description": "A leading zero in the minor core component is rejected.", + "attributes": { + "app_version": "1.02.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "minor-leading-zero", + "variationType": "STRING" + }, + { + "description": "A leading zero in the patch core component is rejected.", + "attributes": { + "app_version": "1.2.03" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "patch-leading-zero", + "variationType": "STRING" + }, + { + "description": "A trailing prerelease delimiter with no identifiers is rejected.", + "attributes": { + "app_version": "1.2.3-" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-prerelease", + "variationType": "STRING" + }, + { + "description": "A trailing build delimiter with no identifiers is rejected.", + "attributes": { + "app_version": "1.2.3+" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-build", + "variationType": "STRING" + }, + { + "description": "Underscores are not permitted in prerelease identifiers.", + "attributes": { + "app_version": "1.2.3-alpha_1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "underscore-prerelease", + "variationType": "STRING" + }, + { + "description": "More than one build metadata delimiter is rejected.", + "attributes": { + "app_version": "1.2.3-alpha+build+other" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "multiple-build-delimiters", + "variationType": "STRING" + }, + { + "description": "Leading whitespace is not accepted by the SemVer parser.", + "attributes": { + "app_version": " 1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "leading-whitespace", + "variationType": "STRING" + }, + { + "description": "Trailing whitespace is not accepted by the SemVer parser.", + "attributes": { + "app_version": "1.2.3 " + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "trailing-whitespace", + "variationType": "STRING" + }, + { + "description": "A non-string attribute (JSON number) does not match any SemVer condition and falls through to the default rather than erroring.", + "attributes": { + "app_version": 1.2 + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "non-string-attribute", + "variationType": "STRING" + }, + { + "description": "A flag with an invalid configured SemVer comparand returns a parse error while the rest of the configuration remains usable.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-comparand", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json new file mode 100644 index 00000000000..bac7fdade7b --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json @@ -0,0 +1,118 @@ +[ + { + "description": "A one-part version is normalized with missing minor and patch parts: \"18\" compares equal to \"18.0.0\".", + "attributes": { + "scenario": "one-part-input", + "app_version": "18" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "one-part-input" + }, + "targetingKey": "one-part-input", + "variationType": "STRING" + }, + { + "description": "A one-part configured comparand is valid and normalized: \"18.0.0\" compares equal to configured \"18\".", + "attributes": { + "app_version": "18.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-one-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "one-part-comparand" + }, + "targetingKey": "one-part-comparand", + "variationType": "STRING" + }, + { + "description": "A two-part version is normalized with a missing patch part: \"18.0\" compares equal to \"18.0.0\".", + "attributes": { + "scenario": "two-part-input", + "app_version": "18.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "two-part-input" + }, + "targetingKey": "two-part-input", + "variationType": "STRING" + }, + { + "description": "A two-part configured comparand is valid and normalized: \"18.0.0\" compares equal to configured \"18.0\".", + "attributes": { + "app_version": "18.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-two-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "two-part-comparand" + }, + "targetingKey": "two-part-comparand", + "variationType": "STRING" + }, + { + "description": "A four-part version is accepted as an attribute and compares above a canonical three-part lower version.", + "attributes": { + "scenario": "four-part-input", + "app_version": "18.0.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "four-part-input" + }, + "targetingKey": "four-part-input", + "variationType": "STRING" + }, + { + "description": "A four-part configured comparand is accepted and compares below a canonical three-part higher version.", + "attributes": { + "app_version": "19.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-four-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "four-part-comparand" + }, + "targetingKey": "four-part-comparand", + "variationType": "STRING" + }, + { + "description": "A five-part version is accepted as an attribute and compares above a canonical three-part lower version.", + "attributes": { + "scenario": "five-part-input", + "app_version": "18.0.0.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "five-part-input" + }, + "targetingKey": "five-part-input", + "variationType": "STRING" + }, + { + "description": "A five-part configured comparand is accepted and compares below a canonical three-part higher version.", + "attributes": { + "app_version": "19.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-five-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "five-part-comparand" + }, + "targetingKey": "five-part-comparand", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json new file mode 100644 index 00000000000..8bfadabb0e7 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json @@ -0,0 +1,97 @@ +[ + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + }, + { + "flag": "serial-id-zero-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 0, + "doLog": true + } + }, + { + "flag": "serial-id-missing-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": "control", + "reason": "STATIC", + "serialId": null, + "doLog": true + } + }, + { + "flag": "serial-id-targeting-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-vip", + "attributes": { + "segment": "vip" + }, + "result": { + "value": "vip", + "reason": "TARGETING_MATCH", + "serialId": 201, + "doLog": true + } + }, + { + "flag": "serial-id-targeting-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-standard", + "attributes": { + "segment": "standard" + }, + "result": { + "value": "standard", + "reason": "STATIC", + "serialId": 200, + "doLog": true + } + }, + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-repeat", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + }, + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-repeat", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json new file mode 100644 index 00000000000..dd988ba8583 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json @@ -0,0 +1,67 @@ +[ + { + "flag": "basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 100, + "doLog": true + } + }, + { + "flag": "experiment-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": "treatment", + "reason": "STATIC", + "serialId": 108, + "doLog": true + } + }, + { + "flag": "no-log-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-no-log", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 200, + "doLog": false + } + }, + { + "flag": "multi-serial-flag-1", + "variationType": "INTEGER", + "defaultValue": 0, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": 1, + "reason": "STATIC", + "serialId": 128, + "doLog": true + } + }, + { + "flag": "multi-serial-flag-2", + "variationType": "INTEGER", + "defaultValue": 0, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": 2, + "reason": "STATIC", + "serialId": 130, + "doLog": true + } + } +] diff --git a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json similarity index 65% rename from tests/parametric/test_ffe/test-case-start-and-end-date-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json index 7a48ec35886..53bf8859cdd 100644 --- a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "version": "1.15.0", - "country": "US" + "country": "US", + "version": "1.15.0" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "version": "0.20.1", - "country": "Canada" + "country": "Canada", + "version": "0.20.1" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "version": "2.1.13" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json new file mode 100644 index 00000000000..1bc2c914534 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json @@ -0,0 +1,16 @@ +[ + { + "description": "The flag and surrounding UFC config include unknown object fields that SDKs must ignore while evaluating the known schema fields.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "unknown-fields-tolerance-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "on" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json new file mode 100644 index 00000000000..c37fde4d573 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json @@ -0,0 +1,31 @@ +[ + { + "description": "The first allocation uses an unknown operator and the second is a catch-all trap. SDKs must return a parse error without falling through to the catch-all allocation.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "operator-grease-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the unknown-operator flag is skipped.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json new file mode 100644 index 00000000000..ca3b775ad61 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json @@ -0,0 +1,27 @@ +[ + { + "description": "A flag whose variation violates its declared type returns a parse error while the rest of the configuration remains usable.", + "attributes": {}, + "defaultValue": 0, + "flag": "variant-type-mismatch-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "user-1", + "variationType": "INTEGER" + }, + { + "description": "A valid flag in the same configuration continues to evaluate when a malformed neighboring flag returns a parse error.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json new file mode 100644 index 00000000000..0be74f1f7ac --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json @@ -0,0 +1,46 @@ +[ + { + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "alice", + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "bob", + "variationType": "NUMERIC" + }, + { + "attributes": { + "age": 50 + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" + } +] diff --git a/tests/parametric/test_ffe/test-json-config-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json similarity index 62% rename from tests/parametric/test_ffe/test-json-config-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json index ecc799546b0..45a9c2ee973 100644 --- a/tests/parametric/test_ffe/test-json-config-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json @@ -1,72 +1,76 @@ [ { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 1.0, "integer": 1, - "string": "one", - "float": 1.0 + "string": "one" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "diana", "attributes": { "Force Empty": true }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "TARGETING_MATCH", "value": {} - } + }, + "targetingKey": "diana", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json similarity index 70% rename from tests/parametric/test_ffe/test-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json index 45867e5897c..2b03b9a39a1 100644 --- a/tests/parametric/test_ffe/test-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "message": "Hello, world!" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "message": "Hello, world!" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json index 120647ec3b5..1ed7795bb95 100644 --- a/tests/parametric/test_ffe/test-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json @@ -1,54 +1,58 @@ [ { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ash", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "kümmert", "b": "schön" } - } + }, + "targetingKey": "ash", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ben", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "піклуватися", "b": "любов" } - } + }, + "targetingKey": "ben", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "cameron", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "照顾", "b": "漂亮" } - } + }, + "targetingKey": "cameron", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "darryl", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "🤗", "b": "🌸" } - } + }, + "targetingKey": "darryl", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-string-with-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-string-with-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json index e56322d44f0..32397a48398 100644 --- a/tests/parametric/test_ffe/test-string-with-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json @@ -1,794 +1,860 @@ [ { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_spaces", "attributes": { "string_with_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " a b c d e f " - } + }, + "targetingKey": "string_with_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_space", "attributes": { "string_with_only_one_space": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_one_space", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_spaces", "attributes": { "string_with_only_multiple_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_multiple_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dots", "attributes": { "string_with_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ".a.b.c.d.e.f." - } + }, + "targetingKey": "string_with_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dot", "attributes": { "string_with_only_one_dot": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "." - } + }, + "targetingKey": "string_with_only_one_dot", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dots", "attributes": { "string_with_only_multiple_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "......." - } + }, + "targetingKey": "string_with_only_multiple_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_comas", "attributes": { "string_with_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",a,b,c,d,e,f," - } + }, + "targetingKey": "string_with_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_coma", "attributes": { "string_with_only_one_coma": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "," - } + }, + "targetingKey": "string_with_only_one_coma", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_comas", "attributes": { "string_with_only_multiple_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",,,,,,," - } + }, + "targetingKey": "string_with_only_multiple_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_colons", "attributes": { "string_with_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":a:b:c:d:e:f:" - } + }, + "targetingKey": "string_with_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_colon", "attributes": { "string_with_only_one_colon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":" - } + }, + "targetingKey": "string_with_only_one_colon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_colons", "attributes": { "string_with_only_multiple_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":::::::" - } + }, + "targetingKey": "string_with_only_multiple_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_semicolons", "attributes": { "string_with_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";a;b;c;d;e;f;" - } + }, + "targetingKey": "string_with_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_semicolon", "attributes": { "string_with_only_one_semicolon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";" - } + }, + "targetingKey": "string_with_only_one_semicolon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_semicolons", "attributes": { "string_with_only_multiple_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";;;;;;;" - } + }, + "targetingKey": "string_with_only_multiple_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_slashes", "attributes": { "string_with_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/a/b/c/d/e/f/" - } + }, + "targetingKey": "string_with_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_slash", "attributes": { "string_with_only_one_slash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/" - } + }, + "targetingKey": "string_with_only_one_slash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_slashes", "attributes": { "string_with_only_multiple_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "///////" - } + }, + "targetingKey": "string_with_only_multiple_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dashes", "attributes": { "string_with_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-a-b-c-d-e-f-" - } + }, + "targetingKey": "string_with_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dash", "attributes": { "string_with_only_one_dash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-" - } + }, + "targetingKey": "string_with_only_one_dash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dashes", "attributes": { "string_with_only_multiple_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-------" - } + }, + "targetingKey": "string_with_only_multiple_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_underscores", "attributes": { "string_with_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_a_b_c_d_e_f_" - } + }, + "targetingKey": "string_with_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_underscore", "attributes": { "string_with_only_one_underscore": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_" - } + }, + "targetingKey": "string_with_only_one_underscore", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_underscores", "attributes": { "string_with_only_multiple_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_______" - } + }, + "targetingKey": "string_with_only_multiple_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_plus_signs", "attributes": { "string_with_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+a+b+c+d+e+f+" - } + }, + "targetingKey": "string_with_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_plus_sign", "attributes": { "string_with_only_one_plus_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+" - } + }, + "targetingKey": "string_with_only_one_plus_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_plus_signs", "attributes": { "string_with_only_multiple_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+++++++" - } + }, + "targetingKey": "string_with_only_multiple_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_equal_signs", "attributes": { "string_with_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=a=b=c=d=e=f=" - } + }, + "targetingKey": "string_with_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_equal_sign", "attributes": { "string_with_only_one_equal_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=" - } + }, + "targetingKey": "string_with_only_one_equal_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_equal_signs", "attributes": { "string_with_only_multiple_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=======" - } + }, + "targetingKey": "string_with_only_multiple_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dollar_signs", "attributes": { "string_with_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$a$b$c$d$e$f$" - } + }, + "targetingKey": "string_with_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dollar_sign", "attributes": { "string_with_only_one_dollar_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$" - } + }, + "targetingKey": "string_with_only_one_dollar_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dollar_signs", "attributes": { "string_with_only_multiple_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$$$$$$$" - } + }, + "targetingKey": "string_with_only_multiple_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_at_signs", "attributes": { "string_with_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@a@b@c@d@e@f@" - } + }, + "targetingKey": "string_with_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_at_sign", "attributes": { "string_with_only_one_at_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@" - } + }, + "targetingKey": "string_with_only_one_at_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_at_signs", "attributes": { "string_with_only_multiple_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@@@@@@@" - } + }, + "targetingKey": "string_with_only_multiple_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_amp_signs", "attributes": { "string_with_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&a&b&c&d&e&f&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026a\u0026b\u0026c\u0026d\u0026e\u0026f\u0026" + }, + "targetingKey": "string_with_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_amp_sign", "attributes": { "string_with_only_one_amp_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026" + }, + "targetingKey": "string_with_only_one_amp_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_amp_signs", "attributes": { "string_with_only_multiple_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&&&&&&&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026\u0026\u0026\u0026\u0026\u0026\u0026" + }, + "targetingKey": "string_with_only_multiple_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_hash_signs", "attributes": { "string_with_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#a#b#c#d#e#f#" - } + }, + "targetingKey": "string_with_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_hash_sign", "attributes": { "string_with_only_one_hash_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#" - } + }, + "targetingKey": "string_with_only_one_hash_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_hash_signs", "attributes": { "string_with_only_multiple_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#######" - } + }, + "targetingKey": "string_with_only_multiple_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_percentage_signs", "attributes": { "string_with_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%a%b%c%d%e%f%" - } + }, + "targetingKey": "string_with_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_percentage_sign", "attributes": { "string_with_only_one_percentage_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%" - } + }, + "targetingKey": "string_with_only_one_percentage_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_percentage_signs", "attributes": { "string_with_only_multiple_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%%%%%%%" - } + }, + "targetingKey": "string_with_only_multiple_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_tilde_signs", "attributes": { "string_with_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~a~b~c~d~e~f~" - } + }, + "targetingKey": "string_with_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_tilde_sign", "attributes": { "string_with_only_one_tilde_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~" - } + }, + "targetingKey": "string_with_only_one_tilde_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_tilde_signs", "attributes": { "string_with_only_multiple_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~~~~~~~" - } + }, + "targetingKey": "string_with_only_multiple_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_asterix_signs", "attributes": { "string_with_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*a*b*c*d*e*f*" - } + }, + "targetingKey": "string_with_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_asterix_sign", "attributes": { "string_with_only_one_asterix_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*" - } + }, + "targetingKey": "string_with_only_one_asterix_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_asterix_signs", "attributes": { "string_with_only_multiple_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*******" - } + }, + "targetingKey": "string_with_only_multiple_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_single_quotes", "attributes": { "string_with_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'a'b'c'd'e'f'" - } + }, + "targetingKey": "string_with_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_single_quote", "attributes": { "string_with_only_one_single_quote": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'" - } + }, + "targetingKey": "string_with_only_one_single_quote", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_single_quotes", "attributes": { "string_with_only_multiple_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'''''''" - } + }, + "targetingKey": "string_with_only_multiple_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_question_marks", "attributes": { "string_with_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?a?b?c?d?e?f?" - } + }, + "targetingKey": "string_with_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_question_mark", "attributes": { "string_with_only_one_question_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?" - } + }, + "targetingKey": "string_with_only_one_question_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_question_marks", "attributes": { "string_with_only_multiple_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "???????" - } + }, + "targetingKey": "string_with_only_multiple_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_exclamation_marks", "attributes": { "string_with_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!a!b!c!d!e!f!" - } + }, + "targetingKey": "string_with_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_exclamation_mark", "attributes": { "string_with_only_one_exclamation_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!" - } + }, + "targetingKey": "string_with_only_one_exclamation_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_exclamation_marks", "attributes": { "string_with_only_multiple_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!!!!!!!" - } + }, + "targetingKey": "string_with_only_multiple_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_opening_parentheses", "attributes": { "string_with_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(a(b(c(d(e(f(" - } + }, + "targetingKey": "string_with_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_opening_parenthese", "attributes": { "string_with_only_one_opening_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(" - } + }, + "targetingKey": "string_with_only_one_opening_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_opening_parentheses", "attributes": { "string_with_only_multiple_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(((((((" - } + }, + "targetingKey": "string_with_only_multiple_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_closing_parentheses", "attributes": { "string_with_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")a)b)c)d)e)f)" - } + }, + "targetingKey": "string_with_closing_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_closing_parenthese", "attributes": { "string_with_only_one_closing_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")" - } + }, + "targetingKey": "string_with_only_one_closing_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_closing_parentheses", "attributes": { "string_with_only_multiple_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")))))))" - } + }, + "targetingKey": "string_with_only_multiple_closing_parentheses", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/flags-v1.json b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json similarity index 60% rename from tests/parametric/test_ffe/flags-v1.json rename to tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json index f01bfe2ef86..a436b889b15 100644 --- a/tests/parametric/test_ffe/flags-v1.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json @@ -2,8 +2,11 @@ "createdAt": "2024-04-17T19:40:53.716Z", "format": "SERVER", "environment": { - "name": "Test" + "name": "Test", + "unknownEnvironmentField": "ignored" }, + "unknownTopLevelField": "ignored", + "unknownTopLevelNull": null, "flags": { "empty_flag": { "key": "empty_flag", @@ -78,9 +81,109 @@ "test": { "key": "test", "value": "test" + }, + "capturing-groups": { + "key": "capturing-groups", + "value": "capturing-groups" + }, + "unicode-word-boundary": { + "key": "unicode-word-boundary", + "value": "unicode-word-boundary" + }, + "comma-quantifier": { + "key": "comma-quantifier", + "value": "comma-quantifier" + }, + "literal-whitespace": { + "key": "literal-whitespace", + "value": "literal-whitespace" } }, "allocations": [ + { + "key": "comma-quantifier", + "rules": [ + { + "conditions": [ + { + "attribute": "comma_regex_case", + "operator": "MATCHES", + "value": "a{1,3}" + } + ] + } + ], + "splits": [ + { + "variationKey": "comma-quantifier", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "literal-whitespace", + "rules": [ + { + "conditions": [ + { + "attribute": "whitespace_regex_case", + "operator": "MATCHES", + "value": " exact " + } + ] + } + ], + "splits": [ + { + "variationKey": "literal-whitespace", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "capturing-groups", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "^([[:alnum:]._%+-]+)@capture\\.example$" + } + ] + } + ], + "splits": [ + { + "variationKey": "capturing-groups", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "unicode-word-boundary", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "(?u)\\bmañana\\b" + } + ] + } + ], + "splits": [ + { + "variationKey": "unicode-word-boundary", + "shards": [] + } + ], + "doLog": true + }, { "key": "partial-example", "rules": [ @@ -612,6 +715,279 @@ } ] }, + "semver-comparison-test": { + "key": "semver-comparison-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "legacy": { + "key": "legacy", + "value": "legacy" + }, + "stable": { + "key": "stable", + "value": "stable" + }, + "beta": { + "key": "beta", + "value": "beta" + }, + "launch": { + "key": "launch", + "value": "launch" + }, + "future": { + "key": "future", + "value": "future" + }, + "maximum": { + "key": "maximum", + "value": "maximum" + } + }, + "allocations": [ + { + "key": "maximum-version", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "9007199254740991.9007199254740991.9007199254740991" + } + ] + } + ], + "splits": [ + { + "variationKey": "maximum", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "legacy-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_LTE", + "value": "1.2.3-alpha.4" + } + ] + }, + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "1.2.3-alpha.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "2.3.4" + } + ] + } + ], + "splits": [ + { + "variationKey": "legacy", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "stable-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "2.3.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "3.4.5-beta.2" + } + ] + } + ], + "splits": [ + { + "variationKey": "stable", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "beta-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "3.4.5-beta.2" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "4.5.6-rc.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "launch-version", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "4.5.6-rc.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "launch", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "future-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "4.5.6-rc.1" + }, + { + "attribute": "app_version", + "operator": "SEMVER_NEQ", + "value": "5.6.7" + } + ] + } + ], + "splits": [ + { + "variationKey": "future", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-test": { + "key": "semver-invalid-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses an invalid configured SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18446744073709551616.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-waterfall-test": { + "key": "semver-invalid-comparand-waterfall-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately places an invalid configured SemVer comparand in a non-last allocation of a waterfall, followed by a valid default allocation. SDKs must reject the whole flag during parsing rather than skipping the invalid allocation and falling through to the default.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + }, + "fallback": { + "key": "fallback", + "value": "fallback" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18446744073709551616.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "default", + "rules": [], + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, "start-and-end-date-test": { "key": "start-and-end-date-test", "enabled": true, @@ -3293,6 +3669,1739 @@ "doLog": true } ] + }, + "microsecond-date-test": { + "key": "microsecond-date-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "expired": { + "key": "expired", + "value": "expired" + }, + "active": { + "key": "active", + "value": "active" + }, + "future": { + "key": "future", + "value": "future" + } + }, + "allocations": [ + { + "key": "expired-allocation", + "splits": [ + { + "variationKey": "expired", + "shards": [] + } + ], + "endAt": "2002-10-31T09:00:00.594321Z", + "doLog": true + }, + { + "key": "future-allocation", + "splits": [ + { + "variationKey": "future", + "shards": [] + } + ], + "startAt": "2052-10-31T09:00:00.123456Z", + "doLog": true + }, + { + "key": "active-allocation", + "splits": [ + { + "variationKey": "active", + "shards": [] + } + ], + "startAt": "2022-10-31T09:00:00.235982Z", + "endAt": "2050-10-31T09:00:00.987654Z", + "doLog": true + } + ] + }, + "unknown-fields-tolerance-flag": { + "key": "unknown-fields-tolerance-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately contains unknown fields at every fixed-schema UFC level. SDKs must ignore those fields and evaluate the flag normally.", + "unknownFlagField": "ignored", + "variations": { + "on": { + "key": "on", + "value": "on", + "unknownVariationField": "ignored" + } + }, + "allocations": [ + { + "key": "unknown-fields-allocation", + "unknownAllocationField": "ignored", + "rules": [ + { + "unknownRuleField": "ignored", + "conditions": [ + { + "attribute": "country", + "operator": "ONE_OF", + "value": [ + "US" + ], + "unknownConditionField": "ignored" + } + ] + } + ], + "splits": [ + { + "variationKey": "on", + "unknownSplitField": "ignored", + "shards": [ + { + "salt": "unknown-fields-tolerance-flag", + "totalShards": 10000, + "unknownShardField": "ignored", + "ranges": [ + { + "start": 0, + "end": 10000, + "unknownRangeField": "ignored" + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "operator-grease-flag": { + "key": "operator-grease-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses an unknown condition operator. SDKs must remove this flag rather than falling through to the catch-all allocation or rejecting the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": [ + { + "key": "grease-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "country", + "operator": "NOT_A_REAL_OPERATOR", + "value": "anything" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-regex-flag": { + "key": "invalid-regex-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses the invalid regex *@datadoghq.com. SDKs must remove this flag rather than falling through to a later allocation or rejecting the whole config.", + "variations": { + "invalid-regex-match": { + "key": "invalid-regex-match", + "value": "invalid-regex-match" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-regex-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "*@datadoghq.com" + } + ] + } + ], + "splits": [ + { + "variationKey": "invalid-regex-match", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-regex-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "variant-type-mismatch-flag": { + "key": "variant-type-mismatch-flag", + "enabled": true, + "variationType": "INTEGER", + "comment": "This flag is intentionally invalid: one variation contains a string even though the flag declares INTEGER values. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "invalid": { + "key": "invalid", + "value": "not-an-integer" + }, + "valid": { + "key": "valid", + "value": 0 + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "invalid", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-gt-condition-flag": { + "key": "invalid-gt-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives GT a string operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-gt-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "age", + "operator": "GT", + "value": "not-a-number" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-gt-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-one-of-condition-flag": { + "key": "invalid-one-of-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives ONE_OF a scalar operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-one-of-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "role", + "operator": "ONE_OF", + "value": "admin" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-one-of-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-is-null-condition-flag": { + "key": "invalid-is-null-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives IS_NULL a non-boolean operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-is-null-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "deleted_at", + "operator": "IS_NULL", + "value": "not-a-boolean" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-is-null-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "malformed-allocations-flag": { + "key": "malformed-allocations-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: allocations must be an array. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": "this-is-not-a-list" + }, + "missing-split-shards-flag": { + "key": "missing-split-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards is required even when the intended split is unsharded. SDKs must treat omitted shards differently from an explicit empty shards array.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-split-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap" + } + ], + "doLog": true + } + ] + }, + "overflow-total-shards-flag": { + "key": "overflow-total-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].totalShards must fit in an unsigned 32-bit integer. SDKs must remove this flag rather than overflowing or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "overflow-total-shards-flag", + "totalShards": 4294967296, + "ranges": [ + { + "start": 0, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "negative-shard-range-start-flag": { + "key": "negative-shard-range-start-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[].start must be an unsigned integer. SDKs must remove this flag rather than treating a negative range as matching traffic.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "negative-shard-range-start-flag", + "totalShards": 100, + "ranges": [ + { + "start": -1, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "null-shard-range-flag": { + "key": "null-shard-range-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[] contains null. SDKs must remove this flag rather than panicking or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "null-shard-range-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "null-shard-range-flag", + "totalShards": 8192, + "ranges": [ + null + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "null-shard-range-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "valid-flag-after-invalid-config": { + "key": "valid-flag-after-invalid-config", + "enabled": true, + "variationType": "STRING", + "comment": "This valid neighbor proves malformed or unsupported flags do not poison the rest of the remote config.", + "variations": { + "expected": { + "key": "expected", + "value": "expected" + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "expected", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-precedence-test": { + "key": "semver-precedence-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates SemVer precedence rules that are easy to get wrong: multi-digit numeric prerelease ordering (beta.2 < beta.11), build-metadata-ignored across every operator (EQ/NEQ/LT/LTE/GT/GTE), and alphanumeric prerelease ordering (alpha < alpha.1 < alpha.beta < beta < release). Each allocation gates on a unique SEMVER_EQ attribute version so test attributes cannot cross-match other allocations; negative build-metadata cases (NEQ/LT/GT) fall through to DEFAULT.", + "variations": { + "numeric-prerelease": { + "key": "numeric-prerelease", + "value": "numeric-prerelease" + }, + "build-equal-precedence": { + "key": "build-equal-precedence", + "value": "build-equal-precedence" + }, + "build-dotted-eq": { + "key": "build-dotted-eq", + "value": "build-dotted-eq" + }, + "build-lte": { + "key": "build-lte", + "value": "build-lte" + }, + "build-gte": { + "key": "build-gte", + "value": "build-gte" + }, + "alpha-before-alpha-1": { + "key": "alpha-before-alpha-1", + "value": "alpha-before-alpha-1" + }, + "alpha-1-before-alpha-beta": { + "key": "alpha-1-before-alpha-beta", + "value": "alpha-1-before-alpha-beta" + }, + "alpha-beta-before-beta": { + "key": "alpha-beta-before-beta", + "value": "alpha-beta-before-beta" + }, + "beta-before-release": { + "key": "beta-before-release", + "value": "beta-before-release" + }, + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "numeric-prerelease", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "10.0.0-beta.2" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "10.0.0-beta.11" + } + ] + } + ], + "splits": [ + { + "variationKey": "numeric-prerelease", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-equal-precedence", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "20.0.0+linux" + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "20.0.0+darwin" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-equal-precedence", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-dotted-eq", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "30.0.0+exp.sha.5114f85" + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "30.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-dotted-eq", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-lte", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "40.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LTE", + "value": "40.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-lte", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-gte", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "50.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "50.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-gte", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-neq", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "60.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_NEQ", + "value": "60.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-lt", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "70.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "70.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-gt", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "80.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "80.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-before-alpha-1", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "90.0.0-alpha" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "90.0.0-alpha.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-before-alpha-1", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-1-before-alpha-beta", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "91.0.0-alpha.1" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "91.0.0-alpha.beta" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-1-before-alpha-beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-beta-before-beta", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "92.0.0-alpha.beta" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "92.0.0-beta" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-beta-before-beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "beta-before-release", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "93.0.0-beta" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "93.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "beta-before-release", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-input-test": { + "key": "semver-version-parts-input-test", + "enabled": true, + "variationType": "STRING", + "comment": "Exercises one- through five-part runtime attributes using only canonical three-part configured comparands, so comparand ingestion cannot prevent an input case from running. Scenario gates isolate each runtime comparison.", + "variations": { + "one-part-input": { + "key": "one-part-input", + "value": "one-part-input" + }, + "two-part-input": { + "key": "two-part-input", + "value": "two-part-input" + }, + "four-part-input": { + "key": "four-part-input", + "value": "four-part-input" + }, + "five-part-input": { + "key": "five-part-input", + "value": "five-part-input" + } + }, + "allocations": [ + { + "key": "one-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "one-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "one-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "two-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "two-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "two-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "four-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "four-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "17.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "four-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "five-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "five-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "17.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "five-part-input", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-one-part-comparand-test": { + "key": "semver-version-parts-one-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a one-part configured SemVer comparand from every other extended comparand shape.", + "variations": { + "one-part-comparand": { + "key": "one-part-comparand", + "value": "one-part-comparand" + } + }, + "allocations": [ + { + "key": "one-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18" + } + ] + } + ], + "splits": [ + { + "variationKey": "one-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-two-part-comparand-test": { + "key": "semver-version-parts-two-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a two-part configured SemVer comparand from every other extended comparand shape.", + "variations": { + "two-part-comparand": { + "key": "two-part-comparand", + "value": "two-part-comparand" + } + }, + "allocations": [ + { + "key": "two-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "two-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-four-part-comparand-test": { + "key": "semver-version-parts-four-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a four-part configured SemVer comparand while the runtime attribute remains a canonical three-part version.", + "variations": { + "four-part-comparand": { + "key": "four-part-comparand", + "value": "four-part-comparand" + } + }, + "allocations": [ + { + "key": "four-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "18.0.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "four-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-five-part-comparand-test": { + "key": "semver-version-parts-five-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a five-part configured SemVer comparand while the runtime attribute remains a canonical three-part version.", + "variations": { + "five-part-comparand": { + "key": "five-part-comparand", + "value": "five-part-comparand" + } + }, + "allocations": [ + { + "key": "five-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "18.0.0.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "five-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-syntax-test": { + "key": "semver-invalid-comparand-syntax-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a non-parseable SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "not-a-version" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-valid-comparand-hyphenated-prerelease-test": { + "key": "semver-valid-comparand-hyphenated-prerelease-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag uses a valid SemVer comparand with repeated hyphens inside its prerelease identifiers.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "valid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.2.3----R-S.12.9.1--.12+meta" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-consecutive-dots-test": { + "key": "semver-invalid-comparand-consecutive-dots-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses consecutive dots in its SemVer prerelease comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.0.0-alpha..1" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-dot-suffix-test": { + "key": "semver-invalid-comparand-dot-suffix-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a Ruby-style dot-suffixed SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.2.3.DEV" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-vprefix-test": { + "key": "semver-invalid-comparand-vprefix-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a v-prefixed SemVer comparand, which is not valid SemVer. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "v1.2.3" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-leading-zero-test": { + "key": "semver-invalid-comparand-leading-zero-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a leading-zero core component in its SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "01.2.3" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-non-string-test": { + "key": "semver-invalid-comparand-non-string-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a non-string (JSON number) SemVer comparand. SDKs must reject this as an invalid comparand without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": 1.2 + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "serial-id-basic-flag": { + "key": "serial-id-basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 42 + } + ], + "doLog": true + } + ] + }, + "serial-id-zero-flag": { + "key": "serial-id-zero-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "zero-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 0 + } + ], + "doLog": true + } + ] + }, + "serial-id-missing-flag": { + "key": "serial-id-missing-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "treatment": { + "key": "treatment", + "value": "treatment" + } + }, + "allocations": [ + { + "key": "no-serial-allocation", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "serial-id-targeting-flag": { + "key": "serial-id-targeting-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "vip": { + "key": "vip", + "value": "vip" + }, + "standard": { + "key": "standard", + "value": "standard" + } + }, + "allocations": [ + { + "key": "vip-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "segment", + "operator": "ONE_OF", + "value": [ + "vip" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "vip", + "shards": [], + "serialId": 201 + } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "standard", + "shards": [], + "serialId": 200 + } + ], + "doLog": true + } + ] + }, + "basic-flag": { + "key": "basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 100 + } + ], + "doLog": true + } + ] + }, + "experiment-flag": { + "key": "experiment-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "treatment": { + "key": "treatment", + "value": "treatment" + } + }, + "allocations": [ + { + "key": "experiment-allocation", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 108 + } + ], + "doLog": true + } + ] + }, + "no-log-flag": { + "key": "no-log-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "no-log-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 200 + } + ], + "doLog": false + } + ] + }, + "multi-serial-flag-1": { + "key": "multi-serial-flag-1", + "enabled": true, + "variationType": "INTEGER", + "variations": { + "v1": { + "key": "v1", + "value": 1 + } + }, + "allocations": [ + { + "key": "allocation-1", + "rules": [], + "splits": [ + { + "variationKey": "v1", + "shards": [], + "serialId": 128 + } + ], + "doLog": true + } + ] + }, + "multi-serial-flag-2": { + "key": "multi-serial-flag-2", + "enabled": true, + "variationType": "INTEGER", + "variations": { + "v2": { + "key": "v2", + "value": 2 + } + }, + "allocations": [ + { + "key": "allocation-2", + "rules": [], + "splits": [ + { + "variationKey": "v2", + "shards": [], + "serialId": 130 + } + ], + "doLog": true + } + ] } } -} \ No newline at end of file +} diff --git a/tests/parametric/test_ffe/serial-id-metadata-flags.json b/tests/parametric/test_ffe/serial-id-metadata-flags.json deleted file mode 100644 index 35bf0ab4c74..00000000000 --- a/tests/parametric/test_ffe/serial-id-metadata-flags.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "createdAt": "2024-01-01T00:00:00.000Z", - "format": "SERVER", - "environment": { "name": "Test" }, - "flags": { - "serial-id-basic-flag": { - "key": "serial-id-basic-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { "key": "on", "value": true }, - "off": { "key": "off", "value": false } - }, - "allocations": [ - { - "key": "on-allocation", - "rules": [], - "splits": [ - { "variationKey": "on", "shards": [], "serialId": 42 } - ], - "doLog": true - } - ] - }, - "serial-id-zero-flag": { - "key": "serial-id-zero-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { "key": "on", "value": true }, - "off": { "key": "off", "value": false } - }, - "allocations": [ - { - "key": "zero-allocation", - "rules": [], - "splits": [ - { "variationKey": "on", "shards": [], "serialId": 0 } - ], - "doLog": true - } - ] - }, - "serial-id-missing-flag": { - "key": "serial-id-missing-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "control": { "key": "control", "value": "control" }, - "treatment": { "key": "treatment", "value": "treatment" } - }, - "allocations": [ - { - "key": "no-serial-allocation", - "rules": [], - "splits": [ - { "variationKey": "control", "shards": [] } - ], - "doLog": true - } - ] - }, - "serial-id-targeting-flag": { - "key": "serial-id-targeting-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "vip": { "key": "vip", "value": "vip" }, - "standard": { "key": "standard", "value": "standard" } - }, - "allocations": [ - { - "key": "vip-allocation", - "rules": [ - { - "conditions": [ - { "attribute": "segment", "operator": "ONE_OF", "value": ["vip"] } - ] - } - ], - "splits": [ - { "variationKey": "vip", "shards": [], "serialId": 201 } - ], - "doLog": true - }, - { - "key": "default-allocation", - "rules": [], - "splits": [ - { "variationKey": "standard", "shards": [], "serialId": 200 } - ], - "doLog": true - } - ] - } - } -} diff --git a/tests/parametric/test_ffe/span-enrichment-flags.json b/tests/parametric/test_ffe/span-enrichment-flags.json deleted file mode 100644 index 232315bcbe8..00000000000 --- a/tests/parametric/test_ffe/span-enrichment-flags.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "createdAt": "2024-04-17T19:40:53.716Z", - "format": "SERVER", - "environment": { - "name": "Test" - }, - "flags": { - "basic-flag": { - "key": "basic-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { - "key": "on", - "value": true - }, - "off": { - "key": "off", - "value": false - } - }, - "allocations": [ - { - "key": "on-allocation", - "rules": [], - "splits": [ - { - "variationKey": "on", - "shards": [], - "serialId": 100 - } - ], - "doLog": true - } - ] - }, - "experiment-flag": { - "key": "experiment-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "control": { - "key": "control", - "value": "control" - }, - "treatment": { - "key": "treatment", - "value": "treatment" - } - }, - "allocations": [ - { - "key": "experiment-allocation", - "rules": [], - "splits": [ - { - "variationKey": "treatment", - "shards": [], - "serialId": 108 - } - ], - "doLog": true - } - ] - }, - "no-log-flag": { - "key": "no-log-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { - "key": "on", - "value": true - }, - "off": { - "key": "off", - "value": false - } - }, - "allocations": [ - { - "key": "no-log-allocation", - "rules": [], - "splits": [ - { - "variationKey": "on", - "shards": [], - "serialId": 200 - } - ], - "doLog": false - } - ] - }, - "multi-serial-flag-1": { - "key": "multi-serial-flag-1", - "enabled": true, - "variationType": "INTEGER", - "variations": { - "v1": { - "key": "v1", - "value": 1 - } - }, - "allocations": [ - { - "key": "allocation-1", - "rules": [], - "splits": [ - { - "variationKey": "v1", - "shards": [], - "serialId": 128 - } - ], - "doLog": true - } - ] - }, - "multi-serial-flag-2": { - "key": "multi-serial-flag-2", - "enabled": true, - "variationType": "INTEGER", - "variations": { - "v2": { - "key": "v2", - "value": 2 - } - }, - "allocations": [ - { - "key": "allocation-2", - "rules": [], - "splits": [ - { - "variationKey": "v2", - "shards": [], - "serialId": 130 - } - ], - "doLog": true - } - ] - } - } -} diff --git a/tests/parametric/test_ffe/test-case-regex-flag.json b/tests/parametric/test_ffe/test-case-regex-flag.json deleted file mode 100644 index 94aa87f23a9..00000000000 --- a/tests/parametric/test_ffe/test-case-regex-flag.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "alice", - "attributes": { - "version": "1.15.0", - "email": "alice@example.com" - }, - "result": { - "value": "partial-example" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "bob", - "attributes": { - "version": "0.20.1", - "email": "bob@test.com" - }, - "result": { - "value": "test" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "charlie", - "attributes": { - "version": "2.1.13" - }, - "result": { - "value": "none" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "derek", - "attributes": { - "version": "2.1.13", - "email": "derek@gmail.com" - }, - "result": { - "value": "none" - } - } -] diff --git a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/test-flag-that-does-not-exist.json deleted file mode 100644 index 7499bba1c50..00000000000 --- a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json +++ /dev/null @@ -1,40 +0,0 @@ -[ - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", - "attributes": { - "age": 50 - }, - "result": { - "value": 0.0 - } - } -] diff --git a/tests/parametric/test_ffe/test_configuration_sources.py b/tests/parametric/test_ffe/test_configuration_sources.py index 79a4a716b8f..ca533dc42bd 100644 --- a/tests/parametric/test_ffe/test_configuration_sources.py +++ b/tests/parametric/test_ffe/test_configuration_sources.py @@ -34,7 +34,7 @@ parametrize = pytest.mark.parametrize pytest_plugins = ["utils.mocked_backend.ffe"] -UFC_VALID_FIXTURE = Path(__file__).parent / "flags-v1.json" +UFC_VALID_FIXTURE = Path(__file__).parent / "ffe-system-test-data" / "ufc-config.json" RC_PRODUCT = "FFE_FLAGS" TEST_API_KEY = "system-tests-mock-api-key" MOCK_STATUS_ATTEMPTS = 25 diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 35b15215c50..642a73d56d0 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -18,6 +18,8 @@ RC_PATH = f"datadog/2/{RC_PRODUCT}" FFE_READY_RETRY_ATTEMPTS = 10 FFE_READY_RETRY_INTERVAL_SECONDS = 0.2 +FFE_SYSTEM_TEST_DATA_DIR = Path(__file__).parent / "ffe-system-test-data" +FFE_EVALUATION_CASES_DIR = FFE_SYSTEM_TEST_DATA_DIR / "evaluation-cases" parametrize = pytest.mark.parametrize @@ -25,10 +27,7 @@ # Load the UFC fixture file at module level def _load_ufc_fixture() -> dict[str, Any]: """Load the UFC fixture file.""" - fixture_path = Path(__file__).parent / "flags-v1.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") + fixture_path = FFE_SYSTEM_TEST_DATA_DIR / "ufc-config.json" with fixture_path.open() as f: return json.load(f) @@ -36,13 +35,10 @@ def _load_ufc_fixture() -> dict[str, Any]: def _get_test_case_files() -> list[str]: """Get all test case files from the fixtures directory.""" - test_data_dir = Path(__file__).parent - if not test_data_dir.exists(): - return [] - - # Exclude base fixtures that aren't test cases - excluded = {"flags-v1.json", "serial-id-metadata-flags.json", "span-enrichment-flags.json"} - return [f.name for f in test_data_dir.iterdir() if f.suffix == ".json" and f.name not in excluded] + case_files = sorted(path.name for path in FFE_EVALUATION_CASES_DIR.glob("*.json")) + if not case_files: + raise FileNotFoundError(f"No canonical evaluation cases found in {FFE_EVALUATION_CASES_DIR}") + return case_files # Load fixture at module level for reuse across tests @@ -103,7 +99,7 @@ def _ffe_evaluate_with_rc_retry( flag: str, variation_type: str, default_value: bool | str | float | dict[str, Any], - targeting_key: str, + targeting_key: str | None, attributes: dict[str, Any] | None = None, ) -> dict[str, Any]: result = test_library.ffe_evaluate( @@ -159,13 +155,12 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI """ # Load the test case file - test_case_path = Path(__file__).parent / test_case_file - - if not test_case_path.exists(): - pytest.skip(f"Test case file not found: {test_case_path}") + test_case_path = FFE_EVALUATION_CASES_DIR / test_case_file with test_case_path.open() as f: test_cases = json.load(f) + assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file}" + assert test_cases, f"No evaluation cases in {test_case_file}" # Set up UFC Remote Config and wait for it to be applied _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) @@ -181,7 +176,7 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI default_value = test_case["defaultValue"] targeting_key = test_case["targetingKey"] attributes = test_case.get("attributes", {}) - expected_result = test_case["result"]["value"] + expected_result = test_case["result"] result = _ffe_evaluate_with_rc_retry( test_library, @@ -198,8 +193,19 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI actual_value = result.get("value") # Assert the evaluation result matches expected value - assert actual_value == expected_result, ( + assert actual_value == expected_result["value"], ( f"Test case {i} in {test_case_file} failed: " f"flag='{flag}', targetingKey='{targeting_key}', " - f"expected={expected_result}, actual={actual_value}" + f"expected={expected_result}, actual={result}" + ) + assert result.get("reason") == expected_result["reason"], ( + f"Test case {i} in {test_case_file}: expected reason={expected_result['reason']}, result={result}" ) + if "errorCode" in expected_result: + assert result.get("errorCode") == expected_result["errorCode"], ( + f"Test case {i} in {test_case_file}: expected errorCode={expected_result['errorCode']}, result={result}" + ) + elif expected_result["reason"] != "ERROR": + assert result.get("errorCode") in {None, ""}, ( + f"Test case {i} in {test_case_file}: unexpected evaluation error; result={result}" + ) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 8e8a9e14d45..ec0d15b6f3e 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -23,7 +23,6 @@ import json import time -from pathlib import Path from typing import Any from utils import pytest @@ -32,6 +31,7 @@ from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary +from tests.parametric.test_ffe.utils import load_ufc_fixture_for_cases RC_PRODUCT = "FFE_FLAGS" RC_PATH = f"datadog/2/{RC_PRODUCT}" @@ -43,18 +43,7 @@ parametrize = pytest.mark.parametrize -def _load_serial_id_metadata_fixture() -> dict[str, Any]: - """Load the UFC fixture file for serial ID metadata tests.""" - fixture_path = Path(__file__).parent / "serial-id-metadata-flags.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") - - with fixture_path.open() as f: - return json.load(f) - - -UFC_SERIAL_ID_METADATA_DATA = _load_serial_id_metadata_fixture() +UFC_SERIAL_ID_METADATA_DATA = load_ufc_fixture_for_cases("test-case-serial-id-metadata.json") DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", diff --git a/tests/parametric/test_ffe/test_span_enrichment.py b/tests/parametric/test_ffe/test_span_enrichment.py index 6a4e54bdacb..749913bb455 100644 --- a/tests/parametric/test_ffe/test_span_enrichment.py +++ b/tests/parametric/test_ffe/test_span_enrichment.py @@ -21,14 +21,18 @@ import json from utils import pytest -from pathlib import Path from typing import Any from utils import features, scenarios from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary -from tests.parametric.test_ffe.utils import decode_delta_varint, encode_delta_varint, hash_targeting_key +from tests.parametric.test_ffe.utils import ( + decode_delta_varint, + encode_delta_varint, + hash_targeting_key, + load_ufc_fixture_for_cases, +) RC_PRODUCT = "FFE_FLAGS" @@ -37,18 +41,7 @@ parametrize = pytest.mark.parametrize -def _load_span_enrichment_fixture() -> dict[str, Any]: - """Load the UFC fixture file for span enrichment tests.""" - fixture_path = Path(__file__).parent / "span-enrichment-flags.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") - - with fixture_path.open() as f: - return json.load(f) - - -UFC_SPAN_ENRICHMENT_DATA = _load_span_enrichment_fixture() +UFC_SPAN_ENRICHMENT_DATA = load_ufc_fixture_for_cases("test-case-span-enrichment-inputs.json") DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", diff --git a/tests/parametric/test_ffe/utils.py b/tests/parametric/test_ffe/utils.py index 325c825f949..ad655f9cf84 100644 --- a/tests/parametric/test_ffe/utils.py +++ b/tests/parametric/test_ffe/utils.py @@ -11,12 +11,26 @@ import base64 import hashlib +import json +from pathlib import Path +from typing import Any # Varint encoding constants VARINT_DATA_BITS = 0x7F # Lower 7 bits contain data VARINT_CONTINUATION_BIT = 0x80 # MSB indicates more bytes follow +def load_ufc_fixture_for_cases(case_filename: str) -> dict[str, Any]: + """Load the canonical flag definitions used by a specific integration suite.""" + fixture_root = Path(__file__).parent / "ffe-system-test-data" + config = json.loads((fixture_root / "ufc-config.json").read_text()) + cases = json.loads((fixture_root / "evaluation-cases" / case_filename).read_text()) + # Keep unrelated evaluator edge cases out of transport/span integration tests. + flag_keys = dict.fromkeys(case["flag"] for case in cases) + config["flags"] = {key: config["flags"][key] for key in flag_keys} + return config + + def decode_varint(data: bytes, offset: int = 0) -> tuple[int, int]: """Decode a single varint from bytes. diff --git a/tests/test_the_test/test_update_ffe_fixtures.py b/tests/test_the_test/test_update_ffe_fixtures.py new file mode 100644 index 00000000000..8f53acd82ee --- /dev/null +++ b/tests/test_the_test/test_update_ffe_fixtures.py @@ -0,0 +1,145 @@ +# ruff: noqa: SLF001 + +from pathlib import Path +import shutil + +from utils import pytest + +from utils import scenarios +from utils.scripts import update_ffe_fixtures + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["main", "feature/fixture-update", "v1.2.3", "ea8b5cc5"]) +def test_validate_fixture_ref_accepts_git_refs(fixture_ref: str) -> None: + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["", "-main", "../main", "main^", "main ref"]) +def test_validate_fixture_ref_rejects_unsafe_refs(fixture_ref: str) -> None: + with pytest.raises(ValueError, match="Invalid FFE fixture ref"): + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_imports_only_server_evaluator_data(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + (source / "evaluation-cases").mkdir(parents=True) + snapshot.mkdir() + (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + (source / "README.md").write_text("upstream documentation", encoding="utf-8") + (source / "AGENTS.md").write_text("upstream instructions", encoding="utf-8") + (source / "precomputed-assignments").mkdir() + + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + assert (snapshot / "ufc-config.json").is_file() + assert (snapshot / "evaluation-cases" / "case.json").is_file() + assert not (snapshot / "README.md").exists() + assert not (snapshot / "AGENTS.md").exists() + assert not (snapshot / "precomputed-assignments").exists() + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_rejects_symbolic_links(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + source.mkdir() + snapshot.mkdir() + (source / "evaluation-cases").mkdir() + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + target = source / "target.json" + target.write_text("{}", encoding="utf-8") + (source / "evaluation-cases" / "linked.json").symlink_to(target) + + with pytest.raises(ValueError, match="Refusing to copy symbolic link"): + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + +@scenarios.test_the_test +def test_validate_fixture_snapshot_counts_evaluations(tmp_path: Path) -> None: + cases_directory = tmp_path / "evaluation-cases" + cases_directory.mkdir() + (tmp_path / "ufc-config.json").write_text('{"flags": {"one": {}}}', encoding="utf-8") + (cases_directory / "first.json").write_text('[{"flag": "one"}]', encoding="utf-8") + (cases_directory / "second.json").write_text('[{"flag": "two"}, {"flag": "three"}]', encoding="utf-8") + + assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 3 + + +@scenarios.test_the_test +def test_have_same_contents_ignores_generated_source_metadata(tmp_path: Path) -> None: + snapshot = tmp_path / "snapshot" + destination = tmp_path / "destination" + snapshot.mkdir() + destination.mkdir() + (snapshot / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "SOURCE.md").write_text("generated", encoding="utf-8") + + assert update_ffe_fixtures._have_same_contents(snapshot, destination) + + (destination / "fixture.json").write_text('{"changed": true}', encoding="utf-8") + assert not update_ffe_fixtures._have_same_contents(snapshot, destination) + + +@scenarios.test_the_test +def test_update_fixtures_preserves_snapshot_on_validation_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "upstream" + destination = tmp_path / "destination" + (source / "evaluation-cases").mkdir(parents=True) + (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") + (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + destination.mkdir() + (destination / "SOURCE.md").write_text("original snapshot", encoding="utf-8") + + def fake_git(working_directory: Path, *arguments: str) -> str: + if arguments[0] == "checkout": + shutil.copytree(source, working_directory, dirs_exist_ok=True) + return "a" * 40 + + monkeypatch.setattr(update_ffe_fixtures, "_run_git", fake_git) + monkeypatch.setattr(update_ffe_fixtures, "FIXTURE_DESTINATION", destination) + with pytest.raises(ValueError, match="at least one evaluation case"): + update_ffe_fixtures.update_fixtures("main") + assert (destination / "SOURCE.md").read_text() == "original snapshot" + + +@scenarios.test_the_test +def test_update_fixtures_records_source_and_removes_stale_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "upstream" + destination = tmp_path / "destination" + output = tmp_path / "outputs" + (source / "evaluation-cases").mkdir(parents=True) + (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") + case_file = source / "evaluation-cases" / "case.json" + case_file.write_text('[{"flag": "flag"}]', encoding="utf-8") + destination.mkdir() + (destination / "stale.json").write_text("[]", encoding="utf-8") + + def fake_git(working_directory: Path, *arguments: str) -> str: + if arguments[0] == "checkout": + shutil.copytree(source, working_directory, dirs_exist_ok=True) + return "a" * 40 + + monkeypatch.setattr(update_ffe_fixtures, "_run_git", fake_git) + monkeypatch.setattr(update_ffe_fixtures, "FIXTURE_DESTINATION", destination) + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + assert update_ffe_fixtures.update_fixtures("main") + assert not (destination / "stale.json").exists() + assert (destination / "evaluation-cases" / "case.json").read_bytes() == case_file.read_bytes() + metadata = (destination / "SOURCE.md").read_bytes() + assert ("a" * 40).encode() in metadata + assert "fixture_count=1\nchanged=true\n" in output.read_text() + + (source / "README.md").write_text("documentation-only upstream change", encoding="utf-8") + assert not update_ffe_fixtures.update_fixtures("main") + assert (destination / "SOURCE.md").read_bytes() == metadata + assert output.read_text().endswith("changed=false\n") diff --git a/utils/docker_fixtures/_test_clients/_test_client_parametric.py b/utils/docker_fixtures/_test_clients/_test_client_parametric.py index 5a512b87c19..895f006e227 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_parametric.py +++ b/utils/docker_fixtures/_test_clients/_test_client_parametric.py @@ -824,7 +824,7 @@ def ffe_evaluate( flag: str, variation_type: str, default_value: bool | str | float | dict, - targeting_key: str, + targeting_key: str | None, attributes: dict | None = None, span_id: int | str | None = None, ) -> dict: @@ -1267,7 +1267,7 @@ def ffe_evaluate( flag: str, variation_type: str, default_value: bool | str | float | dict, - targeting_key: str, + targeting_key: str | None, attributes: dict | None = None, span_id: int | str | None = None, ) -> dict: diff --git a/utils/mocked_backend/ffe.py b/utils/mocked_backend/ffe.py index 5b288d176e2..dbb1314a5f4 100644 --- a/utils/mocked_backend/ffe.py +++ b/utils/mocked_backend/ffe.py @@ -48,7 +48,7 @@ EXPECTED_DD_ENV = "test" CONFIG_QUERY = urlencode({"dd_env": EXPECTED_DD_ENV}) REPO_ROOT = Path(__file__).parents[2] -UFC_FIXTURE_PATH = REPO_ROOT / "tests" / "parametric" / "test_ffe" / "flags-v1.json" +UFC_FIXTURE_PATH = REPO_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" / "ufc-config.json" MALFORMED_UFC_BYTES = b'{"flags": [' UFC_RESPONSE_TYPE = "universal-flag-configuration" diff --git a/utils/scripts/update_ffe_fixtures.py b/utils/scripts/update_ffe_fixtures.py new file mode 100644 index 00000000000..e7f3b95f484 --- /dev/null +++ b/utils/scripts/update_ffe_fixtures.py @@ -0,0 +1,206 @@ +"""Refresh the checked-in FFE fixture snapshot from its canonical repository.""" + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_DESTINATION = REPOSITORY_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" +FIXTURE_REPOSITORY = "https://github.com/DataDog/ffe-system-test-data.git" +FIXTURE_REF_PATTERN = re.compile(r"[A-Za-z0-9._/-]+") + + +def _validate_fixture_ref(fixture_ref: str) -> None: + if ( + not fixture_ref + or fixture_ref.startswith("-") + or ".." in fixture_ref + or FIXTURE_REF_PATTERN.fullmatch(fixture_ref) is None + ): + msg = f"Invalid FFE fixture ref: {fixture_ref!r}" + raise ValueError(msg) + + +def _run_git(working_directory: Path, *arguments: str) -> str: + environment = { + **os.environ, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + } + result = subprocess.run( + ["git", *arguments], + cwd=working_directory, + env=environment, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _copy_entry(source: Path, destination: Path) -> None: + if source.is_symlink(): + msg = f"Refusing to copy symbolic link from FFE fixture repository: {source}" + raise ValueError(msg) + + if source.is_file(): + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + return + + if not source.is_dir(): + msg = f"Refusing to copy unsupported filesystem entry from FFE fixture repository: {source}" + raise ValueError(msg) + + destination.mkdir(parents=True, exist_ok=True) + for child in sorted(source.iterdir()): + _copy_entry(child, destination / child.name) + + +def _copy_fixture_snapshot(source: Path, snapshot: Path) -> None: + # Import only the server evaluator corpus that this suite consumes. Other + # upstream suites (for example precomputed assignments) have different formats. + for name in ("ufc-config.json", "evaluation-cases"): + _copy_entry(source / name, snapshot / name) + + +def _validate_fixture_snapshot(snapshot: Path) -> int: + config_path = snapshot / "ufc-config.json" + cases_directory = snapshot / "evaluation-cases" + if not config_path.is_file() or not cases_directory.is_dir(): + msg = "FFE fixture repository does not contain the expected fixture layout" + raise ValueError(msg) + + with config_path.open(encoding="utf-8") as config_file: + config = json.load(config_file) + if not isinstance(config, dict) or not isinstance(config.get("flags"), dict) or not config["flags"]: + msg = "FFE UFC fixture must contain a non-empty flags object" + raise ValueError(msg) + + case_files = sorted(cases_directory.glob("*.json")) + if any(not path.is_file() or path.suffix != ".json" for path in cases_directory.iterdir()): + msg = "FFE evaluation-cases must contain only JSON files" + raise ValueError(msg) + if not case_files: + msg = "No FFE JSON fixture files found" + raise ValueError(msg) + + fixture_count = 0 + for case_file in case_files: + with case_file.open(encoding="utf-8") as fixture_file: + test_cases = json.load(fixture_file) + if not isinstance(test_cases, list): + msg = f"{case_file} must contain a JSON array of test cases" + raise TypeError(msg) + if not test_cases: + msg = f"{case_file} must contain at least one evaluation case" + raise ValueError(msg) + fixture_count += len(test_cases) + + if fixture_count == 0: + msg = "No FFE fixture test cases found" + raise ValueError(msg) + + return fixture_count + + +def _relative_files(directory: Path, excluded_files: frozenset[str] = frozenset()) -> list[Path]: + return sorted( + path.relative_to(directory) + for path in directory.rglob("*") + if path.is_file() and path.relative_to(directory).as_posix() not in excluded_files + ) + + +def _have_same_contents(snapshot: Path, destination: Path) -> bool: + if not destination.is_dir(): + return False + + snapshot_files = _relative_files(snapshot) + destination_files = _relative_files(destination, frozenset({"SOURCE.md"})) + if snapshot_files != destination_files: + return False + + return all( + (snapshot / relative_path).read_bytes() == (destination / relative_path).read_bytes() + for relative_path in snapshot_files + ) + + +def _write_source_metadata(snapshot: Path, source_commit: str) -> None: + (snapshot / "SOURCE.md").write_text( + f"""# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: {source_commit} + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in +ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed +draft test PR only when the server evaluator fixture contents change. +""", + encoding="utf-8", + ) + + +def _set_github_outputs(source_commit: str, fixture_count: int, *, changed: bool) -> None: + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with Path(github_output).open("a", encoding="utf-8") as output_file: + output_file.write(f"source_commit={source_commit}\n") + output_file.write(f"fixture_count={fixture_count}\n") + output_file.write(f"changed={str(changed).lower()}\n") + + +def update_fixtures(fixture_ref: str) -> bool: + """Update the snapshot and return whether its allowed contents changed.""" + _validate_fixture_ref(fixture_ref) + + with tempfile.TemporaryDirectory(prefix="ffe-system-test-data-") as temporary_directory: + temporary_path = Path(temporary_directory) + source = temporary_path / "source" + snapshot = temporary_path / "snapshot" + source.mkdir() + snapshot.mkdir() + + _run_git(source, "init", "--quiet") + _run_git(source, "remote", "add", "origin", FIXTURE_REPOSITORY) + _run_git(source, "fetch", "--quiet", "--depth", "1", "origin", fixture_ref) + _run_git(source, "checkout", "--quiet", "--detach", "FETCH_HEAD") + source_commit = _run_git(source, "rev-parse", "HEAD") + + _copy_fixture_snapshot(source, snapshot) + fixture_count = _validate_fixture_snapshot(snapshot) + changed = not _have_same_contents(snapshot, FIXTURE_DESTINATION) + + if changed: + _write_source_metadata(snapshot, source_commit) + if FIXTURE_DESTINATION.exists(): + shutil.rmtree(FIXTURE_DESTINATION) + shutil.copytree(snapshot, FIXTURE_DESTINATION) + + _set_github_outputs(source_commit, fixture_count, changed=changed) + print(f"Checked FFE fixtures from DataDog/ffe-system-test-data@{source_commit}") + print(f"Loaded {fixture_count} canonical fixture cases") + print(f"Fixture snapshot changed: {str(changed).lower()}") + return changed + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ref", default="main", help="Branch, tag, or commit from DataDog/ffe-system-test-data") + arguments = parser.parse_args() + update_fixtures(arguments.ref) + + +if __name__ == "__main__": + main() From 9e3ad6aae646be3e195754f06d546c24d1dbd119 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 21 Oct 2026 20:14:26 +0000 Subject: [PATCH 2/7] test(ffe): preserve evaluation details in parametric adapters Expose Node.js error codes and Ruby resolution details to canonical assertions. Convert Ruby numeric defaults to Float before calling the OpenFeature client. Environment: Datadog workspace --- .../build/docker/nodejs/parametric/server.js | 1 + utils/build/docker/ruby/parametric/server.rb | 24 ++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/utils/build/docker/nodejs/parametric/server.js b/utils/build/docker/nodejs/parametric/server.js index 78024c9231f..32b021b3e80 100644 --- a/utils/build/docker/nodejs/parametric/server.js +++ b/utils/build/docker/nodejs/parametric/server.js @@ -650,6 +650,7 @@ app.post('/ffe/evaluate', async (req, res) => { res.json({ value: details.value, reason: details.reason, + errorCode: details.errorCode, flagMetadata: details.flagMetadata ?? null }); }); diff --git a/utils/build/docker/ruby/parametric/server.rb b/utils/build/docker/ruby/parametric/server.rb index cf5fed16bb2..0461034fd50 100644 --- a/utils/build/docker/ruby/parametric/server.rb +++ b/utils/build/docker/ruby/parametric/server.rb @@ -869,20 +869,26 @@ def handle_ffe_evaluation(req, res) options = { flag_key: args.flag, default_value: args.default_value, evaluation_context: context } + options[:default_value] = args.default_value.to_f if args.variation_type == 'NUMERIC' - value = + details = case args.variation_type - when 'BOOLEAN'then client.fetch_boolean_value(**options) - when 'STRING' then client.fetch_string_value(**options) - when 'INTEGER' then client.fetch_integer_value(**options) - when 'NUMERIC' then client.fetch_float_value(**options) - when 'JSON' then client.fetch_object_value(**options) - else 'FATAL_UNEXPECTED_VARIATION_TYPE' + when 'BOOLEAN'then client.fetch_boolean_details(**options) + when 'STRING' then client.fetch_string_details(**options) + when 'INTEGER' then client.fetch_integer_details(**options) + when 'NUMERIC' then client.fetch_float_details(**options) + when 'JSON' then client.fetch_object_details(**options) + else raise ArgumentError, "Unexpected variation type: #{args.variation_type}" end - res.write({value: value, reason: 'DEFAULT'}.to_json) + res.write({ + value: details.value, + reason: details.reason, + errorCode: details.error_code, + flagMetadata: details.flag_metadata + }.to_json) rescue => e - res.write({value: args.default_value, reason: 'ERROR'}.to_json) + res.write({value: args.default_value, reason: 'ERROR', errorCode: 'GENERAL', errorMessage: e.message}.to_json) end end From 1e5e1793f095bbbd97cfdde548b24ea51e0574b9 Mon Sep 17 00:00:00 2001 From: Tyler Potter Date: Mon, 26 Oct 2026 11:20:41 -0600 Subject: [PATCH 3/7] test: honor FFE evaluation case platform skip/include filters (#8159) Co-authored-by: Codex GPT-5 --- .../test_ffe/ffe-system-test-data/SOURCE.md | 2 +- .../test-case-disabled-flag.json | 18 + .../test-case-empty-flag.json | 18 + ...t-case-experimentation-opt-out-do-log.json | 22 ++ ...t-case-experimentation-opt-out-future.json | 146 +++++++++ .../test-case-experimentation-opt-out.json | 96 ++++-- .../test-case-split-features.json | 109 +++++++ .../ffe-system-test-data/ufc-config.json | 308 +++++++++++++++++- .../test_ffe/test_dynamic_evaluation.py | 22 ++ 9 files changed, 706 insertions(+), 35 deletions(-) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md index 7a06e3fc2be..50d004210d7 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -3,7 +3,7 @@ These files are copied from the canonical FFE fixture repository. Canonical source: https://github.com/DataDog/ffe-system-test-data -Source commit: e5a64a8be81e0f435dbb5c950c06d45d4969b505 +Source commit: caa6d1ac3e5fbda0ce57ef3ffff3772901f3f466 Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in ffe-system-test-data first, then refresh this snapshot. diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json index 208e5c96943..2278ff33f64 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json @@ -1,5 +1,11 @@ [ { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], "attributes": { "country": "US", "email": "alice@mycompany.com" @@ -14,6 +20,12 @@ "variationType": "INTEGER" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], "attributes": { "country": "Canada", "email": "bob@example.com" @@ -28,6 +40,12 @@ "variationType": "INTEGER" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], "attributes": { "age": 50 }, diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json index 18ee5381fdc..f2805230ef3 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json @@ -1,5 +1,11 @@ [ { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], "attributes": { "country": "US", "email": "alice@mycompany.com" @@ -14,6 +20,12 @@ "variationType": "STRING" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], "attributes": { "country": "Canada", "email": "bob@example.com" @@ -28,6 +40,12 @@ "variationType": "STRING" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], "attributes": { "age": 50 }, diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json new file mode 100644 index 00000000000..0d2e8d5b2f5 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json @@ -0,0 +1,22 @@ +[ + { + "description": "ordinary allocation remains eligible when activation equals cutoff", + "flag": "no-log-flag", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": true, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": true, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": true, "reason": "STATIC" } + } + }, + { + "description": "ordinary allocation remains eligible when activation is after cutoff", + "flag": "no-log-flag", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T11:59:59Z" }, + "result": { "value": true, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": true, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": true, "reason": "STATIC" } + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json new file mode 100644 index 00000000000..ea33f207067 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json @@ -0,0 +1,146 @@ +[ + { + "description": "older allocation continues", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T12:00:00.000001Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "description": "equal activation is excluded", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T12:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "newer allocation is excluded", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T11:59:59.999999Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "equal instant with negative offset", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T08:00:00-04:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "equal instant with positive offset", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T14:00:00+02:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "future cutoff is not an evaluation schedule", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "description": "skip newer targeted allocation and continue older randomization", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 3, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "description": "targeted allocation remains eligible before cutoff", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-21T12:00:00Z" }, + "result": { "value": 3, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 3, "reason": "TARGETING_MATCH" } + } + }, + { + "description": "skip both allocations", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-01T12:00:00Z" }, + "result": { "value": 3, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "fractional activation equals cutoff", + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "description": "allocation is one microsecond older", + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123001Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" } + } + }, + { + "description": "allocation is one microsecond newer", + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.122999Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "description": "missing activation is not treated as older", + "flag": "kill-switch", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "alice", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": true, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": false, "reason": "DEFAULT" } + } + }, + { + "description": "null activation is not treated as older", + "flag": "numeric_flag", "variationType": "NUMERIC", "defaultValue": 0, "targetingKey": "alice", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 3.1415926, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "activation is not startAt or document createdAt", + "flag": "start-and-end-date-test", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "current", "reason": "DEFAULT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "DEFAULT" } + } + }, + { + "description": "old activation does not override endAt or startAt", + "flag": "start-and-end-date-test", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-21T12:00:00Z" }, + "result": { "value": "current", "reason": "DEFAULT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": "current", "reason": "DEFAULT" } + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json index 8120c5fb73d..69ca8ccaf01 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json @@ -4,15 +4,23 @@ "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, "result": { "value": "default", "reason": "ERROR", "errorCode": "FLAG_NOT_FOUND" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports GENERAL when the targeting key is missing." + } + ], "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": null, "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": {}, "reason": "ERROR", "errorCode": "TARGETING_KEY_MISSING" }, "capabilities": { - "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } } }, { @@ -20,7 +28,8 @@ "attributes": { "country": "UK" }, "result": { "value": "red", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "red", "reason": "TARGETING_MATCH" } + "experimentationOptOutImmediate": { "value": "red", "reason": "TARGETING_MATCH" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "red", "reason": "TARGETING_MATCH" } } }, { @@ -28,7 +37,8 @@ "attributes": { "country": "UK", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": "red", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "red", "reason": "TARGETING_MATCH" } } }, { @@ -36,7 +46,8 @@ "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": "green", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "green", "reason": "TARGETING_MATCH" } + "experimentationOptOutImmediate": { "value": "green", "reason": "TARGETING_MATCH" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "green", "reason": "TARGETING_MATCH" } } }, { @@ -44,7 +55,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } } }, { @@ -52,7 +64,8 @@ "attributes": {}, "result": { "value": 2, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 2, "reason": "SPLIT" } + "experimentationOptOutImmediate": { "value": 2, "reason": "SPLIT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 2, "reason": "SPLIT" } } }, { @@ -60,7 +73,8 @@ "attributes": {}, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 1, "reason": "SPLIT" } + "experimentationOptOutImmediate": { "value": 1, "reason": "SPLIT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -68,7 +82,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": 2, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 2, "reason": "SPLIT" } } }, { @@ -76,7 +91,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -84,7 +100,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": 3.1415926, "reason": "STATIC" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } } }, { @@ -92,7 +109,8 @@ "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": true, "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": false, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": false, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": false, "reason": "DEFAULT" } } }, { @@ -100,7 +118,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123456789Z" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -108,7 +127,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T08:00:00-04:00" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -116,7 +136,8 @@ "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } } }, { @@ -124,7 +145,8 @@ "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -132,7 +154,8 @@ "attributes": { "dd_experimentation_opt_out_at": "" }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -140,7 +163,8 @@ "attributes": { "dd_experimentation_opt_out_at": true }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -148,7 +172,8 @@ "attributes": { "dd_experimentation_opt_out_at": null }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -156,7 +181,8 @@ "attributes": { "dd_experimentation_opt_out_at": 1792411200 }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -164,7 +190,8 @@ "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "not-a-timestamp" }, "result": { "value": "green", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -172,7 +199,8 @@ "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": false }, "result": { "value": "green", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -180,7 +208,8 @@ "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": null }, "result": { "value": "green", "reason": "TARGETING_MATCH" }, "capabilities": { - "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { @@ -188,23 +217,38 @@ "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": {}, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": {}, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": { }, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { + "_skip": [ + { + "platform": "python", + "reason": "Python rejects object opt-out attributes before flag evaluation." + } + ], "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", "attributes": { "dd_experimentation_opt_out_at": { "timestamp": "2026-10-19T12:00:00Z" } }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } }, { + "_skip": [ + { + "platform": "python", + "reason": "Python rejects array opt-out attributes before flag evaluation." + } + ], "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", "attributes": { "dd_experimentation_opt_out_at": ["2026-10-19T12:00:00Z"] }, "result": { "value": 1, "reason": "SPLIT" }, "capabilities": { - "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } } } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json new file mode 100644 index 00000000000..05b3348f630 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json @@ -0,0 +1,109 @@ +[ + { + "flag": "split-features-destinations", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 7, + "doLog": true, + "features": { + "HOOK": { + "holdout.key": "holdout-q3", + "holdout.assignment_group": "status_quo", + "shared": true, + "empty": "" + }, + "EXPOSURE": { + "holdout.assignment_group": "status_quo", + "exposure.only": false, + "shared": true + }, + "EVALUATION": { + "evaluation.only": 1.25, + "shared": true, + "zero": 0 + } + } + } + }, + { + "flag": "split-features-first-valid-key", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 7, + "doLog": true, + "features": { + "HOOK": { + "invalid-first": "accepted", + "mixed": "kept" + }, + "EXPOSURE": { + "first-wins": "first", + "empty-destinations": false + }, + "EVALUATION": {} + } + } + }, + { + "flag": "split-features-selected-allocation", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": { + "tier": "vip" + }, + "result": { + "value": true, + "reason": "TARGETING_MATCH", + "serialId": 0, + "doLog": true, + "features": { + "HOOK": { + "selected": "vip" + }, + "EXPOSURE": { + "selected": "vip" + }, + "EVALUATION": { + "selected": "vip" + } + } + } + }, + { + "flag": "split-features-selected-allocation", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": { + "tier": "standard" + }, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 7, + "doLog": false, + "features": { + "HOOK": { + "selected": "fallback" + }, + "EXPOSURE": { + "selected": "fallback" + }, + "EVALUATION": { + "selected": "fallback" + } + } + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json index a436b889b15..677e719c1b5 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json @@ -65,7 +65,8 @@ "shards": [] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": null } ] }, @@ -1016,7 +1017,8 @@ } ], "endAt": "2002-10-31T09:00:00.594Z", - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2000-01-01T00:00:00Z" }, { "key": "future-versions", @@ -1027,7 +1029,8 @@ } ], "startAt": "2052-10-31T09:00:00.594Z", - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-01T12:00:00Z" }, { "key": "current-versions", @@ -1039,7 +1042,8 @@ ], "startAt": "2022-10-31T09:00:00.594Z", "endAt": "2050-10-31T09:00:00.594Z", - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-20T12:00:00Z" } ] }, @@ -1278,7 +1282,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-01T12:00:00Z" }, { "key": "rollout", @@ -1384,7 +1389,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-20T12:00:00Z" }, { "key": "50/50 split", @@ -1421,7 +1427,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-01T12:00:00Z" } ] }, @@ -1644,7 +1651,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-19T12:00:00.123Z" }, { "key": "allocation-default", @@ -5341,6 +5349,7 @@ "allocations": [ { "key": "no-log-allocation", + "randomizationActivatedAt": "2026-10-19T12:00:00Z", "rules": [], "splits": [ { @@ -5402,6 +5411,289 @@ "doLog": true } ] + }, + "split-features-destinations": { + "key": "split-features-destinations", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + } + }, + "allocations": [ + { + "key": "allocation", + "doLog": true, + "splits": [ + { + "variationKey": "on", + "serialId": 7, + "shards": [], + "features": [ + { + "key": "holdout.key", + "value": "holdout-q3", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK", + "EXPOSURE" + ] + }, + { + "key": "exposure.only", + "value": false, + "destinations": [ + "EXPOSURE" + ] + }, + { + "key": "evaluation.only", + "value": 1.25, + "destinations": [ + "EVALUATION" + ] + }, + { + "key": "shared", + "value": true, + "destinations": [ + "HOOK", + "EXPOSURE", + "EVALUATION", + "HOOK", + "FUTURE" + ] + }, + { + "key": "empty", + "value": "", + "destinations": [ + "HOOK" + ] + }, + { + "key": "zero", + "value": 0, + "destinations": [ + "EVALUATION" + ] + }, + { + "key": "unknown.only", + "value": true, + "destinations": [ + "FUTURE", + "hook" + ] + } + ] + } + ] + } + ] + }, + "split-features-first-valid-key": { + "key": "split-features-first-valid-key", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + } + }, + "allocations": [ + { + "key": "allocation", + "doLog": true, + "splits": [ + { + "variationKey": "on", + "serialId": 7, + "shards": [], + "features": [ + { + "key": "invalid-first", + "value": {}, + "destinations": [ + "HOOK" + ] + }, + { + "key": "invalid-first", + "value": "accepted", + "destinations": [ + "HOOK" + ] + }, + { + "key": "invalid-first", + "value": "later", + "destinations": [ + "EXPOSURE" + ] + }, + { + "key": "first-wins", + "value": "first", + "destinations": [ + "EXPOSURE" + ] + }, + { + "key": "first-wins", + "value": "later", + "destinations": [ + "HOOK", + "EVALUATION" + ] + }, + { + "key": "unknown-first", + "value": "first", + "destinations": [ + "FUTURE" + ] + }, + { + "key": "unknown-first", + "value": "later", + "destinations": [ + "HOOK" + ] + }, + { + "key": "empty-destinations", + "value": "invalid", + "destinations": [] + }, + { + "key": "empty-destinations", + "value": false, + "destinations": [ + "EXPOSURE" + ] + }, + null, + { + "key": "array", + "value": [], + "destinations": [ + "HOOK" + ] + }, + { + "key": "null", + "value": null, + "destinations": [ + "HOOK" + ] + }, + { + "key": "", + "value": true, + "destinations": [ + "HOOK" + ] + }, + { + "key": "mixed", + "value": "kept", + "destinations": [ + "HOOK", + null, + 42, + "" + ] + }, + { + "key": "bad-destinations", + "value": true, + "destinations": "HOOK" + } + ] + } + ] + } + ] + }, + "split-features-selected-allocation": { + "key": "split-features-selected-allocation", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + } + }, + "allocations": [ + { + "key": "vip", + "doLog": true, + "rules": [ + { + "conditions": [ + { + "attribute": "tier", + "operator": "ONE_OF", + "value": [ + "vip" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "on", + "serialId": 0, + "shards": [], + "features": [ + { + "key": "selected", + "value": "vip", + "destinations": [ + "HOOK", + "EXPOSURE", + "EVALUATION" + ] + } + ] + } + ] + }, + { + "key": "allocation", + "doLog": false, + "splits": [ + { + "variationKey": "on", + "serialId": 7, + "shards": [], + "features": [ + { + "key": "selected", + "value": "fallback", + "destinations": [ + "HOOK", + "EXPOSURE", + "EVALUATION" + ] + } + ] + } + ] + } + ] } } } diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 642a73d56d0..e2aaf50d10b 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -7,6 +7,7 @@ from typing import Any from utils import ( + context, features, scenarios, ) @@ -41,6 +42,22 @@ def _get_test_case_files() -> list[str]: return case_files +def _filter_test_cases_for_platform(test_cases: list[dict[str, Any]], platform: str) -> list[dict[str, Any]]: + """Apply the canonical corpus _skip/_include selection rules.""" + applicable_cases = [] + for test_case in test_cases: + if "_skip" in test_case and "_include" in test_case: + raise ValueError("Evaluation cases cannot define both _skip and _include") + skipped_platforms = {entry["platform"] for entry in test_case.get("_skip", [])} + included_platforms = {entry["platform"] for entry in test_case.get("_include", [])} + if platform in skipped_platforms: + continue + if included_platforms and platform not in included_platforms: + continue + applicable_cases.append(test_case) + return applicable_cases + + # Load fixture at module level for reuse across tests UFC_FIXTURE_DATA = _load_ufc_fixture() ALL_TEST_CASE_FILES = _get_test_case_files() @@ -162,6 +179,11 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file}" assert test_cases, f"No evaluation cases in {test_case_file}" + platform = context.library.name + test_cases = _filter_test_cases_for_platform(test_cases, platform) + if not test_cases: + pytest.skip(f"No cases in {test_case_file} apply to platform {platform}") + # Set up UFC Remote Config and wait for it to be applied _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) From 51867b13eca51097c73d418ee3b15b4b328d9101 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 26 Oct 2026 18:28:46 +0000 Subject: [PATCH 4/7] Load the Java development OpenFeature provider alongside the agent Download dd-openfeature.jar from the selected Java snapshot branch so FFE tests do not pair the development agent with an incompatible release provider. Cover default and custom branches, Java Lambda, and missing providers. Environment: Datadog workspace --- tests/test_the_test/test_load_binary.py | 37 +++++++++++++++++++++++-- utils/scripts/load-binary.sh | 2 ++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/test_the_test/test_load_binary.py b/tests/test_the_test/test_load_binary.py index 149f2f47bb3..3bad919cb9c 100644 --- a/tests/test_the_test/test_load_binary.py +++ b/tests/test_the_test/test_load_binary.py @@ -4,7 +4,7 @@ from pathlib import Path import subprocess -from utils import scenarios +from utils import pytest, scenarios SCRIPT = Path("utils/scripts/load-binary.sh") @@ -23,6 +23,7 @@ def _run_loader( tmp_path: Path, version: str, *, + target: str = "c", extra_env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: bin_dir = tmp_path / "bin" @@ -39,6 +40,14 @@ def _run_loader( if [[ "${{MISSING_BRANCH:-}}" != "" && "$url" == *"${{MISSING_BRANCH}}"* ]]; then exit 22 fi +previous= +for argument in "$@"; do + if [[ "$previous" == "--output" ]]; then + printf '%s\\n' "$url" > "$argument" + exit 0 + fi + previous="$argument" +done if [[ "$url" == *"DataDog/dd-trace-c"* ]]; then printf '%s\\n' '{{"commit":{{"sha":"{C_LIBRARY_SHA}"}}}}' else @@ -68,7 +77,7 @@ def _run_loader( env.pop("AUTO_INJECT_TARGET_BRANCH", None) env.update(extra_env or {}) return subprocess.run( - ["bash", str(SCRIPT), "c", version], + ["bash", str(SCRIPT), target, version], check=False, capture_output=True, text=True, @@ -158,3 +167,27 @@ def test_missing_package_fails_with_clear_error(self, tmp_path: Path) -> None: assert result.returncode != 0 assert "OCI package does not exist or is not accessible" in result.stderr + + +@scenarios.test_the_test +class Test_LoadBinaryJava: + @pytest.mark.parametrize("target", ["java", "java_lambda"]) + @pytest.mark.parametrize("branch", [None, "feature/flagging"]) + def test_development_agent_and_provider_use_same_branch( + self, tmp_path: Path, target: str, branch: str | None + ) -> None: + extra_env = {"LIBRARY_TARGET_BRANCH": branch} if branch else {} + result = _run_loader(tmp_path, "dev", target=target, extra_env=extra_env) + + assert result.returncode == 0, result.stderr + base_url = f"https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/{branch or 'master'}" + for artifact in ("dd-java-agent.jar", "dd-openfeature.jar"): + assert (tmp_path / "binaries" / artifact).read_text(encoding="utf-8").strip() == f"{base_url}/{artifact}" + + @pytest.mark.parametrize("target", ["java", "java_lambda"]) + def test_missing_development_provider_fails(self, tmp_path: Path, target: str) -> None: + result = _run_loader(tmp_path, "dev", target=target, extra_env={"MISSING_BRANCH": "dd-openfeature.jar"}) + + assert result.returncode != 0 + assert (tmp_path / "binaries/dd-java-agent.jar").exists() + assert not (tmp_path / "binaries/dd-openfeature.jar").exists() diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index 378da8b92ea..7bcfe172bb2 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -205,6 +205,8 @@ elif [ "$TARGET" = "java" ] || [ "$TARGET" = "java_lambda" ]; then LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-master}" curl --fail --location --silent --show-error --output dd-java-agent.jar "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${LIBRARY_TARGET_BRANCH}/dd-java-agent.jar" + # The development provider must use the same feature-flagging API as the agent. + curl --fail --location --silent --show-error --output dd-openfeature.jar "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${LIBRARY_TARGET_BRANCH}/dd-openfeature.jar" elif [ "$TARGET" = "dotnet" ]; then assert_version_is_dev From 88bdb421327c772735c2c3e09fa163eeef9aa523 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 27 Oct 2026 21:36:20 +0000 Subject: [PATCH 5/7] Assert canonical split-feature metadata at hook and intake boundaries --- manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/php.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + tests/ffe/README.md | 23 +++-- .../test_ffe/test_dynamic_evaluation.py | 83 +++++++++++++++++++ tests/parametric/test_ffe/utils.py | 36 ++++++++ .../test_the_test/test_ffe_split_features.py | 72 ++++++++++++++++ .../FeatureFlagEvaluatorController.java | 35 +++++++- utils/docker_fixtures/_test_agent.py | 28 +++++++ 13 files changed, 277 insertions(+), 7 deletions(-) create mode 100644 tests/test_the_test/test_ffe_split_features.py diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index f68f3bf8bd1..8c4564a811f 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1200,6 +1200,7 @@ manifest: component_version: <3.55.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=3.54.0' # TODO: a lower version might be supported diff --git a/manifests/golang.yml b/manifests/golang.yml index bd5a9473a42..6f60ea2231c 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -2175,6 +2175,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (DD_TRACE_PROPAGATION_EXTRACT_FIRST=true is unsupported; dd-trace-go panics when extraction fails) tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev # Easy win for all weblogs and version 2.10.1 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=2.10.0' # TODO: a lower version might be supported diff --git a/manifests/java.yml b/manifests/java.yml index 6ac486606c3..2aeb7f950b3 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4240,6 +4240,7 @@ manifest: : v1.65.0 tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0 + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: v1.69.0-SNAPSHOT tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: v1.66.0 tests/parametric/test_ffe/test_span_enrichment.py: '>=1.64.2' # TODO: a lower version might be supported diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 37d4f01699d..8d5d5ac2347 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2673,6 +2673,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: *ref_5_56_0 tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0 + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: 7.0.0-pre # Exact development version until the first containing release is confirmed. tests/parametric/test_ffe/test_serial_id_metadata.py: v6.18.0 tests/parametric/test_ffe/test_span_enrichment.py: # TODO: a lower version might be supported diff --git a/manifests/php.yml b/manifests/php.yml index 14bbc2f2b11..732e648a487 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1627,6 +1627,7 @@ manifest: : missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=1.23.3' # TODO: a lower version might be supported diff --git a/manifests/python.yml b/manifests/python.yml index fd11d834e4a..e266705a8ff 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -3074,6 +3074,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: v2.20.0 tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0 + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=4.9.0' # TODO: a lower version might be supported diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 7ad348dad55..c0117382cb6 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2694,6 +2694,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_ffe/test_configuration_sources.py: v2.46.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0 + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: missing_feature diff --git a/tests/ffe/README.md b/tests/ffe/README.md index 9e807f443ba..d440ec62eea 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -34,12 +34,23 @@ leaves any existing fixture update PR untouched. Review and resolve newly expose SDK failures before merging an update. Dynamic evaluation checks each case's value, reason, and explicit error code. -It does not yet assert `result.features` expectations for hook, exposure, or -evaluation metadata. The imported split-feature cases therefore validate only -their baseline evaluation result. Follow-up coverage must expose those three -metadata channels through the parametric harness and assert their canonical -expectations, including false, zero, and empty-string values; tracked in -[the review follow-up](https://github.com/DataDog/system-tests/pull/8139#discussion_r4382943374). +`Test_Feature_Flag_Split_Features` additionally checks every canonical case with +`result.features` against the public exposure-hook callback and actual exposure +and evaluation payloads received by the parametric test agent. Exact maps detect +missing values, extra keys, destination leaks, and boolean/number coercion while +preserving JSON numeric equivalence. This covers false, zero, empty strings, +first-valid-key selection, and selected-allocation isolation. For `doLog=false`, +selected metadata is checked separately and exposure delivery must stay absent. + +The destination test currently runs with Java's `1.69.0-SNAPSHOT` SDK and later; +other clients are marked `incomplete_test_app` until they capture their public +hook output. Their baseline evaluation cases continue to run. The Java harness +loads the optional hook API only when this test enables capture, so older SDKs +can still compile and run baseline tests. Unsupported clients skip this slower +delivery test before container setup. An opt-in test-agent app factory registers +the FFE intake routes missing from the pinned agent image and records their +original payloads. This proves local delivery, not Datadog intake or production. + Cases with optional `capabilities` still run their baseline `result`; enabling capability-specific provider options requires separate harness support. Remote Configuration tests and the mocked agentless backend read the same UFC. diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index e2aaf50d10b..6266cdc69cc 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -10,10 +10,12 @@ context, features, scenarios, + slow, ) from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary +from tests.parametric.test_ffe.utils import assert_feature_map, ffe_events_from_requests RC_PRODUCT = "FFE_FLAGS" RC_PATH = f"datadog/2/{RC_PRODUCT}" @@ -61,6 +63,12 @@ def _filter_test_cases_for_platform(test_cases: list[dict[str, Any]], platform: # Load fixture at module level for reuse across tests UFC_FIXTURE_DATA = _load_ufc_fixture() ALL_TEST_CASE_FILES = _get_test_case_files() +SPLIT_FEATURE_CASES = [ + case + for filename in ALL_TEST_CASE_FILES + for case in json.loads((FFE_EVALUATION_CASES_DIR / filename).read_text()) + if "features" in case["result"] +] DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", @@ -231,3 +239,78 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI assert result.get("errorCode") in {None, ""}, ( f"Test case {i} in {test_case_file}: unexpected evaluation error; result={result}" ) + + +@scenarios.parametric +@features.feature_flags_dynamic_evaluation +@slow +class Test_Feature_Flag_Split_Features: + """Check canonical destination maps at the public hook and EVP boundaries.""" + + @parametrize( + "library_env", + [ + { + **DEFAULT_ENVVARS, + "SYSTEM_TESTS_FFE_CAPTURE_HOOK_FEATURES": "true", + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": "true", + "DD_FEATURE_FLAGS_EXPOSURES_DATADOG_LOGGING_ENABLED": "true", + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "true", + } + ], + ) + @parametrize("agent_env", [{"SYSTEM_TESTS_FFE_INTAKE": "true"}]) + @parametrize("case", SPLIT_FEATURE_CASES) + def test_split_feature_destinations( + self, case: dict[str, Any], test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + if not _filter_test_cases_for_platform([case], context.library.name): + pytest.skip(f"Case does not apply to {context.library.name}") + expected = case["result"] + _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) + assert test_library.ffe_start(UFC_FIXTURE_DATA), "Failed to start FFE provider" + result = _ffe_evaluate_with_rc_retry( + test_library, + flag=case["flag"], + variation_type=case["variationType"], + default_value=case["defaultValue"], + targeting_key=case["targetingKey"], + attributes=case.get("attributes", {}), + ) + assert result.get("value") == expected["value"], result + assert result.get("reason") == expected["reason"], result + assert result.get("errorCode") in (None, ""), result + assert_feature_map(result.get("hookFeatures"), expected["features"]["HOOK"], "HOOK callback") + + # Selected metadata remains available even when doLog=false suppresses delivery. + # Also check the actual payloads below: metadata alone cannot prove isolation. + metadata = result.get("flagMetadata") + assert isinstance(metadata, dict), result + for destination, prefix in ( + ("EXPOSURE", "__dd_exposure_feature."), + ("EVALUATION", "__dd_evaluation_feature."), + ): + selected = {key.removeprefix(prefix): value for key, value in metadata.items() if key.startswith(prefix)} + assert_feature_map(selected, expected["features"][destination], f"{destination} selected metadata") + + # Evaluation batches flush every 10s; continue briefly after a matching batch + # to observe forbidden exposures for the doLog=false case as well. + deadline = time.monotonic() + 30 + received_at = None + while True: + events = ffe_events_from_requests(test_agent.requests(), case["flag"]) + if events["EVALUATION"] and (events["EXPOSURE"] or not expected["doLog"]): + if received_at is None: + received_at = time.monotonic() + if time.monotonic() - received_at >= 2: + break + assert time.monotonic() < deadline, f"Missing split-feature events for {case['flag']}: {events}" + time.sleep(0.2) + + if not expected["doLog"]: + assert not events["EXPOSURE"], f"doLog=false emitted exposures: {events['EXPOSURE']}" + for destination, observed_events in events.items(): + for event in observed_events: + # Empty features are omitted from the wire format, but the event itself + # must exist. An absent event never stands in for an empty feature map. + assert_feature_map(event.get("features", {}), expected["features"][destination], destination) diff --git a/tests/parametric/test_ffe/utils.py b/tests/parametric/test_ffe/utils.py index ad655f9cf84..b82b1c64c0a 100644 --- a/tests/parametric/test_ffe/utils.py +++ b/tests/parametric/test_ffe/utils.py @@ -9,7 +9,9 @@ Example: [100, 108, 128, 130] -> deltas [100, 8, 20, 2] -> varint bytes -> base64 """ +from collections.abc import Mapping, Sequence import base64 +import gzip import hashlib import json from pathlib import Path @@ -139,3 +141,37 @@ def hash_targeting_key(targeting_key: str) -> str: """ return hashlib.sha256(targeting_key.encode("utf-8")).hexdigest() + + +def assert_feature_map(actual: object, expected: dict[str, Any], destination: str) -> None: + """Compare exact scalar feature maps without equating false with zero.""" + assert isinstance(actual, dict), f"{destination}: missing feature map: {actual!r}" + assert actual.keys() == expected.keys(), f"{destination}: expected {expected!r}, got {actual!r}" + for key, expected_value in expected.items(): + actual_value = actual[key] + if type(expected_value) in (int, float): + valid_type = type(actual_value) in (int, float) + else: + valid_type = type(actual_value) is type(expected_value) + assert valid_type, f"{destination}.{key}: expected type {type(expected_value)}, got {type(actual_value)}" + assert actual_value == expected_value, f"{destination}.{key}: expected {expected_value!r}, got {actual_value!r}" + + +def ffe_events_from_requests(requests: Sequence[Mapping[str, Any]], flag: str) -> dict[str, list[dict[str, Any]]]: + """Read real EVP payloads recorded by the parametric test agent.""" + events: dict[str, list[dict[str, Any]]] = {"EXPOSURE": [], "EVALUATION": []} + for request in requests: + path = request["url"].split("?", 1)[0] + if path.endswith("/api/v2/exposures"): + destination, field = "EXPOSURE", "exposures" + elif path.endswith("/api/v2/flagevaluation"): + destination, field = "EVALUATION", "flagEvaluations" + else: + continue + body = base64.b64decode(request["body"]) + # The agent may already have decompressed a request while retaining its headers. + if body.startswith(b"\x1f\x8b"): + body = gzip.decompress(body) + payload = json.loads(body) + events[destination].extend(event for event in payload[field] if event.get("flag", {}).get("key") == flag) + return events diff --git a/tests/test_the_test/test_ffe_split_features.py b/tests/test_the_test/test_ffe_split_features.py new file mode 100644 index 00000000000..3b72b718a52 --- /dev/null +++ b/tests/test_the_test/test_ffe_split_features.py @@ -0,0 +1,72 @@ +"""Regression controls for canonical split-feature assertions and wire decoding.""" + +import base64 +import gzip +import json +from typing import Any + +from utils import pytest, scenarios +from tests.parametric.test_ffe.utils import assert_feature_map, ffe_events_from_requests + + +@scenarios.test_the_test +@pytest.mark.parametrize( + "actual", + [ + None, + {}, + {"zero": 0}, + {"zero": False, "empty": "", "enabled": False}, + {"zero": 0, "empty": "", "enabled": 0}, + {"zero": "0", "empty": "", "enabled": False}, + {"zero": 0, "empty": "changed", "enabled": False}, + {"zero": 0, "empty": "", "enabled": False, "wrong-destination": True}, + ], +) +def test_feature_map_rejects_missing_changed_misrouted_or_coerced_values(actual: object) -> None: + # Value/reason can remain correct while metadata is dropped or misrouted. + with pytest.raises(AssertionError): + assert_feature_map(actual, {"zero": 0, "empty": "", "enabled": False}, "EVALUATION") + + +@scenarios.test_the_test +def test_feature_map_preserves_json_number_equivalence_and_empty_maps() -> None: + assert_feature_map( + {"zero": 0.0, "number": 1.25, "false": False, "empty": ""}, + {"zero": 0, "number": 1.25, "false": False, "empty": ""}, + "HOOK", + ) + assert_feature_map({}, {}, "EVALUATION") + with pytest.raises(AssertionError): + assert_feature_map({"leaked": True}, {}, "EVALUATION") + with pytest.raises(AssertionError): + assert_feature_map(None, {}, "EVALUATION") + + +@scenarios.test_the_test +@pytest.mark.parametrize("encoding", ["identity", "gzip", "decoded-gzip"]) +def test_ffe_wire_events_keep_destination_boundaries_and_empty_features(encoding: str) -> None: + def request(path: str, field: str, events: list[dict[str, Any]]) -> dict[str, Any]: + body = json.dumps({field: events}).encode() + if encoding == "gzip": + body = gzip.compress(body) + return { + "url": f"http://agent/evp_proxy/v4/api/v2/{path}", + "headers": {"Content-Encoding": "gzip"} if encoding != "identity" else {}, + "body": base64.b64encode(body).decode(), + } + + exposure = {"flag": {"key": "flag"}, "features": {"exposure.only": False}} + evaluation = {"flag": {"key": "flag"}, "features": {"zero": 0}} + empty_evaluation = {"flag": {"key": "flag"}} + unrelated = {"flag": {"key": "other"}, "features": {"leaked": True}} + actual = ffe_events_from_requests( + [ + request("exposures", "exposures", [exposure, unrelated]), + request("flagevaluation", "flagEvaluations", [evaluation, empty_evaluation, unrelated]), + {"url": "/v0.7/config", "headers": {}, "body": "not-json"}, + ], + "flag", + ) + assert actual == {"EXPOSURE": [exposure], "EVALUATION": [evaluation, empty_evaluation]} + assert ffe_events_from_requests([], "flag") == {"EXPOSURE": [], "EVALUATION": []} diff --git a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java index b3da4a7f886..3cb7ea35d97 100644 --- a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java +++ b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java @@ -8,6 +8,7 @@ import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.FeatureProvider; import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.Hook; import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.NoOpProvider; import dev.openfeature.sdk.OpenFeatureAPI; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; @RestController @RequestMapping("/ffe") @@ -37,6 +39,28 @@ public class FeatureFlagEvaluatorController { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlagEvaluatorController.class); + private static final boolean CAPTURE_HOOK_FEATURES = + Boolean.parseBoolean(System.getenv("SYSTEM_TESTS_FFE_CAPTURE_HOOK_FEATURES")); + private static final ThreadLocal HOOK_FEATURES = new ThreadLocal<>(); + + private static void captureHookFeatures(Client client) { + // Load the optional public API only for tests whose manifest requires it. + // Older dd-openfeature versions must still build and run the baseline suite. + try { + Class hookClass = Class.forName("com.datadog.openfeature.ExposureHook"); + Consumer callback = evaluation -> { + try { + HOOK_FEATURES.set(evaluation.getClass().getMethod("getFeatures").invoke(evaluation)); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Cannot capture ExposureHook features", e); + } + }; + client.addHooks((Hook) hookClass.getConstructor(Consumer.class).newInstance(callback)); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Split-feature tests require the public ExposureHook API", e); + } + } + @Configuration public static class FeatureFlagEvaluatorConfig { @@ -61,7 +85,11 @@ public ProviderState getState() { }; } api.setProviderAndWait(provider); - return api.getClient(); + Client client = api.getClient(); + if (CAPTURE_HOOK_FEATURES) { + captureHookFeatures(client); + } + return client; } } @@ -81,6 +109,7 @@ public ResponseEntity start() { @PostMapping(value = "/evaluate", consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) public ResponseEntity> evaluate(@RequestBody final EvaluateRequest request) { + HOOK_FEATURES.remove(); Object value; String reason; String errorCode = null; @@ -123,6 +152,10 @@ public ResponseEntity> evaluate(@RequestBody final EvaluateR result.put("errorCode", errorCode); result.put("value", value); result.put("flagMetadata", flagMetadata); + if (CAPTURE_HOOK_FEATURES) { + result.put("hookFeatures", HOOK_FEATURES.get()); + } + HOOK_FEATURES.remove(); return ResponseEntity.ok(result); } diff --git a/utils/docker_fixtures/_test_agent.py b/utils/docker_fixtures/_test_agent.py index 6422405c3f0..ea6f7953be5 100644 --- a/utils/docker_fixtures/_test_agent.py +++ b/utils/docker_fixtures/_test_agent.py @@ -35,6 +35,31 @@ DEFAULT_OTLP_HTTP_PORT = 4318 DEFAULT_OTLP_GRPC_PORT = 4317 +# The pinned test agent only registers FFE exposures on EVP v2. Opt-in tests use +# its supported app factory to register the remaining FFE intake routes with the +# existing recording/200-response handler; no requests are sent to Datadog. +_FFE_INTAKE_COMMAND = """ +from aiohttp import web +from ddapm_test_agent.agent import main, make_app + + +def make_ffe_app(**kwargs): + kwargs["agent_url"] = "" + app = make_app(**kwargs) + agent = app["agent"] + paths = ( + "/evp_proxy/v2/api/v2/flagevaluation", + "/evp_proxy/v4/api/v2/exposures", + "/evp_proxy/v4/api/v2/flagevaluation", + ) + agent._forward_endpoints.extend(paths) + app.add_routes([web.post(path, agent.handle_evp_proxy_v2_api_v2_exposures) for path in paths]) + return app + + +main(app_factory=make_ffe_app) +""" + def _request_token(request: pytest.FixtureRequest) -> str: token = "" @@ -132,6 +157,9 @@ def start_agent( open(log_path, "w+", encoding="utf-8") as log_file, docker_run( image=self.image, + command=["python", "-c", _FFE_INTAKE_COMMAND] + if agent_env.get("SYSTEM_TESTS_FFE_INTAKE") == "true" + else None, name=container_name, env=env, volumes={ From c7587a8347f5471838f0dc1532a753ea4d8a73b8 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 27 Oct 2026 22:14:55 +0000 Subject: [PATCH 6/7] Gate split-feature capture for unsupported C++ and Rust clients --- manifests/cpp.yml | 1 + manifests/rust.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/manifests/cpp.yml b/manifests/cpp.yml index 142c480f840..16253f9c859 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -191,6 +191,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: missing_feature (Automatic baggage propagation is not implemented) tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (cpp is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=2.1.1' # TODO: a lower version might be supported diff --git a/manifests/rust.yml b/manifests/rust.yml index 5a0ba272912..880b15a3a7a 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -274,6 +274,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (rust is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported From 4a9a8680943efa43652fe71d5a08d84fa2d8e7b1 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 27 Oct 2026 22:22:27 +0000 Subject: [PATCH 7/7] Account for Rust 0.5.4 OpenTelemetry backport in test activation The 0.5.4 release moved back to OpenTelemetry 0.32, which lacks the compression fallback and scope schema URL behavior present in 0.33. Preserve the existing minimum versions and declare only the released 0.5.4 unsupported; keep the 0.5.4 development build enabled. Environment: Datadog workspace --- manifests/rust.yml | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/manifests/rust.yml b/manifests/rust.yml index 880b15a3a7a..5a7bfc948f5 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -84,17 +84,25 @@ manifest: tests/parametric/otel_env_vars/test_otel_blrp_max_queue_size.py::Test_OTEL_BLRP_MAX_QUEUE_SIZE: missing_feature tests/parametric/otel_env_vars/test_otel_blrp_schedule_delay.py::Test_OTEL_BLRP_SCHEDULE_DELAY: missing_feature tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION: '>=0.5.3-dev+2ea3692' # TODO: a lower version might be supported - tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION::test_empty: '>=0.5.3-dev+5da7ebc' # TODO: a lower version might be supported - tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION::test_invalid: '>=0.5.3-dev+5da7ebc' # TODO: a lower version might be supported + # Rust 0.5.4 backported OpenTelemetry to 0.32: https://github.com/DataDog/dd-trace-rs/pull/343 + tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION::test_empty: + - declaration: missing_feature (OpenTelemetry 0.32 rejects empty or invalid OTLP compression values instead of using the default) + component_version: <0.5.3-dev+5da7ebc || =0.5.4 + tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION::test_invalid: + - declaration: missing_feature (OpenTelemetry 0.32 rejects empty or invalid OTLP compression values instead of using the default) + component_version: <0.5.3-dev+5da7ebc || =0.5.4 tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION::test_stable_values: missing_feature tests/parametric/otel_env_vars/test_otel_exporter_otlp_endpoint.py::Test_OTEL_EXPORTER_OTLP_ENDPOINT: v0.3.0 tests/parametric/otel_env_vars/test_otel_exporter_otlp_headers.py::Test_OTEL_EXPORTER_OTLP_HEADERS: v0.3.0 tests/parametric/otel_env_vars/test_otel_exporter_otlp_logs_compression.py::Test_OTEL_EXPORTER_OTLP_LOGS_COMPRESSION: '>=0.5.3-dev+2ea3692' # TODO: a lower version might be supported - tests/parametric/otel_env_vars/test_otel_exporter_otlp_logs_compression.py::Test_OTEL_EXPORTER_OTLP_LOGS_COMPRESSION::test_empty: '>=0.5.3-dev+5da7ebc' # TODO: a lower version might be supported + tests/parametric/otel_env_vars/test_otel_exporter_otlp_logs_compression.py::Test_OTEL_EXPORTER_OTLP_LOGS_COMPRESSION::test_empty: + - declaration: missing_feature (OpenTelemetry 0.32 rejects empty or invalid OTLP compression values instead of using the default) + component_version: <0.5.3-dev+5da7ebc || =0.5.4 ? tests/parametric/otel_env_vars/test_otel_exporter_otlp_logs_compression.py::Test_OTEL_EXPORTER_OTLP_LOGS_COMPRESSION::test_empty_inherits_general : missing_feature ? tests/parametric/otel_env_vars/test_otel_exporter_otlp_logs_compression.py::Test_OTEL_EXPORTER_OTLP_LOGS_COMPRESSION::test_invalid - : '>=0.5.3-dev+5da7ebc' # TODO: a lower version might be supported + : - declaration: missing_feature (OpenTelemetry 0.32 rejects empty or invalid OTLP compression values instead of using the default) + component_version: <0.5.3-dev+5da7ebc || =0.5.4 ? tests/parametric/otel_env_vars/test_otel_exporter_otlp_logs_compression.py::Test_OTEL_EXPORTER_OTLP_LOGS_COMPRESSION::test_signal_precedence : missing_feature ? tests/parametric/otel_env_vars/test_otel_exporter_otlp_logs_compression.py::Test_OTEL_EXPORTER_OTLP_LOGS_COMPRESSION::test_stable_values @@ -112,11 +120,13 @@ manifest: : irrelevant (Datadog uses a sufficiently long 10000 ms timeout for zero) tests/parametric/otel_env_vars/test_otel_exporter_otlp_metrics_compression.py::Test_OTEL_EXPORTER_OTLP_METRICS_COMPRESSION: '>=0.5.3-dev+2ea3692' # TODO: a lower version might be supported ? tests/parametric/otel_env_vars/test_otel_exporter_otlp_metrics_compression.py::Test_OTEL_EXPORTER_OTLP_METRICS_COMPRESSION::test_empty - : '>=0.5.3-dev+5da7ebc' # TODO: a lower version might be supported + : - declaration: missing_feature (OpenTelemetry 0.32 rejects empty or invalid OTLP compression values instead of using the default) + component_version: <0.5.3-dev+5da7ebc || =0.5.4 ? tests/parametric/otel_env_vars/test_otel_exporter_otlp_metrics_compression.py::Test_OTEL_EXPORTER_OTLP_METRICS_COMPRESSION::test_empty_inherits_general : missing_feature ? tests/parametric/otel_env_vars/test_otel_exporter_otlp_metrics_compression.py::Test_OTEL_EXPORTER_OTLP_METRICS_COMPRESSION::test_invalid - : '>=0.5.3-dev+5da7ebc' # TODO: a lower version might be supported + : - declaration: missing_feature (OpenTelemetry 0.32 rejects empty or invalid OTLP compression values instead of using the default) + component_version: <0.5.3-dev+5da7ebc || =0.5.4 ? tests/parametric/otel_env_vars/test_otel_exporter_otlp_metrics_compression.py::Test_OTEL_EXPORTER_OTLP_METRICS_COMPRESSION::test_signal_precedence : missing_feature ? tests/parametric/otel_env_vars/test_otel_exporter_otlp_metrics_compression.py::Test_OTEL_EXPORTER_OTLP_METRICS_COMPRESSION::test_stable_values @@ -399,7 +409,7 @@ manifest: component_version: <0.5.3-dev+671276b tests/parametric/test_otel_logs.py::Test_FR13_Scope_Fields::test_scope_schema_url_field: # TODO: a lower version might be supported - declaration: missing_feature (OpenTelemetry Rust SDK limitation - schema_url not exported by opentelemetry-otlp) - component_version: <0.5.3-dev+5da7ebc + component_version: <0.5.3-dev+5da7ebc || =0.5.4 tests/parametric/test_otel_logs.py::Test_FR13_Scope_Fields::test_scope_version_field: # TODO: a lower version might be supported - declaration: missing_feature (OpenTelemetry Rust SDK limitation - scope version not exported by opentelemetry-otlp) component_version: <0.5.3-dev+671276b