diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3956402af3e..f0c83032c8c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,6 +12,9 @@ /tests/ @DataDog/system-tests-reviewers # FFE +/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core +/.github/workflows/update-ffe-fixtures.yml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core +/utils/scripts/update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/mocked_backend/_backend_ffe.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/build/docker/dotnet/parametric/Endpoints/FfeTestApi.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core @@ -24,6 +27,7 @@ /tests/ffe/ @DataDog/feature-flagging-and-experimentation-sdk /tests/test_the_test/test_ffe_tests.py @DataDog/feature-flagging-and-experimentation-sdk /tests/test_the_test/test_mock_ffe_agentless_backend.py @DataDog/feature-flagging-and-experimentation-sdk +/tests/test_the_test/test_update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk /tests/parametric/test_ffe/ @DataDog/feature-flagging-and-experimentation-sdk # SDK cap diff --git a/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml b/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml new file mode 100644 index 00000000000..7aeba37ded2 --- /dev/null +++ b/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml @@ -0,0 +1,14 @@ +--- +issuer: https://token.actions.githubusercontent.com + +subject: repo:DataDog/system-tests:ref:refs/heads/main + +claim_pattern: + event_name: (workflow_dispatch|schedule) + ref: refs/heads/main + ref_protected: "true" + job_workflow_ref: DataDog/system-tests/\.github/workflows/update-ffe-fixtures\.yml@refs/heads/main + +permissions: + contents: write + pull_requests: write diff --git a/.github/workflows/update-ffe-fixtures.yml b/.github/workflows/update-ffe-fixtures.yml new file mode 100644 index 00000000000..778db406a4b --- /dev/null +++ b/.github/workflows/update-ffe-fixtures.yml @@ -0,0 +1,87 @@ +--- +name: Update FFE fixtures + +on: + workflow_dispatch: + inputs: + fixture_ref: + description: Branch, tag, or commit from DataDog/ffe-system-test-data + required: true + default: main + type: string + schedule: + - cron: "0 0 * * 0" + +concurrency: + group: update-ffe-fixtures + cancel-in-progress: false + +jobs: + update-ffe-fixtures: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + persist-credentials: false + + - name: Update fixture snapshot + id: fixtures + env: + FFE_FIXTURE_REF: ${{ inputs.fixture_ref || 'main' }} + run: python3 utils/scripts/update_ffe_fixtures.py --ref "$FFE_FIXTURE_REF" + + - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4 + if: steps.fixtures.outputs.changed == 'true' + id: octo-sts + with: + scope: DataDog/system-tests + policy: self.update-ffe-fixtures.create-pr + + - name: Check for an open fixture update PR + if: steps.fixtures.outputs.changed == 'true' + id: existing-pr + env: + GH_TOKEN: ${{ steps.octo-sts.outputs.token }} + run: | + count=$(gh pr list --repo "$GITHUB_REPOSITORY" \ + --head actions/update-ffe-fixtures --state open \ + --json headRefName,headRepositoryOwner,isCrossRepository \ + --jq '[.[] | select( + .headRefName == "actions/update-ffe-fixtures" and + .headRepositoryOwner.login == "DataDog" and + (.isCrossRepository | not) + )] | length') + echo "count=$count" >> "$GITHUB_OUTPUT" + + - name: Create Pull Request + if: >- + steps.fixtures.outputs.changed == 'true' && steps.existing-pr.outputs.count == '0' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.octo-sts.outputs.token }} + branch: actions/update-ffe-fixtures + base: main + commit-message: "test(ffe): update canonical fixtures" + title: "test(ffe): update canonical fixtures" + draft: true + sign-commits: true + add-paths: tests/parametric/test_ffe/ffe-system-test-data + body: | + ## Motivation + + Keep SDK behavior consistent with the shared FFE evaluator contract. + + ## Changes + + Refresh the checked-in snapshot to [${{ steps.fixtures.outputs.source_commit }}](https://github.com/DataDog/ffe-system-test-data/commit/${{ steps.fixtures.outputs.source_commit }}), containing ${{ steps.fixtures.outputs.fixture_count }} evaluation cases. + + Ordinary test runs use the snapshot without network access. Resolve newly exposed evaluator failures before merging; correct shared expectations upstream only when they are wrong. Scheduled updates leave an existing update PR untouched to preserve review and fixes. + + ## Workflow + + 1. Keep this PR as a draft while validating the updated fixtures against the supported SDKs. + 2. Follow the repository's [review workflow](https://github.com/DataDog/system-tests/blob/main/.github/pull_request_template.md). diff --git a/manifests/cpp.yml b/manifests/cpp.yml index 0d24f159713..92a25b6e627 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -188,6 +188,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: missing_feature (Automatic baggage propagation is not implemented) tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (cpp is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index b084b22da53..228939bc974 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1207,6 +1207,7 @@ manifest: tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key) diff --git a/manifests/golang.yml b/manifests/golang.yml index 1af90b7e701..f7c7084fb66 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -2176,6 +2176,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev # Easy win for all weblogs and version 2.10.1 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: v2.10.1 tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata: diff --git a/manifests/java.yml b/manifests/java.yml index 7956efdbbe3..92294809ce5 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4251,6 +4251,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: v1.69.0-SNAPSHOT tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: v1.66.0 tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 7b5639166b9..4fe16e1dde4 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2675,6 +2675,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: 7.0.0-pre # Exact development version until the first containing release is confirmed. tests/parametric/test_ffe/test_serial_id_metadata.py: v6.18.0 tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key) diff --git a/manifests/php.yml b/manifests/php.yml index 636d6590e8e..5a6a917d8f3 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1629,6 +1629,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key) diff --git a/manifests/python.yml b/manifests/python.yml index 08779154cc2..c41f362448f 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -3138,6 +3138,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index ea239951a1b..1298fa0e2b1 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2699,6 +2699,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: v2.46.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key) diff --git a/manifests/rust.yml b/manifests/rust.yml index 21d20e796ec..d1f0e90dfd6 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -297,6 +297,7 @@ manifest: tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key) diff --git a/tests/ffe/README.md b/tests/ffe/README.md index b813164dd61..1c30734315a 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -13,6 +13,52 @@ This directory contains system tests for the Feature Flags & Experimentation (FF | `test_flag_eval_evp.py` | Server-side EVP flagevaluation payloads, aggregation, and bounds | | `test_evaluation_telemetry_opt_out.py` | Per-subject evaluation telemetry opt-out, default-off behavior, and unchanged exposures | +## Canonical parametric fixtures + +The parametric evaluation suite reads the checked-in snapshot under +`tests/parametric/test_ffe/ffe-system-test-data`. Its upstream source and exact +commit are recorded in `SOURCE.md` inside that directory. + +Change shared evaluator cases in +[`DataDog/ffe-system-test-data`](https://github.com/DataDog/ffe-system-test-data) +first, then refresh the snapshot: + +```bash +python3 utils/scripts/update_ffe_fixtures.py --ref +``` + +The updater imports `ufc-config.json` and `evaluation-cases/*.json` only. Ordinary +test runs use those checked-in files without submodules or network access. A +weekly/manual workflow opens a signed draft PR when their contents change and +leaves any existing fixture update PR untouched. Review and resolve newly exposed +SDK failures before merging an update. + +Dynamic evaluation checks each case's value, reason, and explicit error code. +`Test_Feature_Flag_Split_Features` additionally checks every canonical case with +`result.features` against the public exposure-hook callback and actual exposure +and evaluation payloads received by the parametric test agent. Exact maps detect +missing values, extra keys, destination leaks, and boolean/number coercion while +preserving JSON numeric equivalence. This covers false, zero, empty strings, +first-valid-key selection, and selected-allocation isolation. For `doLog=false`, +selected metadata is checked separately and exposure delivery must stay absent. + +The destination test currently runs with Java's `1.69.0-SNAPSHOT` SDK and later; +other clients are marked `incomplete_test_app` until they capture their public +hook output. Their baseline evaluation cases continue to run. The Java harness +loads the optional hook API only when this test enables capture, so older SDKs +can still compile and run baseline tests. Unsupported clients skip this slower +delivery test before container setup. An opt-in test-agent app factory registers +the FFE intake routes missing from the pinned agent image and records their +original payloads. This proves local delivery, not Datadog intake or production. + +Cases with optional `capabilities` still run their baseline `result`; enabling +capability-specific provider options requires separate harness support. +Remote Configuration tests and the mocked agentless backend read the same UFC. +Serial-ID metadata and span-enrichment tests select their flag definitions from +the shared corpus using their dedicated evaluation-case files. Their integration +harnesses retain trace construction and metadata assertions; no separate local +configuration copies are needed. + ## Running FFE Tests Standalone rules-configuration helpers are covered in `tests/parametric/test_ffe/test_rules_configuration.py` (Node.js only for now). These tests use `dd-trace/openfeature/rules-based` in a fresh process without initializing a tracer or provider, fetch protobuf over HTTP, and validate the portable wire output, channel credentials, typed errors, deadlines and cancellation during headers/body delivery, transport failures, invalid options, and redirect protection. They do not initialize a browser SDK or contact Datadog. @@ -330,7 +376,7 @@ The tests use `interfaces.agent.get_metrics()` to retrieve metrics from the agen ### 1. Test returns `parse_error` instead of expected reason - Check UFC fixture format (especially `totalShards` placement) -- Verify the fixture matches `flags-v1.json` format in dd-trace-py +- Verify the fixture matches the canonical `ffe-system-test-data/ufc-config.json` format ### 2. Test returns `static` instead of `split` - Need multiple variations with different shard ranges diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 0ab0720d152..ce68c7939bb 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -244,7 +244,7 @@ def assert_no_duplicate_visible_events(events: list[tuple[JSON, JSON]]) -> None: class FlagevaluationEgressContract: """One flag-evaluation contract inherited by each supported topology adapter.""" - # Agentless scenarios preload flags-v1.json before the weblog starts. Use the + # Agentless scenarios preload the canonical UFC before the weblog starts. Use the # same known fixture key in every topology; the Agent scenario installs an # equivalent fixture through Remote Config below. flag_key = "empty-targeting-key-flag" diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md new file mode 100644 index 00000000000..50d004210d7 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -0,0 +1,12 @@ +# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: caa6d1ac3e5fbda0ce57ef3ffff3772901f3f466 + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in +ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed +draft test PR only when the server evaluator fixture contents change. diff --git a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json similarity index 74% rename from tests/parametric/test_ffe/test-case-boolean-false-assignment.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json index cdd2b0a513d..ab718f5bee1 100644 --- a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json @@ -1,38 +1,41 @@ [ { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "alice", "attributes": { "should_disable_feature": true }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": false - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "bob", "attributes": { "should_disable_feature": false }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "charlie", "attributes": { "unknown_attribute": "value" }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "DEFAULT", "value": true - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json similarity index 72% rename from tests/parametric/test_ffe/test-case-boolean-one-of-matches.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json index f616614d936..392ec97ec0f 100644 --- a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json @@ -1,192 +1,208 @@ [ { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "one_of_flag": "True" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "not_matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "not_matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "haley", "attributes": { "not_matches_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "haley", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "ivy", "attributes": { "not_one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "ivy", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "julia", "attributes": { "not_one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "julia", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "kim", "attributes": { "not_one_of_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "kim", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "lucas", "attributes": { "not_one_of_flag": "true" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "lucas", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "mike", "attributes": { "not_one_of_flag": "false" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "mike", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "nicole", "attributes": { "null_flag": "null" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 5 - } + }, + "targetingKey": "nicole", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "owen", "attributes": { "null_flag": null }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "owen", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "pete", "attributes": {}, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "pete", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-comparator-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json index 2d94f30eb30..e4daa2de7c2 100644 --- a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "small" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "medium" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "size": 25 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "large" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json new file mode 100644 index 00000000000..2278ff33f64 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json @@ -0,0 +1,61 @@ +[ + { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, + "defaultValue": 0, + "flag": "disabled_flag", + "result": { + "reason": "DISABLED", + "value": 0 + }, + "targetingKey": "alice", + "variationType": "INTEGER" + }, + { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, + "defaultValue": 0, + "flag": "disabled_flag", + "result": { + "reason": "DISABLED", + "value": 0 + }, + "targetingKey": "bob", + "variationType": "INTEGER" + }, + { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], + "attributes": { + "age": 50 + }, + "defaultValue": 0, + "flag": "disabled_flag", + "result": { + "reason": "DISABLED", + "value": 0 + }, + "targetingKey": "charlie", + "variationType": "INTEGER" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json new file mode 100644 index 00000000000..f2805230ef3 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json @@ -0,0 +1,61 @@ +[ + { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, + "defaultValue": "default_value", + "flag": "empty_flag", + "result": { + "reason": "DEFAULT", + "value": "default_value" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, + "defaultValue": "default_value", + "flag": "empty_flag", + "result": { + "reason": "DEFAULT", + "value": "default_value" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], + "attributes": { + "age": 50 + }, + "defaultValue": "default_value", + "flag": "empty_flag", + "result": { + "reason": "DEFAULT", + "value": "default_value" + }, + "targetingKey": "charlie", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-empty-string-variation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json similarity index 75% rename from tests/parametric/test_ffe/test-case-empty-string-variation.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json index 1af431f3db4..f1fa3994af2 100644 --- a/tests/parametric/test_ffe/test-case-empty-string-variation.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json @@ -1,38 +1,41 @@ [ { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "empty_user", "attributes": { "content_type": "minimal" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "empty_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "full_user", "attributes": { "content_type": "full" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "detailed_content" - } + }, + "targetingKey": "full_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "default_user", "attributes": { "content_type": "unknown" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "default_user", + "variationType": "STRING" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json new file mode 100644 index 00000000000..0d2e8d5b2f5 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-do-log.json @@ -0,0 +1,22 @@ +[ + { + "description": "ordinary allocation remains eligible when activation equals cutoff", + "flag": "no-log-flag", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": true, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": true, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": true, "reason": "STATIC" } + } + }, + { + "description": "ordinary allocation remains eligible when activation is after cutoff", + "flag": "no-log-flag", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T11:59:59Z" }, + "result": { "value": true, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": true, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": true, "reason": "STATIC" } + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json new file mode 100644 index 00000000000..ea33f207067 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out-future.json @@ -0,0 +1,146 @@ +[ + { + "description": "older allocation continues", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T12:00:00.000001Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "description": "equal activation is excluded", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T12:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "newer allocation is excluded", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T11:59:59.999999Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "equal instant with negative offset", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T08:00:00-04:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "equal instant with positive offset", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-01T14:00:00+02:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "future cutoff is not an evaluation schedule", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "description": "skip newer targeted allocation and continue older randomization", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 3, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "description": "targeted allocation remains eligible before cutoff", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-21T12:00:00Z" }, + "result": { "value": 3, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 3, "reason": "TARGETING_MATCH" } + } + }, + { + "description": "skip both allocations", + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-01T12:00:00Z" }, + "result": { "value": 3, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "fractional activation equals cutoff", + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "description": "allocation is one microsecond older", + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123001Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" } + } + }, + { + "description": "allocation is one microsecond newer", + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.122999Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "description": "missing activation is not treated as older", + "flag": "kill-switch", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "alice", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": true, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": false, "reason": "DEFAULT" } + } + }, + { + "description": "null activation is not treated as older", + "flag": "numeric_flag", "variationType": "NUMERIC", "defaultValue": 0, "targetingKey": "alice", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 3.1415926, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "description": "activation is not startAt or document createdAt", + "flag": "start-and-end-date-test", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "current", "reason": "DEFAULT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "DEFAULT" } + } + }, + { + "description": "old activation does not override endAt or startAt", + "flag": "start-and-end-date-test", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-21T12:00:00Z" }, + "result": { "value": "current", "reason": "DEFAULT" }, + "capabilities": { + "experimentationOptOutFutureExperimentsOnly": { "value": "current", "reason": "DEFAULT" } + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json new file mode 100644 index 00000000000..69ca8ccaf01 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json @@ -0,0 +1,254 @@ +[ + { + "flag": "unknown-opt-out-flag", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": "default", "reason": "ERROR", "errorCode": "FLAG_NOT_FOUND" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports GENERAL when the targeting key is missing." + } + ], + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": null, + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": {}, "reason": "ERROR", "errorCode": "TARGETING_KEY_MISSING" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "19", + "attributes": { "country": "UK" }, + "result": { "value": "red", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "red", "reason": "TARGETING_MATCH" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "red", "reason": "TARGETING_MATCH" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "19", + "attributes": { "country": "UK", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "red", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "red", "reason": "TARGETING_MATCH" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "green", "reason": "TARGETING_MATCH" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "green", "reason": "TARGETING_MATCH" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" }, + "experimentationOptOutFutureExperimentsOnly": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "1", + "attributes": {}, + "result": { "value": 2, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 2, "reason": "SPLIT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 2, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": {}, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 1, "reason": "SPLIT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "1", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 2, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 2, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "flag": "numeric_flag", "variationType": "NUMERIC", "defaultValue": 0, "targetingKey": "alice", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 3.1415926, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "kill-switch", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "alice", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": true, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": false, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": false, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123456789Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T08:00:00-04:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 1, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": true }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": null }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": 1792411200 }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": false }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": null }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": {}, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": { }, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "_skip": [ + { + "platform": "python", + "reason": "Python rejects object opt-out attributes before flag evaluation." + } + ], + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": { "timestamp": "2026-10-19T12:00:00Z" } }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "_skip": [ + { + "platform": "python", + "reason": "Python rejects array opt-out attributes before flag evaluation." + } + ], + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": ["2026-10-19T12:00:00Z"] }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" }, + "experimentationOptOutFutureExperimentsOnly": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + } +] diff --git a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json similarity index 73% rename from tests/parametric/test_ffe/test-case-falsy-value-assignments.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json index 4ad259620d8..6a8cee51bcc 100644 --- a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json @@ -1,38 +1,41 @@ [ { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "zero_user", "attributes": { "plan_tier": "free" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 0 - } + }, + "targetingKey": "zero_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "premium_user", "attributes": { "plan_tier": "premium" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 100 - } + }, + "targetingKey": "premium_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "unknown_user", "attributes": { "plan_tier": "trial" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "DEFAULT", "value": 999 - } + }, + "targetingKey": "unknown_user", + "variationType": "INTEGER" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json similarity index 72% rename from tests/parametric/test_ffe/test-case-flag-with-empty-string.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json index 0bf562efa35..6a5d9cd62fd 100644 --- a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json @@ -1,24 +1,26 @@ [ { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": {}, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "SPLIT", "value": "non_empty" - } + }, + "targetingKey": "bob", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-integer-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json similarity index 66% rename from tests/parametric/test_ffe/test-case-integer-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json index 5e930885b44..8ff11aec1fc 100644 --- a/tests/parametric/test_ffe/test-case-integer-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json @@ -1,244 +1,267 @@ [ { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "debra", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "1", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "1", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "2", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "2", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "3", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "3", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "4", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "4", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "5", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "5", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "6", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "6", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "7", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "7", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "8", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "8", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "9", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "9", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "10", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "10", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "11", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "11", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "12", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "12", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "13", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "13", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "14", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "14", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "15", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "15", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "16", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "16", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "17", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "17", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "18", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "18", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "19", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "19", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json new file mode 100644 index 00000000000..be226381455 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json @@ -0,0 +1,59 @@ +[ + { + "description": "GT requires a numeric configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "age": 42 + }, + "defaultValue": "default", + "flag": "invalid-gt-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-gt", + "variationType": "STRING" + }, + { + "description": "ONE_OF requires an array configured operand. A flag with a scalar operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "role": "admin" + }, + "defaultValue": "default", + "flag": "invalid-one-of-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-one-of", + "variationType": "STRING" + }, + { + "description": "IS_NULL requires a boolean configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "deleted_at": null + }, + "defaultValue": "default", + "flag": "invalid-is-null-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-is-null", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same configuration must still evaluate after malformed condition operands are isolated.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json new file mode 100644 index 00000000000..3ad596cbf06 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json @@ -0,0 +1,17 @@ +[ + { + "description": "The malformed flag uses the invalid regex *@datadoghq.com before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": { + "email": "harish@datadoghq.com" + }, + "defaultValue": "default", + "flag": "invalid-regex-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "harish", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json new file mode 100644 index 00000000000..ca8553f099c --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json @@ -0,0 +1,40 @@ +[ + { + "description": "The malformed flag has totalShards above the unsigned 32-bit range. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "overflow-total-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "The malformed flag has a negative shard range start. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "negative-shard-range-start-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after invalid shard-bound flags are skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-kill-switch-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-kill-switch-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json index 8f34a1bc3af..3af700dc711 100644 --- a/tests/parametric/test_ffe/test-case-kill-switch-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json @@ -1,290 +1,314 @@ [ { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "barbara", "attributes": { - "email": "barbara@example.com", - "country": "canada" + "country": "canada", + "email": "barbara@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "barbara", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "charlie", "attributes": { "age": 40 }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "debra", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "1", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "1", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "2", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 50 + "age": 50, + "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "3", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "4", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "5", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "6", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "7", "attributes": { - "country": "US", - "age": 12 + "age": 12, + "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "7", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "8", "attributes": { - "country": "Italy", - "age": 60 + "age": 60, + "country": "Italy" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "8", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "9", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "10", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "10", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "11", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "11", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "12", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "13", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "14", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "14", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "15", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "16", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "17", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "18", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "19", + "variationType": "BOOLEAN" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json new file mode 100644 index 00000000000..41672357c23 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has allocations encoded as a string instead of an array. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "malformed-allocations-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the malformed flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json new file mode 100644 index 00000000000..3dba337ad02 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json @@ -0,0 +1,39 @@ +[ + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "country": "US" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "1.0.0" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "charlie", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json new file mode 100644 index 00000000000..d6dfeb180ea --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has a split that omits required shards. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "missing-split-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the missing-shards flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json similarity index 69% rename from tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json index 8ed3e2a024f..bb08523de4e 100644 --- a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json @@ -1,318 +1,344 @@ [ { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "green" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "debra", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "purple" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "id": "override-id", - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com", + "id": "override-id" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "Zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "Zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "1", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "1", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "2", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 33 + "age": 33, + "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "3", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "4", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "5", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "6", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "7", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "7", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "8", "attributes": { "country": "Italy" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "8", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "9", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "10", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "10", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "11", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "11", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "12", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "13", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "14", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "14", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "15", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "16", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "17", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "18", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "19", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json index 132c39db32a..7d985b8efd4 100644 --- a/tests/parametric/test_ffe/test-case-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "hello": "world" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "hello": "world" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-case-null-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-null-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json index 09e5d78dacd..8063ff8fee3 100644 --- a/tests/parametric/test_ffe/test-case-null-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "charlie", "attributes": { "size": null }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json new file mode 100644 index 00000000000..15f4af5a77d --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json @@ -0,0 +1,15 @@ +[ + { + "description": "The malformed flag has a null shard range before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "null-shard-range-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json new file mode 100644 index 00000000000..a2226d1b21e --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json @@ -0,0 +1,40 @@ +[ + { + "attributes": {}, + "defaultValue": 42, + "flag": "numeric_flag", + "result": { + "reason": "STATIC", + "value": 3.1415926 + }, + "targetingKey": null, + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "France" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "ERROR", + "value": "default" + }, + "targetingKey": null, + "variationType": "STRING" + }, + { + "attributes": { + "country": "France", + "email": "null@mycompany.com" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "TARGETING_MATCH", + "value": "green" + }, + "targetingKey": null, + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json similarity index 52% rename from tests/parametric/test_ffe/test-case-numeric-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json index 757f0f70e55..cc52dcdff4d 100644 --- a/tests/parametric/test_ffe/test-case-numeric-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "alice", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "bob", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json new file mode 100644 index 00000000000..d8a7e8ba092 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json @@ -0,0 +1,13 @@ +[ + { + "attributes": {}, + "defaultValue": 0, + "flag": "numeric-one-of", + "result": { + "reason": "DEFAULT", + "value": 0 + }, + "targetingKey": "race-default", + "variationType": "INTEGER" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-one-of.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json similarity index 70% rename from tests/parametric/test_ffe/test-case-numeric-one-of.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json index 9eaccbc477c..bfa957335d3 100644 --- a/tests/parametric/test_ffe/test-case-numeric-one-of.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json @@ -1,86 +1,93 @@ [ { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "number": 2 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "number": 3 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "number": 4 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "number": "1" }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "number": 123456789 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json similarity index 73% rename from tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json index 03f03ab1ecb..e99a207b6da 100644 --- a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json @@ -1,12 +1,13 @@ [ { - "flag": "empty-targeting-key-flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "", "attributes": {}, + "defaultValue": "default", + "flag": "empty-targeting-key-flag", "result": { + "reason": "STATIC", "value": "on-value" - } + }, + "targetingKey": "", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json new file mode 100644 index 00000000000..b01f6f1e5ca --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json @@ -0,0 +1,122 @@ +[ + { + "attributes": { + "comma_regex_case": "xxaaay" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "comma-quantifier" + }, + "targetingKey": "comma-quantifier", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "prefix exact suffix" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "literal-whitespace" + }, + "targetingKey": "literal-whitespace", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "exact" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "trimmed-whitespace-does-not-match", + "variationType": "STRING" + }, + { + "attributes": { + "email": "user.name+tag@capture.example" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "capturing-groups" + }, + "targetingKey": "capture", + "variationType": "STRING" + }, + { + "attributes": { + "email": "llega mañana temprano" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "unicode-word-boundary" + }, + "targetingKey": "unicode", + "variationType": "STRING" + }, + { + "attributes": { + "email": "alice@example.com", + "version": "1.15.0" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "partial-example" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "email": "bob@test.com", + "version": "0.20.1" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "test" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "charlie", + "variationType": "STRING" + }, + { + "attributes": { + "email": "derek@gmail.com", + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "derek", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json new file mode 100644 index 00000000000..bc6fe8b92bc --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json @@ -0,0 +1,169 @@ +[ + { + "attributes": { + "app_version": "1.2.3-alpha.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "1.2.3-alpha.5" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.3.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "carol", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.9.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "dave", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "eve", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.2" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "frank", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.0" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "grace", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "henry", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "mia", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "future" + }, + "targetingKey": "iris", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "5.6.7" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "jane", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "not-a-semver" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "kyle", + "variationType": "STRING" + }, + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "liam", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json new file mode 100644 index 00000000000..4f00961d9f9 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json @@ -0,0 +1,120 @@ +[ + { + "description": "A non-parseable SemVer comparand (\"not-a-version\") rejects the flag at parse time with a PARSE_ERROR rather than matching or coercing.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-syntax-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-syntax", + "variationType": "STRING" + }, + { + "description": "A SemVer comparand with repeated hyphens inside its prerelease identifiers (\"1.2.3----R-S.12.9.1--.12+meta\") is valid and matches normally.", + "attributes": { + "app_version": "1.2.3----R-S.12.9.1--.12+meta" + }, + "defaultValue": "unknown", + "flag": "semver-valid-comparand-hyphenated-prerelease-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "matched" + }, + "targetingKey": "valid-hyphenated-prerelease", + "variationType": "STRING" + }, + { + "description": "A SemVer comparand with consecutive dots in its prerelease identifiers (\"1.0.0-alpha..1\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.0.0-alpha.1" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-consecutive-dots-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-consecutive-dots", + "variationType": "STRING" + }, + { + "description": "A Ruby-style dot-suffixed SemVer comparand (\"1.2.3.DEV\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-dot-suffix-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-dot-suffix", + "variationType": "STRING" + }, + { + "description": "A v-prefixed SemVer comparand (\"v1.2.3\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-vprefix-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-vprefix", + "variationType": "STRING" + }, + { + "description": "A leading-zero core component in the SemVer comparand (\"01.2.3\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-leading-zero-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-leading-zero", + "variationType": "STRING" + }, + { + "description": "A non-string (JSON number) SemVer comparand rejects the flag at parse time with a PARSE_ERROR rather than coercing the number to a string.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-non-string-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-non-string", + "variationType": "STRING" + }, + { + "description": "The rejected invalid-comparand flags must not poison the rest of the configuration: a neighboring valid SemVer flag still evaluates normally.", + "attributes": { + "app_version": "2.3.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "after-invalid-categories", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json new file mode 100644 index 00000000000..cbddbb87d03 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json @@ -0,0 +1,17 @@ +[ + { + "description": "An invalid configured SemVer comparand in a non-last allocation must reject the whole flag at parse time, even though a valid default allocation follows it in the waterfall. An implementation that skips the invalid allocation and falls through to the default would return \"fallback\" with a STATIC reason instead of an error.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-waterfall-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "waterfall", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json new file mode 100644 index 00000000000..ef027a06660 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json @@ -0,0 +1,170 @@ +[ + { + "description": "Multi-digit numeric prerelease identifiers compare by numeric value, not lexicographically: 1.0.0-beta.2 must precede 1.0.0-beta.11. An implementation that string-compares identifiers would treat \"2\" as greater than \"11\" and fail to match.", + "attributes": { + "app_version": "10.0.0-beta.2" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "numeric-prerelease" + }, + "targetingKey": "numeric-prerelease", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for equality: 1.0.0+linux and 1.0.0+darwin have equal precedence, so SEMVER_EQ matches.", + "attributes": { + "app_version": "20.0.0+linux" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-equal-precedence" + }, + "targetingKey": "build-equal-precedence", + "variationType": "STRING" + }, + { + "description": "Dotted/multi-segment build metadata is ignored for equality: 4.0.0+exp.sha.5114f85 equals 4.0.0.", + "attributes": { + "app_version": "30.0.0+exp.sha.5114f85" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-dotted-eq" + }, + "targetingKey": "build-dotted-eq", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_LTE: 4.0.0+build.42 <= 4.0.0 is true (match).", + "attributes": { + "app_version": "40.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-lte" + }, + "targetingKey": "build-lte", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_GTE: 4.0.0+build.42 >= 4.0.0 is true (match).", + "attributes": { + "app_version": "50.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-gte" + }, + "targetingKey": "build-gte", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_NEQ: 4.0.0+build.42 != 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "60.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-neq", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_LT: 4.0.0+build.42 < 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "70.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-lt", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_GT: 4.0.0+build.42 > 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "80.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-gt", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: 1.0.0-alpha precedes 1.0.0-alpha.1 (fewer identifiers first), so SEMVER_LT matches.", + "attributes": { + "app_version": "90.0.0-alpha" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-before-alpha-1" + }, + "targetingKey": "alpha-before-alpha-1", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: numeric identifiers have lower precedence than alphanumeric (1.0.0-alpha.1 precedes 1.0.0-alpha.beta), so SEMVER_LT matches.", + "attributes": { + "app_version": "91.0.0-alpha.1" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-1-before-alpha-beta" + }, + "targetingKey": "alpha-1-before-alpha-beta", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: identifiers are compared lexicographically (1.0.0-alpha.beta precedes 1.0.0-beta), so SEMVER_LT matches.", + "attributes": { + "app_version": "92.0.0-alpha.beta" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-beta-before-beta" + }, + "targetingKey": "alpha-beta-before-beta", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: a prerelease version precedes the corresponding release (1.0.0-beta precedes 1.0.0), so SEMVER_LT matches.", + "attributes": { + "app_version": "93.0.0-beta" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta-before-release" + }, + "targetingKey": "beta-before-release", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json new file mode 100644 index 00000000000..9b4bfea3710 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json @@ -0,0 +1,255 @@ +[ + { + "description": "The maximum safe integer value is valid in every core version component.", + "attributes": { + "app_version": "9007199254740991.9007199254740991.9007199254740991" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "maximum" + }, + "targetingKey": "max-core", + "variationType": "STRING" + }, + { + "description": "Semver does not accept a v prefix.", + "attributes": { + "app_version": "v1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "v-prefix", + "variationType": "STRING" + }, + { + "description": "Core numeric components cannot have leading zeros.", + "attributes": { + "app_version": "01.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "core-leading-zero", + "variationType": "STRING" + }, + { + "description": "Numeric prerelease identifiers cannot have leading zeros.", + "attributes": { + "app_version": "1.2.3-01" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "prerelease-leading-zero", + "variationType": "STRING" + }, + { + "description": "Prerelease identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3-alpha..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-prerelease-identifier", + "variationType": "STRING" + }, + { + "description": "Build metadata identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3+build..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-build-identifier", + "variationType": "STRING" + }, + { + "description": "Prerelease and build identifiers are ASCII-only.", + "attributes": { + "app_version": "1.2.3-α" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "non-ascii-prerelease", + "variationType": "STRING" + }, + { + "description": "An empty string is not a valid SemVer version.", + "attributes": { + "app_version": "" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-string", + "variationType": "STRING" + }, + { + "description": "A leading zero in the minor core component is rejected.", + "attributes": { + "app_version": "1.02.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "minor-leading-zero", + "variationType": "STRING" + }, + { + "description": "A leading zero in the patch core component is rejected.", + "attributes": { + "app_version": "1.2.03" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "patch-leading-zero", + "variationType": "STRING" + }, + { + "description": "A trailing prerelease delimiter with no identifiers is rejected.", + "attributes": { + "app_version": "1.2.3-" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-prerelease", + "variationType": "STRING" + }, + { + "description": "A trailing build delimiter with no identifiers is rejected.", + "attributes": { + "app_version": "1.2.3+" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-build", + "variationType": "STRING" + }, + { + "description": "Underscores are not permitted in prerelease identifiers.", + "attributes": { + "app_version": "1.2.3-alpha_1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "underscore-prerelease", + "variationType": "STRING" + }, + { + "description": "More than one build metadata delimiter is rejected.", + "attributes": { + "app_version": "1.2.3-alpha+build+other" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "multiple-build-delimiters", + "variationType": "STRING" + }, + { + "description": "Leading whitespace is not accepted by the SemVer parser.", + "attributes": { + "app_version": " 1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "leading-whitespace", + "variationType": "STRING" + }, + { + "description": "Trailing whitespace is not accepted by the SemVer parser.", + "attributes": { + "app_version": "1.2.3 " + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "trailing-whitespace", + "variationType": "STRING" + }, + { + "description": "A non-string attribute (JSON number) does not match any SemVer condition and falls through to the default rather than erroring.", + "attributes": { + "app_version": 1.2 + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "non-string-attribute", + "variationType": "STRING" + }, + { + "description": "A flag with an invalid configured SemVer comparand returns a parse error while the rest of the configuration remains usable.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-comparand", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json new file mode 100644 index 00000000000..bac7fdade7b --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json @@ -0,0 +1,118 @@ +[ + { + "description": "A one-part version is normalized with missing minor and patch parts: \"18\" compares equal to \"18.0.0\".", + "attributes": { + "scenario": "one-part-input", + "app_version": "18" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "one-part-input" + }, + "targetingKey": "one-part-input", + "variationType": "STRING" + }, + { + "description": "A one-part configured comparand is valid and normalized: \"18.0.0\" compares equal to configured \"18\".", + "attributes": { + "app_version": "18.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-one-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "one-part-comparand" + }, + "targetingKey": "one-part-comparand", + "variationType": "STRING" + }, + { + "description": "A two-part version is normalized with a missing patch part: \"18.0\" compares equal to \"18.0.0\".", + "attributes": { + "scenario": "two-part-input", + "app_version": "18.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "two-part-input" + }, + "targetingKey": "two-part-input", + "variationType": "STRING" + }, + { + "description": "A two-part configured comparand is valid and normalized: \"18.0.0\" compares equal to configured \"18.0\".", + "attributes": { + "app_version": "18.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-two-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "two-part-comparand" + }, + "targetingKey": "two-part-comparand", + "variationType": "STRING" + }, + { + "description": "A four-part version is accepted as an attribute and compares above a canonical three-part lower version.", + "attributes": { + "scenario": "four-part-input", + "app_version": "18.0.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "four-part-input" + }, + "targetingKey": "four-part-input", + "variationType": "STRING" + }, + { + "description": "A four-part configured comparand is accepted and compares below a canonical three-part higher version.", + "attributes": { + "app_version": "19.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-four-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "four-part-comparand" + }, + "targetingKey": "four-part-comparand", + "variationType": "STRING" + }, + { + "description": "A five-part version is accepted as an attribute and compares above a canonical three-part lower version.", + "attributes": { + "scenario": "five-part-input", + "app_version": "18.0.0.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "five-part-input" + }, + "targetingKey": "five-part-input", + "variationType": "STRING" + }, + { + "description": "A five-part configured comparand is accepted and compares below a canonical three-part higher version.", + "attributes": { + "app_version": "19.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-five-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "five-part-comparand" + }, + "targetingKey": "five-part-comparand", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json new file mode 100644 index 00000000000..8bfadabb0e7 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json @@ -0,0 +1,97 @@ +[ + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + }, + { + "flag": "serial-id-zero-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 0, + "doLog": true + } + }, + { + "flag": "serial-id-missing-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": "control", + "reason": "STATIC", + "serialId": null, + "doLog": true + } + }, + { + "flag": "serial-id-targeting-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-vip", + "attributes": { + "segment": "vip" + }, + "result": { + "value": "vip", + "reason": "TARGETING_MATCH", + "serialId": 201, + "doLog": true + } + }, + { + "flag": "serial-id-targeting-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-standard", + "attributes": { + "segment": "standard" + }, + "result": { + "value": "standard", + "reason": "STATIC", + "serialId": 200, + "doLog": true + } + }, + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-repeat", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + }, + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-repeat", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json new file mode 100644 index 00000000000..dd988ba8583 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json @@ -0,0 +1,67 @@ +[ + { + "flag": "basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 100, + "doLog": true + } + }, + { + "flag": "experiment-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": "treatment", + "reason": "STATIC", + "serialId": 108, + "doLog": true + } + }, + { + "flag": "no-log-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-no-log", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 200, + "doLog": false + } + }, + { + "flag": "multi-serial-flag-1", + "variationType": "INTEGER", + "defaultValue": 0, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": 1, + "reason": "STATIC", + "serialId": 128, + "doLog": true + } + }, + { + "flag": "multi-serial-flag-2", + "variationType": "INTEGER", + "defaultValue": 0, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": 2, + "reason": "STATIC", + "serialId": 130, + "doLog": true + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json new file mode 100644 index 00000000000..05b3348f630 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-split-features.json @@ -0,0 +1,109 @@ +[ + { + "flag": "split-features-destinations", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 7, + "doLog": true, + "features": { + "HOOK": { + "holdout.key": "holdout-q3", + "holdout.assignment_group": "status_quo", + "shared": true, + "empty": "" + }, + "EXPOSURE": { + "holdout.assignment_group": "status_quo", + "exposure.only": false, + "shared": true + }, + "EVALUATION": { + "evaluation.only": 1.25, + "shared": true, + "zero": 0 + } + } + } + }, + { + "flag": "split-features-first-valid-key", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 7, + "doLog": true, + "features": { + "HOOK": { + "invalid-first": "accepted", + "mixed": "kept" + }, + "EXPOSURE": { + "first-wins": "first", + "empty-destinations": false + }, + "EVALUATION": {} + } + } + }, + { + "flag": "split-features-selected-allocation", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": { + "tier": "vip" + }, + "result": { + "value": true, + "reason": "TARGETING_MATCH", + "serialId": 0, + "doLog": true, + "features": { + "HOOK": { + "selected": "vip" + }, + "EXPOSURE": { + "selected": "vip" + }, + "EVALUATION": { + "selected": "vip" + } + } + } + }, + { + "flag": "split-features-selected-allocation", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "feature-subject", + "attributes": { + "tier": "standard" + }, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 7, + "doLog": false, + "features": { + "HOOK": { + "selected": "fallback" + }, + "EXPOSURE": { + "selected": "fallback" + }, + "EVALUATION": { + "selected": "fallback" + } + } + } + } +] diff --git a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json similarity index 65% rename from tests/parametric/test_ffe/test-case-start-and-end-date-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json index 7a48ec35886..53bf8859cdd 100644 --- a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "version": "1.15.0", - "country": "US" + "country": "US", + "version": "1.15.0" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "version": "0.20.1", - "country": "Canada" + "country": "Canada", + "version": "0.20.1" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "version": "2.1.13" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json new file mode 100644 index 00000000000..1bc2c914534 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json @@ -0,0 +1,16 @@ +[ + { + "description": "The flag and surrounding UFC config include unknown object fields that SDKs must ignore while evaluating the known schema fields.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "unknown-fields-tolerance-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "on" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json new file mode 100644 index 00000000000..c37fde4d573 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json @@ -0,0 +1,31 @@ +[ + { + "description": "The first allocation uses an unknown operator and the second is a catch-all trap. SDKs must return a parse error without falling through to the catch-all allocation.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "operator-grease-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the unknown-operator flag is skipped.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json new file mode 100644 index 00000000000..ca3b775ad61 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json @@ -0,0 +1,27 @@ +[ + { + "description": "A flag whose variation violates its declared type returns a parse error while the rest of the configuration remains usable.", + "attributes": {}, + "defaultValue": 0, + "flag": "variant-type-mismatch-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "user-1", + "variationType": "INTEGER" + }, + { + "description": "A valid flag in the same configuration continues to evaluate when a malformed neighboring flag returns a parse error.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json new file mode 100644 index 00000000000..0be74f1f7ac --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json @@ -0,0 +1,46 @@ +[ + { + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "alice", + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "bob", + "variationType": "NUMERIC" + }, + { + "attributes": { + "age": 50 + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" + } +] diff --git a/tests/parametric/test_ffe/test-json-config-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json similarity index 62% rename from tests/parametric/test_ffe/test-json-config-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json index ecc799546b0..45a9c2ee973 100644 --- a/tests/parametric/test_ffe/test-json-config-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json @@ -1,72 +1,76 @@ [ { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 1.0, "integer": 1, - "string": "one", - "float": 1.0 + "string": "one" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "diana", "attributes": { "Force Empty": true }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "TARGETING_MATCH", "value": {} - } + }, + "targetingKey": "diana", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json similarity index 70% rename from tests/parametric/test_ffe/test-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json index 45867e5897c..2b03b9a39a1 100644 --- a/tests/parametric/test_ffe/test-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "message": "Hello, world!" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "message": "Hello, world!" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json index 120647ec3b5..1ed7795bb95 100644 --- a/tests/parametric/test_ffe/test-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json @@ -1,54 +1,58 @@ [ { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ash", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "kümmert", "b": "schön" } - } + }, + "targetingKey": "ash", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ben", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "піклуватися", "b": "любов" } - } + }, + "targetingKey": "ben", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "cameron", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "照顾", "b": "漂亮" } - } + }, + "targetingKey": "cameron", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "darryl", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "🤗", "b": "🌸" } - } + }, + "targetingKey": "darryl", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-string-with-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-string-with-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json index e56322d44f0..32397a48398 100644 --- a/tests/parametric/test_ffe/test-string-with-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json @@ -1,794 +1,860 @@ [ { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_spaces", "attributes": { "string_with_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " a b c d e f " - } + }, + "targetingKey": "string_with_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_space", "attributes": { "string_with_only_one_space": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_one_space", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_spaces", "attributes": { "string_with_only_multiple_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_multiple_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dots", "attributes": { "string_with_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ".a.b.c.d.e.f." - } + }, + "targetingKey": "string_with_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dot", "attributes": { "string_with_only_one_dot": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "." - } + }, + "targetingKey": "string_with_only_one_dot", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dots", "attributes": { "string_with_only_multiple_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "......." - } + }, + "targetingKey": "string_with_only_multiple_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_comas", "attributes": { "string_with_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",a,b,c,d,e,f," - } + }, + "targetingKey": "string_with_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_coma", "attributes": { "string_with_only_one_coma": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "," - } + }, + "targetingKey": "string_with_only_one_coma", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_comas", "attributes": { "string_with_only_multiple_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",,,,,,," - } + }, + "targetingKey": "string_with_only_multiple_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_colons", "attributes": { "string_with_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":a:b:c:d:e:f:" - } + }, + "targetingKey": "string_with_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_colon", "attributes": { "string_with_only_one_colon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":" - } + }, + "targetingKey": "string_with_only_one_colon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_colons", "attributes": { "string_with_only_multiple_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":::::::" - } + }, + "targetingKey": "string_with_only_multiple_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_semicolons", "attributes": { "string_with_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";a;b;c;d;e;f;" - } + }, + "targetingKey": "string_with_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_semicolon", "attributes": { "string_with_only_one_semicolon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";" - } + }, + "targetingKey": "string_with_only_one_semicolon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_semicolons", "attributes": { "string_with_only_multiple_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";;;;;;;" - } + }, + "targetingKey": "string_with_only_multiple_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_slashes", "attributes": { "string_with_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/a/b/c/d/e/f/" - } + }, + "targetingKey": "string_with_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_slash", "attributes": { "string_with_only_one_slash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/" - } + }, + "targetingKey": "string_with_only_one_slash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_slashes", "attributes": { "string_with_only_multiple_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "///////" - } + }, + "targetingKey": "string_with_only_multiple_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dashes", "attributes": { "string_with_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-a-b-c-d-e-f-" - } + }, + "targetingKey": "string_with_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dash", "attributes": { "string_with_only_one_dash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-" - } + }, + "targetingKey": "string_with_only_one_dash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dashes", "attributes": { "string_with_only_multiple_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-------" - } + }, + "targetingKey": "string_with_only_multiple_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_underscores", "attributes": { "string_with_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_a_b_c_d_e_f_" - } + }, + "targetingKey": "string_with_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_underscore", "attributes": { "string_with_only_one_underscore": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_" - } + }, + "targetingKey": "string_with_only_one_underscore", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_underscores", "attributes": { "string_with_only_multiple_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_______" - } + }, + "targetingKey": "string_with_only_multiple_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_plus_signs", "attributes": { "string_with_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+a+b+c+d+e+f+" - } + }, + "targetingKey": "string_with_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_plus_sign", "attributes": { "string_with_only_one_plus_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+" - } + }, + "targetingKey": "string_with_only_one_plus_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_plus_signs", "attributes": { "string_with_only_multiple_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+++++++" - } + }, + "targetingKey": "string_with_only_multiple_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_equal_signs", "attributes": { "string_with_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=a=b=c=d=e=f=" - } + }, + "targetingKey": "string_with_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_equal_sign", "attributes": { "string_with_only_one_equal_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=" - } + }, + "targetingKey": "string_with_only_one_equal_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_equal_signs", "attributes": { "string_with_only_multiple_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=======" - } + }, + "targetingKey": "string_with_only_multiple_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dollar_signs", "attributes": { "string_with_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$a$b$c$d$e$f$" - } + }, + "targetingKey": "string_with_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dollar_sign", "attributes": { "string_with_only_one_dollar_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$" - } + }, + "targetingKey": "string_with_only_one_dollar_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dollar_signs", "attributes": { "string_with_only_multiple_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$$$$$$$" - } + }, + "targetingKey": "string_with_only_multiple_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_at_signs", "attributes": { "string_with_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@a@b@c@d@e@f@" - } + }, + "targetingKey": "string_with_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_at_sign", "attributes": { "string_with_only_one_at_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@" - } + }, + "targetingKey": "string_with_only_one_at_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_at_signs", "attributes": { "string_with_only_multiple_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@@@@@@@" - } + }, + "targetingKey": "string_with_only_multiple_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_amp_signs", "attributes": { "string_with_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&a&b&c&d&e&f&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026a\u0026b\u0026c\u0026d\u0026e\u0026f\u0026" + }, + "targetingKey": "string_with_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_amp_sign", "attributes": { "string_with_only_one_amp_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026" + }, + "targetingKey": "string_with_only_one_amp_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_amp_signs", "attributes": { "string_with_only_multiple_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&&&&&&&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026\u0026\u0026\u0026\u0026\u0026\u0026" + }, + "targetingKey": "string_with_only_multiple_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_hash_signs", "attributes": { "string_with_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#a#b#c#d#e#f#" - } + }, + "targetingKey": "string_with_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_hash_sign", "attributes": { "string_with_only_one_hash_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#" - } + }, + "targetingKey": "string_with_only_one_hash_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_hash_signs", "attributes": { "string_with_only_multiple_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#######" - } + }, + "targetingKey": "string_with_only_multiple_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_percentage_signs", "attributes": { "string_with_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%a%b%c%d%e%f%" - } + }, + "targetingKey": "string_with_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_percentage_sign", "attributes": { "string_with_only_one_percentage_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%" - } + }, + "targetingKey": "string_with_only_one_percentage_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_percentage_signs", "attributes": { "string_with_only_multiple_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%%%%%%%" - } + }, + "targetingKey": "string_with_only_multiple_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_tilde_signs", "attributes": { "string_with_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~a~b~c~d~e~f~" - } + }, + "targetingKey": "string_with_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_tilde_sign", "attributes": { "string_with_only_one_tilde_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~" - } + }, + "targetingKey": "string_with_only_one_tilde_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_tilde_signs", "attributes": { "string_with_only_multiple_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~~~~~~~" - } + }, + "targetingKey": "string_with_only_multiple_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_asterix_signs", "attributes": { "string_with_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*a*b*c*d*e*f*" - } + }, + "targetingKey": "string_with_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_asterix_sign", "attributes": { "string_with_only_one_asterix_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*" - } + }, + "targetingKey": "string_with_only_one_asterix_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_asterix_signs", "attributes": { "string_with_only_multiple_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*******" - } + }, + "targetingKey": "string_with_only_multiple_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_single_quotes", "attributes": { "string_with_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'a'b'c'd'e'f'" - } + }, + "targetingKey": "string_with_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_single_quote", "attributes": { "string_with_only_one_single_quote": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'" - } + }, + "targetingKey": "string_with_only_one_single_quote", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_single_quotes", "attributes": { "string_with_only_multiple_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'''''''" - } + }, + "targetingKey": "string_with_only_multiple_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_question_marks", "attributes": { "string_with_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?a?b?c?d?e?f?" - } + }, + "targetingKey": "string_with_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_question_mark", "attributes": { "string_with_only_one_question_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?" - } + }, + "targetingKey": "string_with_only_one_question_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_question_marks", "attributes": { "string_with_only_multiple_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "???????" - } + }, + "targetingKey": "string_with_only_multiple_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_exclamation_marks", "attributes": { "string_with_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!a!b!c!d!e!f!" - } + }, + "targetingKey": "string_with_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_exclamation_mark", "attributes": { "string_with_only_one_exclamation_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!" - } + }, + "targetingKey": "string_with_only_one_exclamation_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_exclamation_marks", "attributes": { "string_with_only_multiple_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!!!!!!!" - } + }, + "targetingKey": "string_with_only_multiple_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_opening_parentheses", "attributes": { "string_with_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(a(b(c(d(e(f(" - } + }, + "targetingKey": "string_with_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_opening_parenthese", "attributes": { "string_with_only_one_opening_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(" - } + }, + "targetingKey": "string_with_only_one_opening_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_opening_parentheses", "attributes": { "string_with_only_multiple_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(((((((" - } + }, + "targetingKey": "string_with_only_multiple_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_closing_parentheses", "attributes": { "string_with_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")a)b)c)d)e)f)" - } + }, + "targetingKey": "string_with_closing_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_closing_parenthese", "attributes": { "string_with_only_one_closing_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")" - } + }, + "targetingKey": "string_with_only_one_closing_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_closing_parentheses", "attributes": { "string_with_only_multiple_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")))))))" - } + }, + "targetingKey": "string_with_only_multiple_closing_parentheses", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/flags-v1.json b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json similarity index 56% rename from tests/parametric/test_ffe/flags-v1.json rename to tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json index f01bfe2ef86..677e719c1b5 100644 --- a/tests/parametric/test_ffe/flags-v1.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json @@ -2,8 +2,11 @@ "createdAt": "2024-04-17T19:40:53.716Z", "format": "SERVER", "environment": { - "name": "Test" + "name": "Test", + "unknownEnvironmentField": "ignored" }, + "unknownTopLevelField": "ignored", + "unknownTopLevelNull": null, "flags": { "empty_flag": { "key": "empty_flag", @@ -62,7 +65,8 @@ "shards": [] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": null } ] }, @@ -78,9 +82,109 @@ "test": { "key": "test", "value": "test" + }, + "capturing-groups": { + "key": "capturing-groups", + "value": "capturing-groups" + }, + "unicode-word-boundary": { + "key": "unicode-word-boundary", + "value": "unicode-word-boundary" + }, + "comma-quantifier": { + "key": "comma-quantifier", + "value": "comma-quantifier" + }, + "literal-whitespace": { + "key": "literal-whitespace", + "value": "literal-whitespace" } }, "allocations": [ + { + "key": "comma-quantifier", + "rules": [ + { + "conditions": [ + { + "attribute": "comma_regex_case", + "operator": "MATCHES", + "value": "a{1,3}" + } + ] + } + ], + "splits": [ + { + "variationKey": "comma-quantifier", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "literal-whitespace", + "rules": [ + { + "conditions": [ + { + "attribute": "whitespace_regex_case", + "operator": "MATCHES", + "value": " exact " + } + ] + } + ], + "splits": [ + { + "variationKey": "literal-whitespace", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "capturing-groups", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "^([[:alnum:]._%+-]+)@capture\\.example$" + } + ] + } + ], + "splits": [ + { + "variationKey": "capturing-groups", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "unicode-word-boundary", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "(?u)\\bmañana\\b" + } + ] + } + ], + "splits": [ + { + "variationKey": "unicode-word-boundary", + "shards": [] + } + ], + "doLog": true + }, { "key": "partial-example", "rules": [ @@ -612,122 +716,187 @@ } ] }, - "start-and-end-date-test": { - "key": "start-and-end-date-test", + "semver-comparison-test": { + "key": "semver-comparison-test", "enabled": true, "variationType": "STRING", "variations": { - "old": { - "key": "old", - "value": "old" + "legacy": { + "key": "legacy", + "value": "legacy" }, - "current": { - "key": "current", - "value": "current" + "stable": { + "key": "stable", + "value": "stable" }, - "new": { - "key": "new", - "value": "new" + "beta": { + "key": "beta", + "value": "beta" + }, + "launch": { + "key": "launch", + "value": "launch" + }, + "future": { + "key": "future", + "value": "future" + }, + "maximum": { + "key": "maximum", + "value": "maximum" } }, "allocations": [ { - "key": "old-versions", + "key": "maximum-version", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "9007199254740991.9007199254740991.9007199254740991" + } + ] + } + ], "splits": [ { - "variationKey": "old", + "variationKey": "maximum", "shards": [] } ], - "endAt": "2002-10-31T09:00:00.594Z", "doLog": true }, { - "key": "future-versions", + "key": "legacy-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_LTE", + "value": "1.2.3-alpha.4" + } + ] + }, + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "1.2.3-alpha.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "2.3.4" + } + ] + } + ], "splits": [ { - "variationKey": "new", + "variationKey": "legacy", "shards": [] } ], - "startAt": "2052-10-31T09:00:00.594Z", "doLog": true }, { - "key": "current-versions", + "key": "stable-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "2.3.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "3.4.5-beta.2" + } + ] + } + ], "splits": [ { - "variationKey": "current", + "variationKey": "stable", "shards": [] } ], - "startAt": "2022-10-31T09:00:00.594Z", - "endAt": "2050-10-31T09:00:00.594Z", "doLog": true - } - ] - }, - "null-operator-test": { - "key": "null-operator-test", - "enabled": true, - "variationType": "STRING", - "variations": { - "old": { - "key": "old", - "value": "old" }, - "new": { - "key": "new", - "value": "new" - } - }, - "allocations": [ { - "key": "null-operator", + "key": "beta-versions", "rules": [ { "conditions": [ { - "attribute": "size", - "operator": "IS_NULL", - "value": true + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "3.4.5-beta.2" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "4.5.6-rc.1" } ] - }, + } + ], + "splits": [ + { + "variationKey": "beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "launch-version", + "rules": [ { "conditions": [ { - "attribute": "size", - "operator": "LT", - "value": 10 + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "4.5.6-rc.1" } ] } ], "splits": [ { - "variationKey": "old", + "variationKey": "launch", "shards": [] } ], "doLog": true }, { - "key": "not-null-operator", + "key": "future-versions", "rules": [ { "conditions": [ { - "attribute": "size", - "operator": "IS_NULL", - "value": false + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "4.5.6-rc.1" + }, + { + "attribute": "app_version", + "operator": "SEMVER_NEQ", + "value": "5.6.7" } ] } ], "splits": [ { - "variationKey": "new", + "variationKey": "future", "shards": [] } ], @@ -735,108 +904,319 @@ } ] }, - "new-user-onboarding": { - "key": "new-user-onboarding", + "semver-invalid-comparand-test": { + "key": "semver-invalid-comparand-test", "enabled": true, "variationType": "STRING", + "comment": "This flag deliberately uses an invalid configured SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", "variations": { - "control": { - "key": "control", - "value": "control" - }, - "red": { - "key": "red", - "value": "red" - }, - "blue": { - "key": "blue", - "value": "blue" - }, - "green": { - "key": "green", - "value": "green" - }, - "yellow": { - "key": "yellow", - "value": "yellow" - }, - "purple": { - "key": "purple", - "value": "purple" + "matched": { + "key": "matched", + "value": "matched" } }, "allocations": [ { - "key": "id rule", + "key": "invalid-comparand", "rules": [ { "conditions": [ { - "attribute": "id", - "operator": "MATCHES", - "value": "zach" + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18446744073709551616.0.0" } ] } ], "splits": [ { - "variationKey": "purple", + "variationKey": "matched", "shards": [] } ], - "doLog": false + "doLog": true + } + ] + }, + "semver-invalid-comparand-waterfall-test": { + "key": "semver-invalid-comparand-waterfall-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately places an invalid configured SemVer comparand in a non-last allocation of a waterfall, followed by a valid default allocation. SDKs must reject the whole flag during parsing rather than skipping the invalid allocation and falling through to the default.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" }, + "fallback": { + "key": "fallback", + "value": "fallback" + } + }, + "allocations": [ { - "key": "internal users", + "key": "invalid-comparand", "rules": [ { "conditions": [ { - "attribute": "email", - "operator": "MATCHES", - "value": "@mycompany.com" + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18446744073709551616.0.0" } ] } ], "splits": [ { - "variationKey": "green", + "variationKey": "matched", "shards": [] } ], - "doLog": false + "doLog": true }, { - "key": "experiment", - "rules": [ + "key": "default", + "rules": [], + "splits": [ { - "conditions": [ - { - "attribute": "country", - "operator": "NOT_ONE_OF", - "value": [ - "US", - "Canada", - "Mexico" - ] - } - ] + "variationKey": "fallback", + "shards": [] } ], - "splits": [ - { - "variationKey": "control", - "shards": [ - { - "salt": "traffic-new-user-onboarding-experiment", - "totalShards": 10000, - "ranges": [ - { - "start": 0, - "end": 6000 - } + "doLog": true + } + ] + }, + "start-and-end-date-test": { + "key": "start-and-end-date-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "old": { + "key": "old", + "value": "old" + }, + "current": { + "key": "current", + "value": "current" + }, + "new": { + "key": "new", + "value": "new" + } + }, + "allocations": [ + { + "key": "old-versions", + "splits": [ + { + "variationKey": "old", + "shards": [] + } + ], + "endAt": "2002-10-31T09:00:00.594Z", + "doLog": true, + "randomizationActivatedAt": "2000-01-01T00:00:00Z" + }, + { + "key": "future-versions", + "splits": [ + { + "variationKey": "new", + "shards": [] + } + ], + "startAt": "2052-10-31T09:00:00.594Z", + "doLog": true, + "randomizationActivatedAt": "2026-10-01T12:00:00Z" + }, + { + "key": "current-versions", + "splits": [ + { + "variationKey": "current", + "shards": [] + } + ], + "startAt": "2022-10-31T09:00:00.594Z", + "endAt": "2050-10-31T09:00:00.594Z", + "doLog": true, + "randomizationActivatedAt": "2026-10-20T12:00:00Z" + } + ] + }, + "null-operator-test": { + "key": "null-operator-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "old": { + "key": "old", + "value": "old" + }, + "new": { + "key": "new", + "value": "new" + } + }, + "allocations": [ + { + "key": "null-operator", + "rules": [ + { + "conditions": [ + { + "attribute": "size", + "operator": "IS_NULL", + "value": true + } + ] + }, + { + "conditions": [ + { + "attribute": "size", + "operator": "LT", + "value": 10 + } + ] + } + ], + "splits": [ + { + "variationKey": "old", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "not-null-operator", + "rules": [ + { + "conditions": [ + { + "attribute": "size", + "operator": "IS_NULL", + "value": false + } + ] + } + ], + "splits": [ + { + "variationKey": "new", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "new-user-onboarding": { + "key": "new-user-onboarding", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "red": { + "key": "red", + "value": "red" + }, + "blue": { + "key": "blue", + "value": "blue" + }, + "green": { + "key": "green", + "value": "green" + }, + "yellow": { + "key": "yellow", + "value": "yellow" + }, + "purple": { + "key": "purple", + "value": "purple" + } + }, + "allocations": [ + { + "key": "id rule", + "rules": [ + { + "conditions": [ + { + "attribute": "id", + "operator": "MATCHES", + "value": "zach" + } + ] + } + ], + "splits": [ + { + "variationKey": "purple", + "shards": [] + } + ], + "doLog": false + }, + { + "key": "internal users", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "@mycompany.com" + } + ] + } + ], + "splits": [ + { + "variationKey": "green", + "shards": [] + } + ], + "doLog": false + }, + { + "key": "experiment", + "rules": [ + { + "conditions": [ + { + "attribute": "country", + "operator": "NOT_ONE_OF", + "value": [ + "US", + "Canada", + "Mexico" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "traffic-new-user-onboarding-experiment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 6000 + } ] }, { @@ -902,7 +1282,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-01T12:00:00Z" }, { "key": "rollout", @@ -1008,7 +1389,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-20T12:00:00Z" }, { "key": "50/50 split", @@ -1045,7 +1427,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-01T12:00:00Z" } ] }, @@ -1268,7 +1651,8 @@ ] } ], - "doLog": true + "doLog": true, + "randomizationActivatedAt": "2026-10-19T12:00:00.123Z" }, { "key": "allocation-default", @@ -3293,6 +3677,2023 @@ "doLog": true } ] + }, + "microsecond-date-test": { + "key": "microsecond-date-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "expired": { + "key": "expired", + "value": "expired" + }, + "active": { + "key": "active", + "value": "active" + }, + "future": { + "key": "future", + "value": "future" + } + }, + "allocations": [ + { + "key": "expired-allocation", + "splits": [ + { + "variationKey": "expired", + "shards": [] + } + ], + "endAt": "2002-10-31T09:00:00.594321Z", + "doLog": true + }, + { + "key": "future-allocation", + "splits": [ + { + "variationKey": "future", + "shards": [] + } + ], + "startAt": "2052-10-31T09:00:00.123456Z", + "doLog": true + }, + { + "key": "active-allocation", + "splits": [ + { + "variationKey": "active", + "shards": [] + } + ], + "startAt": "2022-10-31T09:00:00.235982Z", + "endAt": "2050-10-31T09:00:00.987654Z", + "doLog": true + } + ] + }, + "unknown-fields-tolerance-flag": { + "key": "unknown-fields-tolerance-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately contains unknown fields at every fixed-schema UFC level. SDKs must ignore those fields and evaluate the flag normally.", + "unknownFlagField": "ignored", + "variations": { + "on": { + "key": "on", + "value": "on", + "unknownVariationField": "ignored" + } + }, + "allocations": [ + { + "key": "unknown-fields-allocation", + "unknownAllocationField": "ignored", + "rules": [ + { + "unknownRuleField": "ignored", + "conditions": [ + { + "attribute": "country", + "operator": "ONE_OF", + "value": [ + "US" + ], + "unknownConditionField": "ignored" + } + ] + } + ], + "splits": [ + { + "variationKey": "on", + "unknownSplitField": "ignored", + "shards": [ + { + "salt": "unknown-fields-tolerance-flag", + "totalShards": 10000, + "unknownShardField": "ignored", + "ranges": [ + { + "start": 0, + "end": 10000, + "unknownRangeField": "ignored" + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "operator-grease-flag": { + "key": "operator-grease-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses an unknown condition operator. SDKs must remove this flag rather than falling through to the catch-all allocation or rejecting the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": [ + { + "key": "grease-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "country", + "operator": "NOT_A_REAL_OPERATOR", + "value": "anything" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-regex-flag": { + "key": "invalid-regex-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses the invalid regex *@datadoghq.com. SDKs must remove this flag rather than falling through to a later allocation or rejecting the whole config.", + "variations": { + "invalid-regex-match": { + "key": "invalid-regex-match", + "value": "invalid-regex-match" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-regex-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "*@datadoghq.com" + } + ] + } + ], + "splits": [ + { + "variationKey": "invalid-regex-match", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-regex-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "variant-type-mismatch-flag": { + "key": "variant-type-mismatch-flag", + "enabled": true, + "variationType": "INTEGER", + "comment": "This flag is intentionally invalid: one variation contains a string even though the flag declares INTEGER values. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "invalid": { + "key": "invalid", + "value": "not-an-integer" + }, + "valid": { + "key": "valid", + "value": 0 + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "invalid", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-gt-condition-flag": { + "key": "invalid-gt-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives GT a string operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-gt-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "age", + "operator": "GT", + "value": "not-a-number" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-gt-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-one-of-condition-flag": { + "key": "invalid-one-of-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives ONE_OF a scalar operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-one-of-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "role", + "operator": "ONE_OF", + "value": "admin" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-one-of-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-is-null-condition-flag": { + "key": "invalid-is-null-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives IS_NULL a non-boolean operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-is-null-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "deleted_at", + "operator": "IS_NULL", + "value": "not-a-boolean" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-is-null-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "malformed-allocations-flag": { + "key": "malformed-allocations-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: allocations must be an array. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": "this-is-not-a-list" + }, + "missing-split-shards-flag": { + "key": "missing-split-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards is required even when the intended split is unsharded. SDKs must treat omitted shards differently from an explicit empty shards array.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-split-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap" + } + ], + "doLog": true + } + ] + }, + "overflow-total-shards-flag": { + "key": "overflow-total-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].totalShards must fit in an unsigned 32-bit integer. SDKs must remove this flag rather than overflowing or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "overflow-total-shards-flag", + "totalShards": 4294967296, + "ranges": [ + { + "start": 0, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "negative-shard-range-start-flag": { + "key": "negative-shard-range-start-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[].start must be an unsigned integer. SDKs must remove this flag rather than treating a negative range as matching traffic.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "negative-shard-range-start-flag", + "totalShards": 100, + "ranges": [ + { + "start": -1, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "null-shard-range-flag": { + "key": "null-shard-range-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[] contains null. SDKs must remove this flag rather than panicking or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "null-shard-range-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "null-shard-range-flag", + "totalShards": 8192, + "ranges": [ + null + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "null-shard-range-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "valid-flag-after-invalid-config": { + "key": "valid-flag-after-invalid-config", + "enabled": true, + "variationType": "STRING", + "comment": "This valid neighbor proves malformed or unsupported flags do not poison the rest of the remote config.", + "variations": { + "expected": { + "key": "expected", + "value": "expected" + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "expected", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-precedence-test": { + "key": "semver-precedence-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates SemVer precedence rules that are easy to get wrong: multi-digit numeric prerelease ordering (beta.2 < beta.11), build-metadata-ignored across every operator (EQ/NEQ/LT/LTE/GT/GTE), and alphanumeric prerelease ordering (alpha < alpha.1 < alpha.beta < beta < release). Each allocation gates on a unique SEMVER_EQ attribute version so test attributes cannot cross-match other allocations; negative build-metadata cases (NEQ/LT/GT) fall through to DEFAULT.", + "variations": { + "numeric-prerelease": { + "key": "numeric-prerelease", + "value": "numeric-prerelease" + }, + "build-equal-precedence": { + "key": "build-equal-precedence", + "value": "build-equal-precedence" + }, + "build-dotted-eq": { + "key": "build-dotted-eq", + "value": "build-dotted-eq" + }, + "build-lte": { + "key": "build-lte", + "value": "build-lte" + }, + "build-gte": { + "key": "build-gte", + "value": "build-gte" + }, + "alpha-before-alpha-1": { + "key": "alpha-before-alpha-1", + "value": "alpha-before-alpha-1" + }, + "alpha-1-before-alpha-beta": { + "key": "alpha-1-before-alpha-beta", + "value": "alpha-1-before-alpha-beta" + }, + "alpha-beta-before-beta": { + "key": "alpha-beta-before-beta", + "value": "alpha-beta-before-beta" + }, + "beta-before-release": { + "key": "beta-before-release", + "value": "beta-before-release" + }, + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "numeric-prerelease", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "10.0.0-beta.2" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "10.0.0-beta.11" + } + ] + } + ], + "splits": [ + { + "variationKey": "numeric-prerelease", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-equal-precedence", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "20.0.0+linux" + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "20.0.0+darwin" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-equal-precedence", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-dotted-eq", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "30.0.0+exp.sha.5114f85" + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "30.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-dotted-eq", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-lte", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "40.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LTE", + "value": "40.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-lte", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-gte", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "50.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "50.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-gte", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-neq", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "60.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_NEQ", + "value": "60.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-lt", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "70.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "70.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-gt", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "80.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "80.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-before-alpha-1", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "90.0.0-alpha" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "90.0.0-alpha.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-before-alpha-1", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-1-before-alpha-beta", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "91.0.0-alpha.1" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "91.0.0-alpha.beta" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-1-before-alpha-beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-beta-before-beta", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "92.0.0-alpha.beta" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "92.0.0-beta" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-beta-before-beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "beta-before-release", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "93.0.0-beta" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "93.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "beta-before-release", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-input-test": { + "key": "semver-version-parts-input-test", + "enabled": true, + "variationType": "STRING", + "comment": "Exercises one- through five-part runtime attributes using only canonical three-part configured comparands, so comparand ingestion cannot prevent an input case from running. Scenario gates isolate each runtime comparison.", + "variations": { + "one-part-input": { + "key": "one-part-input", + "value": "one-part-input" + }, + "two-part-input": { + "key": "two-part-input", + "value": "two-part-input" + }, + "four-part-input": { + "key": "four-part-input", + "value": "four-part-input" + }, + "five-part-input": { + "key": "five-part-input", + "value": "five-part-input" + } + }, + "allocations": [ + { + "key": "one-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "one-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "one-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "two-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "two-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "two-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "four-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "four-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "17.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "four-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "five-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "five-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "17.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "five-part-input", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-one-part-comparand-test": { + "key": "semver-version-parts-one-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a one-part configured SemVer comparand from every other extended comparand shape.", + "variations": { + "one-part-comparand": { + "key": "one-part-comparand", + "value": "one-part-comparand" + } + }, + "allocations": [ + { + "key": "one-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18" + } + ] + } + ], + "splits": [ + { + "variationKey": "one-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-two-part-comparand-test": { + "key": "semver-version-parts-two-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a two-part configured SemVer comparand from every other extended comparand shape.", + "variations": { + "two-part-comparand": { + "key": "two-part-comparand", + "value": "two-part-comparand" + } + }, + "allocations": [ + { + "key": "two-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "two-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-four-part-comparand-test": { + "key": "semver-version-parts-four-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a four-part configured SemVer comparand while the runtime attribute remains a canonical three-part version.", + "variations": { + "four-part-comparand": { + "key": "four-part-comparand", + "value": "four-part-comparand" + } + }, + "allocations": [ + { + "key": "four-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "18.0.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "four-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-five-part-comparand-test": { + "key": "semver-version-parts-five-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a five-part configured SemVer comparand while the runtime attribute remains a canonical three-part version.", + "variations": { + "five-part-comparand": { + "key": "five-part-comparand", + "value": "five-part-comparand" + } + }, + "allocations": [ + { + "key": "five-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "18.0.0.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "five-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-syntax-test": { + "key": "semver-invalid-comparand-syntax-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a non-parseable SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "not-a-version" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-valid-comparand-hyphenated-prerelease-test": { + "key": "semver-valid-comparand-hyphenated-prerelease-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag uses a valid SemVer comparand with repeated hyphens inside its prerelease identifiers.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "valid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.2.3----R-S.12.9.1--.12+meta" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-consecutive-dots-test": { + "key": "semver-invalid-comparand-consecutive-dots-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses consecutive dots in its SemVer prerelease comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.0.0-alpha..1" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-dot-suffix-test": { + "key": "semver-invalid-comparand-dot-suffix-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a Ruby-style dot-suffixed SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.2.3.DEV" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-vprefix-test": { + "key": "semver-invalid-comparand-vprefix-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a v-prefixed SemVer comparand, which is not valid SemVer. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "v1.2.3" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-leading-zero-test": { + "key": "semver-invalid-comparand-leading-zero-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a leading-zero core component in its SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "01.2.3" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-non-string-test": { + "key": "semver-invalid-comparand-non-string-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a non-string (JSON number) SemVer comparand. SDKs must reject this as an invalid comparand without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": 1.2 + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "serial-id-basic-flag": { + "key": "serial-id-basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 42 + } + ], + "doLog": true + } + ] + }, + "serial-id-zero-flag": { + "key": "serial-id-zero-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "zero-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 0 + } + ], + "doLog": true + } + ] + }, + "serial-id-missing-flag": { + "key": "serial-id-missing-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "treatment": { + "key": "treatment", + "value": "treatment" + } + }, + "allocations": [ + { + "key": "no-serial-allocation", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "serial-id-targeting-flag": { + "key": "serial-id-targeting-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "vip": { + "key": "vip", + "value": "vip" + }, + "standard": { + "key": "standard", + "value": "standard" + } + }, + "allocations": [ + { + "key": "vip-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "segment", + "operator": "ONE_OF", + "value": [ + "vip" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "vip", + "shards": [], + "serialId": 201 + } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "standard", + "shards": [], + "serialId": 200 + } + ], + "doLog": true + } + ] + }, + "basic-flag": { + "key": "basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 100 + } + ], + "doLog": true + } + ] + }, + "experiment-flag": { + "key": "experiment-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "treatment": { + "key": "treatment", + "value": "treatment" + } + }, + "allocations": [ + { + "key": "experiment-allocation", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 108 + } + ], + "doLog": true + } + ] + }, + "no-log-flag": { + "key": "no-log-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "no-log-allocation", + "randomizationActivatedAt": "2026-10-19T12:00:00Z", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 200 + } + ], + "doLog": false + } + ] + }, + "multi-serial-flag-1": { + "key": "multi-serial-flag-1", + "enabled": true, + "variationType": "INTEGER", + "variations": { + "v1": { + "key": "v1", + "value": 1 + } + }, + "allocations": [ + { + "key": "allocation-1", + "rules": [], + "splits": [ + { + "variationKey": "v1", + "shards": [], + "serialId": 128 + } + ], + "doLog": true + } + ] + }, + "multi-serial-flag-2": { + "key": "multi-serial-flag-2", + "enabled": true, + "variationType": "INTEGER", + "variations": { + "v2": { + "key": "v2", + "value": 2 + } + }, + "allocations": [ + { + "key": "allocation-2", + "rules": [], + "splits": [ + { + "variationKey": "v2", + "shards": [], + "serialId": 130 + } + ], + "doLog": true + } + ] + }, + "split-features-destinations": { + "key": "split-features-destinations", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + } + }, + "allocations": [ + { + "key": "allocation", + "doLog": true, + "splits": [ + { + "variationKey": "on", + "serialId": 7, + "shards": [], + "features": [ + { + "key": "holdout.key", + "value": "holdout-q3", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK", + "EXPOSURE" + ] + }, + { + "key": "exposure.only", + "value": false, + "destinations": [ + "EXPOSURE" + ] + }, + { + "key": "evaluation.only", + "value": 1.25, + "destinations": [ + "EVALUATION" + ] + }, + { + "key": "shared", + "value": true, + "destinations": [ + "HOOK", + "EXPOSURE", + "EVALUATION", + "HOOK", + "FUTURE" + ] + }, + { + "key": "empty", + "value": "", + "destinations": [ + "HOOK" + ] + }, + { + "key": "zero", + "value": 0, + "destinations": [ + "EVALUATION" + ] + }, + { + "key": "unknown.only", + "value": true, + "destinations": [ + "FUTURE", + "hook" + ] + } + ] + } + ] + } + ] + }, + "split-features-first-valid-key": { + "key": "split-features-first-valid-key", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + } + }, + "allocations": [ + { + "key": "allocation", + "doLog": true, + "splits": [ + { + "variationKey": "on", + "serialId": 7, + "shards": [], + "features": [ + { + "key": "invalid-first", + "value": {}, + "destinations": [ + "HOOK" + ] + }, + { + "key": "invalid-first", + "value": "accepted", + "destinations": [ + "HOOK" + ] + }, + { + "key": "invalid-first", + "value": "later", + "destinations": [ + "EXPOSURE" + ] + }, + { + "key": "first-wins", + "value": "first", + "destinations": [ + "EXPOSURE" + ] + }, + { + "key": "first-wins", + "value": "later", + "destinations": [ + "HOOK", + "EVALUATION" + ] + }, + { + "key": "unknown-first", + "value": "first", + "destinations": [ + "FUTURE" + ] + }, + { + "key": "unknown-first", + "value": "later", + "destinations": [ + "HOOK" + ] + }, + { + "key": "empty-destinations", + "value": "invalid", + "destinations": [] + }, + { + "key": "empty-destinations", + "value": false, + "destinations": [ + "EXPOSURE" + ] + }, + null, + { + "key": "array", + "value": [], + "destinations": [ + "HOOK" + ] + }, + { + "key": "null", + "value": null, + "destinations": [ + "HOOK" + ] + }, + { + "key": "", + "value": true, + "destinations": [ + "HOOK" + ] + }, + { + "key": "mixed", + "value": "kept", + "destinations": [ + "HOOK", + null, + 42, + "" + ] + }, + { + "key": "bad-destinations", + "value": true, + "destinations": "HOOK" + } + ] + } + ] + } + ] + }, + "split-features-selected-allocation": { + "key": "split-features-selected-allocation", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + } + }, + "allocations": [ + { + "key": "vip", + "doLog": true, + "rules": [ + { + "conditions": [ + { + "attribute": "tier", + "operator": "ONE_OF", + "value": [ + "vip" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "on", + "serialId": 0, + "shards": [], + "features": [ + { + "key": "selected", + "value": "vip", + "destinations": [ + "HOOK", + "EXPOSURE", + "EVALUATION" + ] + } + ] + } + ] + }, + { + "key": "allocation", + "doLog": false, + "splits": [ + { + "variationKey": "on", + "serialId": 7, + "shards": [], + "features": [ + { + "key": "selected", + "value": "fallback", + "destinations": [ + "HOOK", + "EXPOSURE", + "EVALUATION" + ] + } + ] + } + ] + } + ] } } -} \ No newline at end of file +} diff --git a/tests/parametric/test_ffe/serial-id-metadata-flags.json b/tests/parametric/test_ffe/serial-id-metadata-flags.json deleted file mode 100644 index 35bf0ab4c74..00000000000 --- a/tests/parametric/test_ffe/serial-id-metadata-flags.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "createdAt": "2024-01-01T00:00:00.000Z", - "format": "SERVER", - "environment": { "name": "Test" }, - "flags": { - "serial-id-basic-flag": { - "key": "serial-id-basic-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { "key": "on", "value": true }, - "off": { "key": "off", "value": false } - }, - "allocations": [ - { - "key": "on-allocation", - "rules": [], - "splits": [ - { "variationKey": "on", "shards": [], "serialId": 42 } - ], - "doLog": true - } - ] - }, - "serial-id-zero-flag": { - "key": "serial-id-zero-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { "key": "on", "value": true }, - "off": { "key": "off", "value": false } - }, - "allocations": [ - { - "key": "zero-allocation", - "rules": [], - "splits": [ - { "variationKey": "on", "shards": [], "serialId": 0 } - ], - "doLog": true - } - ] - }, - "serial-id-missing-flag": { - "key": "serial-id-missing-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "control": { "key": "control", "value": "control" }, - "treatment": { "key": "treatment", "value": "treatment" } - }, - "allocations": [ - { - "key": "no-serial-allocation", - "rules": [], - "splits": [ - { "variationKey": "control", "shards": [] } - ], - "doLog": true - } - ] - }, - "serial-id-targeting-flag": { - "key": "serial-id-targeting-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "vip": { "key": "vip", "value": "vip" }, - "standard": { "key": "standard", "value": "standard" } - }, - "allocations": [ - { - "key": "vip-allocation", - "rules": [ - { - "conditions": [ - { "attribute": "segment", "operator": "ONE_OF", "value": ["vip"] } - ] - } - ], - "splits": [ - { "variationKey": "vip", "shards": [], "serialId": 201 } - ], - "doLog": true - }, - { - "key": "default-allocation", - "rules": [], - "splits": [ - { "variationKey": "standard", "shards": [], "serialId": 200 } - ], - "doLog": true - } - ] - } - } -} diff --git a/tests/parametric/test_ffe/span-enrichment-flags.json b/tests/parametric/test_ffe/span-enrichment-flags.json deleted file mode 100644 index 232315bcbe8..00000000000 --- a/tests/parametric/test_ffe/span-enrichment-flags.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "createdAt": "2024-04-17T19:40:53.716Z", - "format": "SERVER", - "environment": { - "name": "Test" - }, - "flags": { - "basic-flag": { - "key": "basic-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { - "key": "on", - "value": true - }, - "off": { - "key": "off", - "value": false - } - }, - "allocations": [ - { - "key": "on-allocation", - "rules": [], - "splits": [ - { - "variationKey": "on", - "shards": [], - "serialId": 100 - } - ], - "doLog": true - } - ] - }, - "experiment-flag": { - "key": "experiment-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "control": { - "key": "control", - "value": "control" - }, - "treatment": { - "key": "treatment", - "value": "treatment" - } - }, - "allocations": [ - { - "key": "experiment-allocation", - "rules": [], - "splits": [ - { - "variationKey": "treatment", - "shards": [], - "serialId": 108 - } - ], - "doLog": true - } - ] - }, - "no-log-flag": { - "key": "no-log-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { - "key": "on", - "value": true - }, - "off": { - "key": "off", - "value": false - } - }, - "allocations": [ - { - "key": "no-log-allocation", - "rules": [], - "splits": [ - { - "variationKey": "on", - "shards": [], - "serialId": 200 - } - ], - "doLog": false - } - ] - }, - "multi-serial-flag-1": { - "key": "multi-serial-flag-1", - "enabled": true, - "variationType": "INTEGER", - "variations": { - "v1": { - "key": "v1", - "value": 1 - } - }, - "allocations": [ - { - "key": "allocation-1", - "rules": [], - "splits": [ - { - "variationKey": "v1", - "shards": [], - "serialId": 128 - } - ], - "doLog": true - } - ] - }, - "multi-serial-flag-2": { - "key": "multi-serial-flag-2", - "enabled": true, - "variationType": "INTEGER", - "variations": { - "v2": { - "key": "v2", - "value": 2 - } - }, - "allocations": [ - { - "key": "allocation-2", - "rules": [], - "splits": [ - { - "variationKey": "v2", - "shards": [], - "serialId": 130 - } - ], - "doLog": true - } - ] - } - } -} diff --git a/tests/parametric/test_ffe/test-case-disabled-flag.json b/tests/parametric/test_ffe/test-case-disabled-flag.json deleted file mode 100644 index 0da79189ade..00000000000 --- a/tests/parametric/test_ffe/test-case-disabled-flag.json +++ /dev/null @@ -1,40 +0,0 @@ -[ - { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, - "result": { - "value": 0 - } - }, - { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, - "result": { - "value": 0 - } - }, - { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", - "attributes": { - "age": 50 - }, - "result": { - "value": 0 - } - } -] diff --git a/tests/parametric/test_ffe/test-case-empty-flag.json b/tests/parametric/test_ffe/test-case-empty-flag.json deleted file mode 100644 index 52100b1fe47..00000000000 --- a/tests/parametric/test_ffe/test-case-empty-flag.json +++ /dev/null @@ -1,40 +0,0 @@ -[ - { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, - "result": { - "value": "default_value" - } - }, - { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, - "result": { - "value": "default_value" - } - }, - { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "charlie", - "attributes": { - "age": 50 - }, - "result": { - "value": "default_value" - } - } -] diff --git a/tests/parametric/test_ffe/test-case-regex-flag.json b/tests/parametric/test_ffe/test-case-regex-flag.json deleted file mode 100644 index 94aa87f23a9..00000000000 --- a/tests/parametric/test_ffe/test-case-regex-flag.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "alice", - "attributes": { - "version": "1.15.0", - "email": "alice@example.com" - }, - "result": { - "value": "partial-example" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "bob", - "attributes": { - "version": "0.20.1", - "email": "bob@test.com" - }, - "result": { - "value": "test" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "charlie", - "attributes": { - "version": "2.1.13" - }, - "result": { - "value": "none" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "derek", - "attributes": { - "version": "2.1.13", - "email": "derek@gmail.com" - }, - "result": { - "value": "none" - } - } -] diff --git a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/test-flag-that-does-not-exist.json deleted file mode 100644 index 7499bba1c50..00000000000 --- a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json +++ /dev/null @@ -1,40 +0,0 @@ -[ - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", - "attributes": { - "age": 50 - }, - "result": { - "value": 0.0 - } - } -] diff --git a/tests/parametric/test_ffe/test_configuration_sources.py b/tests/parametric/test_ffe/test_configuration_sources.py index eb23203e8d8..2c6b96c05ce 100644 --- a/tests/parametric/test_ffe/test_configuration_sources.py +++ b/tests/parametric/test_ffe/test_configuration_sources.py @@ -33,7 +33,7 @@ parametrize = pytest.mark.parametrize -UFC_VALID_FIXTURE = Path(__file__).parent / "flags-v1.json" +UFC_VALID_FIXTURE = Path(__file__).parent / "ffe-system-test-data" / "ufc-config.json" RC_PRODUCT = "FFE_FLAGS" MOCK_STATUS_ATTEMPTS = 25 MOCK_STATUS_INTERVAL_SECONDS = 0.2 diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 1fb5ed0492a..48088117087 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -7,17 +7,22 @@ from typing import Any from utils import ( + context, features, scenarios, + slow, ) from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary +from tests.parametric.test_ffe.utils import assert_feature_map, ffe_events_from_requests RC_PRODUCT = "FFE_FLAGS" RC_PATH = f"datadog/2/{RC_PRODUCT}" FFE_READY_RETRY_ATTEMPTS = 10 FFE_READY_RETRY_INTERVAL_SECONDS = 0.2 +FFE_SYSTEM_TEST_DATA_DIR = Path(__file__).parent / "ffe-system-test-data" +FFE_EVALUATION_CASES_DIR = FFE_SYSTEM_TEST_DATA_DIR / "evaluation-cases" DEPENDENT_FLAGS_DIR = Path(__file__).parent / "dependent-flags" DEPENDENT_FLAGS_CASES_DIR = DEPENDENT_FLAGS_DIR / "cases" @@ -27,10 +32,7 @@ # Load the UFC fixture file at module level def _load_ufc_fixture() -> dict[str, Any]: """Load the UFC fixture file.""" - fixture_path = Path(__file__).parent / "flags-v1.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") + fixture_path = FFE_SYSTEM_TEST_DATA_DIR / "ufc-config.json" with fixture_path.open() as f: return json.load(f) @@ -38,13 +40,26 @@ def _load_ufc_fixture() -> dict[str, Any]: def _get_test_case_files() -> list[str]: """Get all test case files from the fixtures directory.""" - test_data_dir = Path(__file__).parent - if not test_data_dir.exists(): - return [] - - # Exclude base fixtures that aren't test cases - excluded = {"flags-v1.json", "rules-wire.json", "serial-id-metadata-flags.json", "span-enrichment-flags.json"} - return [f.name for f in test_data_dir.iterdir() if f.suffix == ".json" and f.name not in excluded] + case_files = sorted(path.name for path in FFE_EVALUATION_CASES_DIR.glob("*.json")) + if not case_files: + raise FileNotFoundError(f"No canonical evaluation cases found in {FFE_EVALUATION_CASES_DIR}") + return case_files + + +def _filter_test_cases_for_platform(test_cases: list[dict[str, Any]], platform: str) -> list[dict[str, Any]]: + """Apply the canonical corpus _skip/_include selection rules.""" + applicable_cases = [] + for test_case in test_cases: + if "_skip" in test_case and "_include" in test_case: + raise ValueError("Evaluation cases cannot define both _skip and _include") + skipped_platforms = {entry["platform"] for entry in test_case.get("_skip", [])} + included_platforms = {entry["platform"] for entry in test_case.get("_include", [])} + if platform in skipped_platforms: + continue + if included_platforms and platform not in included_platforms: + continue + applicable_cases.append(test_case) + return applicable_cases def _get_dependent_flags_test_case_files() -> list[Path]: @@ -55,6 +70,12 @@ def _get_dependent_flags_test_case_files() -> list[Path]: # Load fixture at module level for reuse across tests UFC_FIXTURE_DATA = _load_ufc_fixture() ALL_TEST_CASE_FILES = _get_test_case_files() +SPLIT_FEATURE_CASES = [ + case + for filename in ALL_TEST_CASE_FILES + for case in json.loads((FFE_EVALUATION_CASES_DIR / filename).read_text()) + if "features" in case["result"] +] DEPENDENT_FLAGS_UFC_DATA = json.loads((DEPENDENT_FLAGS_DIR / "ufc-config.json").read_text()) DEPENDENT_FLAGS_TEST_CASE_FILES = _get_dependent_flags_test_case_files() @@ -112,7 +133,7 @@ def _ffe_evaluate_with_rc_retry( flag: str, variation_type: str, default_value: bool | str | float | dict[str, Any], - targeting_key: str, + targeting_key: str | None, attributes: dict[str, Any] | None = None, ) -> dict[str, Any]: result = test_library.ffe_evaluate( @@ -168,13 +189,17 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI """ # Load the test case file - test_case_path = Path(__file__).parent / test_case_file - - if not test_case_path.exists(): - pytest.skip(f"Test case file not found: {test_case_path}") + test_case_path = FFE_EVALUATION_CASES_DIR / test_case_file with test_case_path.open() as f: test_cases = json.load(f) + assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file}" + assert test_cases, f"No evaluation cases in {test_case_file}" + + platform = context.library.name + test_cases = _filter_test_cases_for_platform(test_cases, platform) + if not test_cases: + pytest.skip(f"No cases in {test_case_file} apply to platform {platform}") # Set up UFC Remote Config and wait for it to be applied _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) @@ -190,7 +215,7 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI default_value = test_case["defaultValue"] targeting_key = test_case["targetingKey"] attributes = test_case.get("attributes", {}) - expected_result = test_case["result"]["value"] + expected_result = test_case["result"] result = _ffe_evaluate_with_rc_retry( test_library, @@ -207,11 +232,22 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI actual_value = result.get("value") # Assert the evaluation result matches expected value - assert actual_value == expected_result, ( + assert actual_value == expected_result["value"], ( f"Test case {i} in {test_case_file} failed: " f"flag='{flag}', targetingKey='{targeting_key}', " - f"expected={expected_result}, actual={actual_value}" + f"expected={expected_result}, actual={result}" ) + assert result.get("reason") == expected_result["reason"], ( + f"Test case {i} in {test_case_file}: expected reason={expected_result['reason']}, result={result}" + ) + if "errorCode" in expected_result: + assert result.get("errorCode") == expected_result["errorCode"], ( + f"Test case {i} in {test_case_file}: expected errorCode={expected_result['errorCode']}, result={result}" + ) + elif expected_result["reason"] != "ERROR": + assert result.get("errorCode") in {None, ""}, ( + f"Test case {i} in {test_case_file}: unexpected evaluation error; result={result}" + ) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) @parametrize("test_case_file", DEPENDENT_FLAGS_TEST_CASE_FILES) @@ -259,3 +295,78 @@ def test_ffe_dependent_flags( f"Test case {i} in {test_case_file.name} failed: " f"flag='{flag}' unexpectedly returned errorCode={result.get('errorCode')!r}" ) + + +@scenarios.parametric +@features.feature_flags_dynamic_evaluation +@slow +class Test_Feature_Flag_Split_Features: + """Check canonical destination maps at the public hook and EVP boundaries.""" + + @parametrize( + "library_env", + [ + { + **DEFAULT_ENVVARS, + "SYSTEM_TESTS_FFE_CAPTURE_HOOK_FEATURES": "true", + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": "true", + "DD_FEATURE_FLAGS_EXPOSURES_DATADOG_LOGGING_ENABLED": "true", + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "true", + } + ], + ) + @parametrize("agent_env", [{"SYSTEM_TESTS_FFE_INTAKE": "true"}]) + @parametrize("case", SPLIT_FEATURE_CASES) + def test_split_feature_destinations( + self, case: dict[str, Any], test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + if not _filter_test_cases_for_platform([case], context.library.name): + pytest.skip(f"Case does not apply to {context.library.name}") + expected = case["result"] + _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) + assert test_library.ffe_start(UFC_FIXTURE_DATA), "Failed to start FFE provider" + result = _ffe_evaluate_with_rc_retry( + test_library, + flag=case["flag"], + variation_type=case["variationType"], + default_value=case["defaultValue"], + targeting_key=case["targetingKey"], + attributes=case.get("attributes", {}), + ) + assert result.get("value") == expected["value"], result + assert result.get("reason") == expected["reason"], result + assert result.get("errorCode") in (None, ""), result + assert_feature_map(result.get("hookFeatures"), expected["features"]["HOOK"], "HOOK callback") + + # Selected metadata remains available even when doLog=false suppresses delivery. + # Also check the actual payloads below: metadata alone cannot prove isolation. + metadata = result.get("flagMetadata") + assert isinstance(metadata, dict), result + for destination, prefix in ( + ("EXPOSURE", "__dd_exposure_feature."), + ("EVALUATION", "__dd_evaluation_feature."), + ): + selected = {key.removeprefix(prefix): value for key, value in metadata.items() if key.startswith(prefix)} + assert_feature_map(selected, expected["features"][destination], f"{destination} selected metadata") + + # Evaluation batches flush every 10s; continue briefly after a matching batch + # to observe forbidden exposures for the doLog=false case as well. + deadline = time.monotonic() + 30 + received_at = None + while True: + events = ffe_events_from_requests(test_agent.requests(), case["flag"]) + if events["EVALUATION"] and (events["EXPOSURE"] or not expected["doLog"]): + if received_at is None: + received_at = time.monotonic() + if time.monotonic() - received_at >= 2: + break + assert time.monotonic() < deadline, f"Missing split-feature events for {case['flag']}: {events}" + time.sleep(0.2) + + if not expected["doLog"]: + assert not events["EXPOSURE"], f"doLog=false emitted exposures: {events['EXPOSURE']}" + for destination, observed_events in events.items(): + for event in observed_events: + # Empty features are omitted from the wire format, but the event itself + # must exist. An absent event never stands in for an empty feature map. + assert_feature_map(event.get("features", {}), expected["features"][destination], destination) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index ba1408561c4..a12588d9d4a 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -24,13 +24,13 @@ import json import time -from pathlib import Path from typing import Any from utils import features, pytest, scenarios from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary +from tests.parametric.test_ffe.utils import load_ufc_fixture_for_cases RC_PRODUCT = "FFE_FLAGS" RC_PATH = f"datadog/2/{RC_PRODUCT}" @@ -40,18 +40,7 @@ parametrize = pytest.mark.parametrize -def _load_serial_id_metadata_fixture() -> dict[str, Any]: - """Load the UFC fixture file for serial ID metadata tests.""" - fixture_path = Path(__file__).parent / "serial-id-metadata-flags.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") - - with fixture_path.open() as f: - return json.load(f) - - -UFC_SERIAL_ID_METADATA_DATA = _load_serial_id_metadata_fixture() +UFC_SERIAL_ID_METADATA_DATA = load_ufc_fixture_for_cases("test-case-serial-id-metadata.json") DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", diff --git a/tests/parametric/test_ffe/test_span_enrichment.py b/tests/parametric/test_ffe/test_span_enrichment.py index 6a4e54bdacb..749913bb455 100644 --- a/tests/parametric/test_ffe/test_span_enrichment.py +++ b/tests/parametric/test_ffe/test_span_enrichment.py @@ -21,14 +21,18 @@ import json from utils import pytest -from pathlib import Path from typing import Any from utils import features, scenarios from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary -from tests.parametric.test_ffe.utils import decode_delta_varint, encode_delta_varint, hash_targeting_key +from tests.parametric.test_ffe.utils import ( + decode_delta_varint, + encode_delta_varint, + hash_targeting_key, + load_ufc_fixture_for_cases, +) RC_PRODUCT = "FFE_FLAGS" @@ -37,18 +41,7 @@ parametrize = pytest.mark.parametrize -def _load_span_enrichment_fixture() -> dict[str, Any]: - """Load the UFC fixture file for span enrichment tests.""" - fixture_path = Path(__file__).parent / "span-enrichment-flags.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") - - with fixture_path.open() as f: - return json.load(f) - - -UFC_SPAN_ENRICHMENT_DATA = _load_span_enrichment_fixture() +UFC_SPAN_ENRICHMENT_DATA = load_ufc_fixture_for_cases("test-case-span-enrichment-inputs.json") DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", diff --git a/tests/parametric/test_ffe/utils.py b/tests/parametric/test_ffe/utils.py index 325c825f949..b82b1c64c0a 100644 --- a/tests/parametric/test_ffe/utils.py +++ b/tests/parametric/test_ffe/utils.py @@ -9,14 +9,30 @@ Example: [100, 108, 128, 130] -> deltas [100, 8, 20, 2] -> varint bytes -> base64 """ +from collections.abc import Mapping, Sequence import base64 +import gzip import hashlib +import json +from pathlib import Path +from typing import Any # Varint encoding constants VARINT_DATA_BITS = 0x7F # Lower 7 bits contain data VARINT_CONTINUATION_BIT = 0x80 # MSB indicates more bytes follow +def load_ufc_fixture_for_cases(case_filename: str) -> dict[str, Any]: + """Load the canonical flag definitions used by a specific integration suite.""" + fixture_root = Path(__file__).parent / "ffe-system-test-data" + config = json.loads((fixture_root / "ufc-config.json").read_text()) + cases = json.loads((fixture_root / "evaluation-cases" / case_filename).read_text()) + # Keep unrelated evaluator edge cases out of transport/span integration tests. + flag_keys = dict.fromkeys(case["flag"] for case in cases) + config["flags"] = {key: config["flags"][key] for key in flag_keys} + return config + + def decode_varint(data: bytes, offset: int = 0) -> tuple[int, int]: """Decode a single varint from bytes. @@ -125,3 +141,37 @@ def hash_targeting_key(targeting_key: str) -> str: """ return hashlib.sha256(targeting_key.encode("utf-8")).hexdigest() + + +def assert_feature_map(actual: object, expected: dict[str, Any], destination: str) -> None: + """Compare exact scalar feature maps without equating false with zero.""" + assert isinstance(actual, dict), f"{destination}: missing feature map: {actual!r}" + assert actual.keys() == expected.keys(), f"{destination}: expected {expected!r}, got {actual!r}" + for key, expected_value in expected.items(): + actual_value = actual[key] + if type(expected_value) in (int, float): + valid_type = type(actual_value) in (int, float) + else: + valid_type = type(actual_value) is type(expected_value) + assert valid_type, f"{destination}.{key}: expected type {type(expected_value)}, got {type(actual_value)}" + assert actual_value == expected_value, f"{destination}.{key}: expected {expected_value!r}, got {actual_value!r}" + + +def ffe_events_from_requests(requests: Sequence[Mapping[str, Any]], flag: str) -> dict[str, list[dict[str, Any]]]: + """Read real EVP payloads recorded by the parametric test agent.""" + events: dict[str, list[dict[str, Any]]] = {"EXPOSURE": [], "EVALUATION": []} + for request in requests: + path = request["url"].split("?", 1)[0] + if path.endswith("/api/v2/exposures"): + destination, field = "EXPOSURE", "exposures" + elif path.endswith("/api/v2/flagevaluation"): + destination, field = "EVALUATION", "flagEvaluations" + else: + continue + body = base64.b64decode(request["body"]) + # The agent may already have decompressed a request while retaining its headers. + if body.startswith(b"\x1f\x8b"): + body = gzip.decompress(body) + payload = json.loads(body) + events[destination].extend(event for event in payload[field] if event.get("flag", {}).get("key") == flag) + return events diff --git a/tests/test_the_test/test_ffe_split_features.py b/tests/test_the_test/test_ffe_split_features.py new file mode 100644 index 00000000000..3b72b718a52 --- /dev/null +++ b/tests/test_the_test/test_ffe_split_features.py @@ -0,0 +1,72 @@ +"""Regression controls for canonical split-feature assertions and wire decoding.""" + +import base64 +import gzip +import json +from typing import Any + +from utils import pytest, scenarios +from tests.parametric.test_ffe.utils import assert_feature_map, ffe_events_from_requests + + +@scenarios.test_the_test +@pytest.mark.parametrize( + "actual", + [ + None, + {}, + {"zero": 0}, + {"zero": False, "empty": "", "enabled": False}, + {"zero": 0, "empty": "", "enabled": 0}, + {"zero": "0", "empty": "", "enabled": False}, + {"zero": 0, "empty": "changed", "enabled": False}, + {"zero": 0, "empty": "", "enabled": False, "wrong-destination": True}, + ], +) +def test_feature_map_rejects_missing_changed_misrouted_or_coerced_values(actual: object) -> None: + # Value/reason can remain correct while metadata is dropped or misrouted. + with pytest.raises(AssertionError): + assert_feature_map(actual, {"zero": 0, "empty": "", "enabled": False}, "EVALUATION") + + +@scenarios.test_the_test +def test_feature_map_preserves_json_number_equivalence_and_empty_maps() -> None: + assert_feature_map( + {"zero": 0.0, "number": 1.25, "false": False, "empty": ""}, + {"zero": 0, "number": 1.25, "false": False, "empty": ""}, + "HOOK", + ) + assert_feature_map({}, {}, "EVALUATION") + with pytest.raises(AssertionError): + assert_feature_map({"leaked": True}, {}, "EVALUATION") + with pytest.raises(AssertionError): + assert_feature_map(None, {}, "EVALUATION") + + +@scenarios.test_the_test +@pytest.mark.parametrize("encoding", ["identity", "gzip", "decoded-gzip"]) +def test_ffe_wire_events_keep_destination_boundaries_and_empty_features(encoding: str) -> None: + def request(path: str, field: str, events: list[dict[str, Any]]) -> dict[str, Any]: + body = json.dumps({field: events}).encode() + if encoding == "gzip": + body = gzip.compress(body) + return { + "url": f"http://agent/evp_proxy/v4/api/v2/{path}", + "headers": {"Content-Encoding": "gzip"} if encoding != "identity" else {}, + "body": base64.b64encode(body).decode(), + } + + exposure = {"flag": {"key": "flag"}, "features": {"exposure.only": False}} + evaluation = {"flag": {"key": "flag"}, "features": {"zero": 0}} + empty_evaluation = {"flag": {"key": "flag"}} + unrelated = {"flag": {"key": "other"}, "features": {"leaked": True}} + actual = ffe_events_from_requests( + [ + request("exposures", "exposures", [exposure, unrelated]), + request("flagevaluation", "flagEvaluations", [evaluation, empty_evaluation, unrelated]), + {"url": "/v0.7/config", "headers": {}, "body": "not-json"}, + ], + "flag", + ) + assert actual == {"EXPOSURE": [exposure], "EVALUATION": [evaluation, empty_evaluation]} + assert ffe_events_from_requests([], "flag") == {"EXPOSURE": [], "EVALUATION": []} diff --git a/tests/test_the_test/test_load_binary.py b/tests/test_the_test/test_load_binary.py index 149f2f47bb3..99be71f1b5f 100644 --- a/tests/test_the_test/test_load_binary.py +++ b/tests/test_the_test/test_load_binary.py @@ -4,7 +4,7 @@ from pathlib import Path import subprocess -from utils import scenarios +from utils import pytest, scenarios SCRIPT = Path("utils/scripts/load-binary.sh") @@ -23,6 +23,7 @@ def _run_loader( tmp_path: Path, version: str, *, + target: str = "c", extra_env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: bin_dir = tmp_path / "bin" @@ -39,6 +40,14 @@ def _run_loader( if [[ "${{MISSING_BRANCH:-}}" != "" && "$url" == *"${{MISSING_BRANCH}}"* ]]; then exit 22 fi +previous= +for argument in "$@"; do + if [[ "$previous" == "--output" ]]; then + printf '%s\\n' "$url" > "$argument" + exit 0 + fi + previous="$argument" +done if [[ "$url" == *"DataDog/dd-trace-c"* ]]; then printf '%s\\n' '{{"commit":{{"sha":"{C_LIBRARY_SHA}"}}}}' else @@ -68,7 +77,7 @@ def _run_loader( env.pop("AUTO_INJECT_TARGET_BRANCH", None) env.update(extra_env or {}) return subprocess.run( - ["bash", str(SCRIPT), "c", version], + ["bash", str(SCRIPT), target, version], check=False, capture_output=True, text=True, @@ -158,3 +167,40 @@ def test_missing_package_fails_with_clear_error(self, tmp_path: Path) -> None: assert result.returncode != 0 assert "OCI package does not exist or is not accessible" in result.stderr + + +@scenarios.test_the_test +class Test_LoadBinaryJava: + @pytest.mark.parametrize("branch", [None, "feature/flagging"]) + def test_development_agent_and_provider_use_same_branch(self, tmp_path: Path, branch: str | None) -> None: + extra_env = {"LIBRARY_TARGET_BRANCH": branch} if branch else {} + result = _run_loader(tmp_path, "dev", target="java", extra_env=extra_env) + + assert result.returncode == 0, result.stderr + base_url = f"https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/{branch or 'master'}" + for artifact in ("dd-java-agent.jar", "dd-openfeature.jar"): + assert (tmp_path / "binaries" / artifact).read_text(encoding="utf-8").strip() == f"{base_url}/{artifact}" + assert (tmp_path / "curl-calls").read_text().splitlines() == [ + f"{base_url}/dd-java-agent.jar", + f"{base_url}/dd-openfeature.jar", + ] + + @pytest.mark.parametrize("branch", [None, "feature/flagging"]) + def test_lambda_development_build_loads_only_agent(self, tmp_path: Path, branch: str | None) -> None: + extra_env = {"MISSING_BRANCH": "dd-openfeature.jar"} + if branch: + extra_env["LIBRARY_TARGET_BRANCH"] = branch + result = _run_loader(tmp_path, "dev", target="java_lambda", extra_env=extra_env) + + assert result.returncode == 0, result.stderr + agent_url = f"https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/{branch or 'master'}/dd-java-agent.jar" + assert (tmp_path / "binaries/dd-java-agent.jar").read_text().strip() == agent_url + assert not (tmp_path / "binaries/dd-openfeature.jar").exists() + assert (tmp_path / "curl-calls").read_text().splitlines() == [agent_url] + + def test_missing_development_provider_fails(self, tmp_path: Path) -> None: + result = _run_loader(tmp_path, "dev", target="java", extra_env={"MISSING_BRANCH": "dd-openfeature.jar"}) + + assert result.returncode != 0 + assert (tmp_path / "binaries/dd-java-agent.jar").exists() + assert not (tmp_path / "binaries/dd-openfeature.jar").exists() diff --git a/tests/test_the_test/test_update_ffe_fixtures.py b/tests/test_the_test/test_update_ffe_fixtures.py new file mode 100644 index 00000000000..8f53acd82ee --- /dev/null +++ b/tests/test_the_test/test_update_ffe_fixtures.py @@ -0,0 +1,145 @@ +# ruff: noqa: SLF001 + +from pathlib import Path +import shutil + +from utils import pytest + +from utils import scenarios +from utils.scripts import update_ffe_fixtures + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["main", "feature/fixture-update", "v1.2.3", "ea8b5cc5"]) +def test_validate_fixture_ref_accepts_git_refs(fixture_ref: str) -> None: + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["", "-main", "../main", "main^", "main ref"]) +def test_validate_fixture_ref_rejects_unsafe_refs(fixture_ref: str) -> None: + with pytest.raises(ValueError, match="Invalid FFE fixture ref"): + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_imports_only_server_evaluator_data(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + (source / "evaluation-cases").mkdir(parents=True) + snapshot.mkdir() + (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + (source / "README.md").write_text("upstream documentation", encoding="utf-8") + (source / "AGENTS.md").write_text("upstream instructions", encoding="utf-8") + (source / "precomputed-assignments").mkdir() + + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + assert (snapshot / "ufc-config.json").is_file() + assert (snapshot / "evaluation-cases" / "case.json").is_file() + assert not (snapshot / "README.md").exists() + assert not (snapshot / "AGENTS.md").exists() + assert not (snapshot / "precomputed-assignments").exists() + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_rejects_symbolic_links(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + source.mkdir() + snapshot.mkdir() + (source / "evaluation-cases").mkdir() + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + target = source / "target.json" + target.write_text("{}", encoding="utf-8") + (source / "evaluation-cases" / "linked.json").symlink_to(target) + + with pytest.raises(ValueError, match="Refusing to copy symbolic link"): + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + +@scenarios.test_the_test +def test_validate_fixture_snapshot_counts_evaluations(tmp_path: Path) -> None: + cases_directory = tmp_path / "evaluation-cases" + cases_directory.mkdir() + (tmp_path / "ufc-config.json").write_text('{"flags": {"one": {}}}', encoding="utf-8") + (cases_directory / "first.json").write_text('[{"flag": "one"}]', encoding="utf-8") + (cases_directory / "second.json").write_text('[{"flag": "two"}, {"flag": "three"}]', encoding="utf-8") + + assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 3 + + +@scenarios.test_the_test +def test_have_same_contents_ignores_generated_source_metadata(tmp_path: Path) -> None: + snapshot = tmp_path / "snapshot" + destination = tmp_path / "destination" + snapshot.mkdir() + destination.mkdir() + (snapshot / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "SOURCE.md").write_text("generated", encoding="utf-8") + + assert update_ffe_fixtures._have_same_contents(snapshot, destination) + + (destination / "fixture.json").write_text('{"changed": true}', encoding="utf-8") + assert not update_ffe_fixtures._have_same_contents(snapshot, destination) + + +@scenarios.test_the_test +def test_update_fixtures_preserves_snapshot_on_validation_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "upstream" + destination = tmp_path / "destination" + (source / "evaluation-cases").mkdir(parents=True) + (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") + (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + destination.mkdir() + (destination / "SOURCE.md").write_text("original snapshot", encoding="utf-8") + + def fake_git(working_directory: Path, *arguments: str) -> str: + if arguments[0] == "checkout": + shutil.copytree(source, working_directory, dirs_exist_ok=True) + return "a" * 40 + + monkeypatch.setattr(update_ffe_fixtures, "_run_git", fake_git) + monkeypatch.setattr(update_ffe_fixtures, "FIXTURE_DESTINATION", destination) + with pytest.raises(ValueError, match="at least one evaluation case"): + update_ffe_fixtures.update_fixtures("main") + assert (destination / "SOURCE.md").read_text() == "original snapshot" + + +@scenarios.test_the_test +def test_update_fixtures_records_source_and_removes_stale_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "upstream" + destination = tmp_path / "destination" + output = tmp_path / "outputs" + (source / "evaluation-cases").mkdir(parents=True) + (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") + case_file = source / "evaluation-cases" / "case.json" + case_file.write_text('[{"flag": "flag"}]', encoding="utf-8") + destination.mkdir() + (destination / "stale.json").write_text("[]", encoding="utf-8") + + def fake_git(working_directory: Path, *arguments: str) -> str: + if arguments[0] == "checkout": + shutil.copytree(source, working_directory, dirs_exist_ok=True) + return "a" * 40 + + monkeypatch.setattr(update_ffe_fixtures, "_run_git", fake_git) + monkeypatch.setattr(update_ffe_fixtures, "FIXTURE_DESTINATION", destination) + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + assert update_ffe_fixtures.update_fixtures("main") + assert not (destination / "stale.json").exists() + assert (destination / "evaluation-cases" / "case.json").read_bytes() == case_file.read_bytes() + metadata = (destination / "SOURCE.md").read_bytes() + assert ("a" * 40).encode() in metadata + assert "fixture_count=1\nchanged=true\n" in output.read_text() + + (source / "README.md").write_text("documentation-only upstream change", encoding="utf-8") + assert not update_ffe_fixtures.update_fixtures("main") + assert (destination / "SOURCE.md").read_bytes() == metadata + assert output.read_text().endswith("changed=false\n") diff --git a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java index b3da4a7f886..3cb7ea35d97 100644 --- a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java +++ b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java @@ -8,6 +8,7 @@ import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.FeatureProvider; import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.Hook; import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.NoOpProvider; import dev.openfeature.sdk.OpenFeatureAPI; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; @RestController @RequestMapping("/ffe") @@ -37,6 +39,28 @@ public class FeatureFlagEvaluatorController { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlagEvaluatorController.class); + private static final boolean CAPTURE_HOOK_FEATURES = + Boolean.parseBoolean(System.getenv("SYSTEM_TESTS_FFE_CAPTURE_HOOK_FEATURES")); + private static final ThreadLocal HOOK_FEATURES = new ThreadLocal<>(); + + private static void captureHookFeatures(Client client) { + // Load the optional public API only for tests whose manifest requires it. + // Older dd-openfeature versions must still build and run the baseline suite. + try { + Class hookClass = Class.forName("com.datadog.openfeature.ExposureHook"); + Consumer callback = evaluation -> { + try { + HOOK_FEATURES.set(evaluation.getClass().getMethod("getFeatures").invoke(evaluation)); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Cannot capture ExposureHook features", e); + } + }; + client.addHooks((Hook) hookClass.getConstructor(Consumer.class).newInstance(callback)); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Split-feature tests require the public ExposureHook API", e); + } + } + @Configuration public static class FeatureFlagEvaluatorConfig { @@ -61,7 +85,11 @@ public ProviderState getState() { }; } api.setProviderAndWait(provider); - return api.getClient(); + Client client = api.getClient(); + if (CAPTURE_HOOK_FEATURES) { + captureHookFeatures(client); + } + return client; } } @@ -81,6 +109,7 @@ public ResponseEntity start() { @PostMapping(value = "/evaluate", consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) public ResponseEntity> evaluate(@RequestBody final EvaluateRequest request) { + HOOK_FEATURES.remove(); Object value; String reason; String errorCode = null; @@ -123,6 +152,10 @@ public ResponseEntity> evaluate(@RequestBody final EvaluateR result.put("errorCode", errorCode); result.put("value", value); result.put("flagMetadata", flagMetadata); + if (CAPTURE_HOOK_FEATURES) { + result.put("hookFeatures", HOOK_FEATURES.get()); + } + HOOK_FEATURES.remove(); return ResponseEntity.ok(result); } diff --git a/utils/build/docker/nodejs/parametric/server.js b/utils/build/docker/nodejs/parametric/server.js index 29c6eb9c627..7305b199021 100644 --- a/utils/build/docker/nodejs/parametric/server.js +++ b/utils/build/docker/nodejs/parametric/server.js @@ -692,6 +692,7 @@ app.post('/ffe/evaluate', async (req, res) => { res.json({ value: details.value, reason: details.reason, + errorCode: details.errorCode, flagMetadata: details.flagMetadata ?? null }); }); diff --git a/utils/build/docker/ruby/parametric/server.rb b/utils/build/docker/ruby/parametric/server.rb index d5e92a0e676..e33dedac6a3 100644 --- a/utils/build/docker/ruby/parametric/server.rb +++ b/utils/build/docker/ruby/parametric/server.rb @@ -877,20 +877,26 @@ def handle_ffe_evaluation(req, res) options = { flag_key: args.flag, default_value: args.default_value, evaluation_context: context } + options[:default_value] = args.default_value.to_f if args.variation_type == 'NUMERIC' - value = + details = case args.variation_type - when 'BOOLEAN'then client.fetch_boolean_value(**options) - when 'STRING' then client.fetch_string_value(**options) - when 'INTEGER' then client.fetch_integer_value(**options) - when 'NUMERIC' then client.fetch_float_value(**options) - when 'JSON' then client.fetch_object_value(**options) - else 'FATAL_UNEXPECTED_VARIATION_TYPE' + when 'BOOLEAN'then client.fetch_boolean_details(**options) + when 'STRING' then client.fetch_string_details(**options) + when 'INTEGER' then client.fetch_integer_details(**options) + when 'NUMERIC' then client.fetch_float_details(**options) + when 'JSON' then client.fetch_object_details(**options) + else raise ArgumentError, "Unexpected variation type: #{args.variation_type}" end - res.write({value: value, reason: 'DEFAULT'}.to_json) + res.write({ + value: details.value, + reason: details.reason, + errorCode: details.error_code, + flagMetadata: details.flag_metadata + }.to_json) rescue => e - res.write({value: args.default_value, reason: 'ERROR'}.to_json) + res.write({value: args.default_value, reason: 'ERROR', errorCode: 'GENERAL', errorMessage: e.message}.to_json) end end diff --git a/utils/docker_fixtures/_test_agent.py b/utils/docker_fixtures/_test_agent.py index 6422405c3f0..ea6f7953be5 100644 --- a/utils/docker_fixtures/_test_agent.py +++ b/utils/docker_fixtures/_test_agent.py @@ -35,6 +35,31 @@ DEFAULT_OTLP_HTTP_PORT = 4318 DEFAULT_OTLP_GRPC_PORT = 4317 +# The pinned test agent only registers FFE exposures on EVP v2. Opt-in tests use +# its supported app factory to register the remaining FFE intake routes with the +# existing recording/200-response handler; no requests are sent to Datadog. +_FFE_INTAKE_COMMAND = """ +from aiohttp import web +from ddapm_test_agent.agent import main, make_app + + +def make_ffe_app(**kwargs): + kwargs["agent_url"] = "" + app = make_app(**kwargs) + agent = app["agent"] + paths = ( + "/evp_proxy/v2/api/v2/flagevaluation", + "/evp_proxy/v4/api/v2/exposures", + "/evp_proxy/v4/api/v2/flagevaluation", + ) + agent._forward_endpoints.extend(paths) + app.add_routes([web.post(path, agent.handle_evp_proxy_v2_api_v2_exposures) for path in paths]) + return app + + +main(app_factory=make_ffe_app) +""" + def _request_token(request: pytest.FixtureRequest) -> str: token = "" @@ -132,6 +157,9 @@ def start_agent( open(log_path, "w+", encoding="utf-8") as log_file, docker_run( image=self.image, + command=["python", "-c", _FFE_INTAKE_COMMAND] + if agent_env.get("SYSTEM_TESTS_FFE_INTAKE") == "true" + else None, name=container_name, env=env, volumes={ diff --git a/utils/docker_fixtures/_test_clients/_test_client_parametric.py b/utils/docker_fixtures/_test_clients/_test_client_parametric.py index a8ca37398c9..9547e48610e 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_parametric.py +++ b/utils/docker_fixtures/_test_clients/_test_client_parametric.py @@ -842,7 +842,7 @@ def ffe_evaluate( flag: str, variation_type: str, default_value: bool | str | float | dict, - targeting_key: str, + targeting_key: str | None, attributes: dict | None = None, span_id: int | str | None = None, ) -> dict: @@ -1301,7 +1301,7 @@ def ffe_evaluate( flag: str, variation_type: str, default_value: bool | str | float | dict, - targeting_key: str, + targeting_key: str | None, attributes: dict | None = None, span_id: int | str | None = None, ) -> dict: diff --git a/utils/mocked_backend/_backend_ffe.py b/utils/mocked_backend/_backend_ffe.py index b998e399e35..be414dd0063 100644 --- a/utils/mocked_backend/_backend_ffe.py +++ b/utils/mocked_backend/_backend_ffe.py @@ -73,7 +73,7 @@ CONFIG_QUERY = urlencode({"dd_env": EXPECTED_DD_ENV}) REPO_ROOT = Path(__file__).parents[2] # relative to the repository root: the mocked backend container mounts it at the same place -UFC_FIXTURE_RELATIVE_PATH = "tests/parametric/test_ffe/flags-v1.json" +UFC_FIXTURE_RELATIVE_PATH = "tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json" UFC_FIXTURE_PATH = REPO_ROOT / UFC_FIXTURE_RELATIVE_PATH MALFORMED_UFC_BYTES = b'{"flags": [' UFC_RESPONSE_TYPE = "universal-flag-configuration" diff --git a/utils/scripts/update_ffe_fixtures.py b/utils/scripts/update_ffe_fixtures.py new file mode 100644 index 00000000000..e7f3b95f484 --- /dev/null +++ b/utils/scripts/update_ffe_fixtures.py @@ -0,0 +1,206 @@ +"""Refresh the checked-in FFE fixture snapshot from its canonical repository.""" + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_DESTINATION = REPOSITORY_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" +FIXTURE_REPOSITORY = "https://github.com/DataDog/ffe-system-test-data.git" +FIXTURE_REF_PATTERN = re.compile(r"[A-Za-z0-9._/-]+") + + +def _validate_fixture_ref(fixture_ref: str) -> None: + if ( + not fixture_ref + or fixture_ref.startswith("-") + or ".." in fixture_ref + or FIXTURE_REF_PATTERN.fullmatch(fixture_ref) is None + ): + msg = f"Invalid FFE fixture ref: {fixture_ref!r}" + raise ValueError(msg) + + +def _run_git(working_directory: Path, *arguments: str) -> str: + environment = { + **os.environ, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + } + result = subprocess.run( + ["git", *arguments], + cwd=working_directory, + env=environment, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _copy_entry(source: Path, destination: Path) -> None: + if source.is_symlink(): + msg = f"Refusing to copy symbolic link from FFE fixture repository: {source}" + raise ValueError(msg) + + if source.is_file(): + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + return + + if not source.is_dir(): + msg = f"Refusing to copy unsupported filesystem entry from FFE fixture repository: {source}" + raise ValueError(msg) + + destination.mkdir(parents=True, exist_ok=True) + for child in sorted(source.iterdir()): + _copy_entry(child, destination / child.name) + + +def _copy_fixture_snapshot(source: Path, snapshot: Path) -> None: + # Import only the server evaluator corpus that this suite consumes. Other + # upstream suites (for example precomputed assignments) have different formats. + for name in ("ufc-config.json", "evaluation-cases"): + _copy_entry(source / name, snapshot / name) + + +def _validate_fixture_snapshot(snapshot: Path) -> int: + config_path = snapshot / "ufc-config.json" + cases_directory = snapshot / "evaluation-cases" + if not config_path.is_file() or not cases_directory.is_dir(): + msg = "FFE fixture repository does not contain the expected fixture layout" + raise ValueError(msg) + + with config_path.open(encoding="utf-8") as config_file: + config = json.load(config_file) + if not isinstance(config, dict) or not isinstance(config.get("flags"), dict) or not config["flags"]: + msg = "FFE UFC fixture must contain a non-empty flags object" + raise ValueError(msg) + + case_files = sorted(cases_directory.glob("*.json")) + if any(not path.is_file() or path.suffix != ".json" for path in cases_directory.iterdir()): + msg = "FFE evaluation-cases must contain only JSON files" + raise ValueError(msg) + if not case_files: + msg = "No FFE JSON fixture files found" + raise ValueError(msg) + + fixture_count = 0 + for case_file in case_files: + with case_file.open(encoding="utf-8") as fixture_file: + test_cases = json.load(fixture_file) + if not isinstance(test_cases, list): + msg = f"{case_file} must contain a JSON array of test cases" + raise TypeError(msg) + if not test_cases: + msg = f"{case_file} must contain at least one evaluation case" + raise ValueError(msg) + fixture_count += len(test_cases) + + if fixture_count == 0: + msg = "No FFE fixture test cases found" + raise ValueError(msg) + + return fixture_count + + +def _relative_files(directory: Path, excluded_files: frozenset[str] = frozenset()) -> list[Path]: + return sorted( + path.relative_to(directory) + for path in directory.rglob("*") + if path.is_file() and path.relative_to(directory).as_posix() not in excluded_files + ) + + +def _have_same_contents(snapshot: Path, destination: Path) -> bool: + if not destination.is_dir(): + return False + + snapshot_files = _relative_files(snapshot) + destination_files = _relative_files(destination, frozenset({"SOURCE.md"})) + if snapshot_files != destination_files: + return False + + return all( + (snapshot / relative_path).read_bytes() == (destination / relative_path).read_bytes() + for relative_path in snapshot_files + ) + + +def _write_source_metadata(snapshot: Path, source_commit: str) -> None: + (snapshot / "SOURCE.md").write_text( + f"""# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: {source_commit} + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in +ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed +draft test PR only when the server evaluator fixture contents change. +""", + encoding="utf-8", + ) + + +def _set_github_outputs(source_commit: str, fixture_count: int, *, changed: bool) -> None: + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with Path(github_output).open("a", encoding="utf-8") as output_file: + output_file.write(f"source_commit={source_commit}\n") + output_file.write(f"fixture_count={fixture_count}\n") + output_file.write(f"changed={str(changed).lower()}\n") + + +def update_fixtures(fixture_ref: str) -> bool: + """Update the snapshot and return whether its allowed contents changed.""" + _validate_fixture_ref(fixture_ref) + + with tempfile.TemporaryDirectory(prefix="ffe-system-test-data-") as temporary_directory: + temporary_path = Path(temporary_directory) + source = temporary_path / "source" + snapshot = temporary_path / "snapshot" + source.mkdir() + snapshot.mkdir() + + _run_git(source, "init", "--quiet") + _run_git(source, "remote", "add", "origin", FIXTURE_REPOSITORY) + _run_git(source, "fetch", "--quiet", "--depth", "1", "origin", fixture_ref) + _run_git(source, "checkout", "--quiet", "--detach", "FETCH_HEAD") + source_commit = _run_git(source, "rev-parse", "HEAD") + + _copy_fixture_snapshot(source, snapshot) + fixture_count = _validate_fixture_snapshot(snapshot) + changed = not _have_same_contents(snapshot, FIXTURE_DESTINATION) + + if changed: + _write_source_metadata(snapshot, source_commit) + if FIXTURE_DESTINATION.exists(): + shutil.rmtree(FIXTURE_DESTINATION) + shutil.copytree(snapshot, FIXTURE_DESTINATION) + + _set_github_outputs(source_commit, fixture_count, changed=changed) + print(f"Checked FFE fixtures from DataDog/ffe-system-test-data@{source_commit}") + print(f"Loaded {fixture_count} canonical fixture cases") + print(f"Fixture snapshot changed: {str(changed).lower()}") + return changed + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ref", default="main", help="Branch, tag, or commit from DataDog/ffe-system-test-data") + arguments = parser.parse_args() + update_fixtures(arguments.ref) + + +if __name__ == "__main__": + main()