test(ffe): consume canonical fixture snapshot

.github/CODEOWNERS

@@ -12,6 +12,9 @@
 /tests/ @DataDog/system-tests-reviewers
 
 # FFE
+/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core
+/.github/workflows/update-ffe-fixtures.yml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core
+/utils/scripts/update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core
 /utils/mocked_backend/_backend_ffe.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core
 /utils/build/docker/dotnet/parametric/Endpoints/FfeTestApi.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core
 /utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core
@@ -24,6 +27,7 @@
 /tests/ffe/ @DataDog/feature-flagging-and-experimentation-sdk
 /tests/test_the_test/test_ffe_tests.py @DataDog/feature-flagging-and-experimentation-sdk
 /tests/test_the_test/test_mock_ffe_agentless_backend.py @DataDog/feature-flagging-and-experimentation-sdk
+/tests/test_the_test/test_update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk
 /tests/parametric/test_ffe/ @DataDog/feature-flagging-and-experimentation-sdk
 
 # SDK cap

.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml

@@ -0,0 +1,14 @@
+---
+issuer: https://token.actions.githubusercontent.com
+
+subject: repo:DataDog/system-tests:ref:refs/heads/main
+
+claim_pattern:
+  event_name: (workflow_dispatch|schedule)
+  ref: refs/heads/main
+  ref_protected: "true"
+  job_workflow_ref: DataDog/system-tests/\.github/workflows/update-ffe-fixtures\.yml@refs/heads/main
+
+permissions:
+  contents: write
+  pull_requests: write

.github/workflows/update-ffe-fixtures.yml

@@ -0,0 +1,87 @@
+---
+name: Update FFE fixtures
+
+on:
+  workflow_dispatch:
+    inputs:
+      fixture_ref:
+        description: Branch, tag, or commit from DataDog/ffe-system-test-data
+        required: true
+        default: main
+        type: string
+  schedule:
+    - cron: "0 0 * * 0"
+
+concurrency:
+  group: update-ffe-fixtures
+  cancel-in-progress: false
+
+jobs:
+  update-ffe-fixtures:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+        with:
+          persist-credentials: false
+
+      - name: Update fixture snapshot
+        id: fixtures
+        env:
+          FFE_FIXTURE_REF: ${{ inputs.fixture_ref || 'main' }}
+        run: python3 utils/scripts/update_ffe_fixtures.py --ref "$FFE_FIXTURE_REF"
+
+      - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a  # v1.0.4
+        if: steps.fixtures.outputs.changed == 'true'
+        id: octo-sts
+        with:
+          scope: DataDog/system-tests
+          policy: self.update-ffe-fixtures.create-pr
+
+      - name: Check for an open fixture update PR
+        if: steps.fixtures.outputs.changed == 'true'
+        id: existing-pr
+        env:
+          GH_TOKEN: ${{ steps.octo-sts.outputs.token }}
+        run: |
+          count=$(gh pr list --repo "$GITHUB_REPOSITORY" \
+            --head actions/update-ffe-fixtures --state open \
+            --json headRefName,headRepositoryOwner,isCrossRepository \
+            --jq '[.[] | select(
+              .headRefName == "actions/update-ffe-fixtures" and
+              .headRepositoryOwner.login == "DataDog" and
+              (.isCrossRepository | not)
+            )] | length')
+          echo "count=$count" >> "$GITHUB_OUTPUT"
+
+      - name: Create Pull Request
+        if: >-
+          steps.fixtures.outputs.changed == 'true' && steps.existing-pr.outputs.count == '0'
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1  # v8.1.1
+        with:
+          token: ${{ steps.octo-sts.outputs.token }}
+          branch: actions/update-ffe-fixtures
+          base: main
+          commit-message: "test(ffe): update canonical fixtures"
+          title: "test(ffe): update canonical fixtures"
+          draft: true
+          sign-commits: true
+          add-paths: tests/parametric/test_ffe/ffe-system-test-data
+          body: |
+            ## Motivation
+
+            Keep SDK behavior consistent with the shared FFE evaluator contract.
+
+            ## Changes
+
+            Refresh the checked-in snapshot to [${{ steps.fixtures.outputs.source_commit }}](https://github.com/DataDog/ffe-system-test-data/commit/${{ steps.fixtures.outputs.source_commit }}), containing ${{ steps.fixtures.outputs.fixture_count }} evaluation cases.
+
+            Ordinary test runs use the snapshot without network access. Resolve newly exposed evaluator failures before merging; correct shared expectations upstream only when they are wrong. Scheduled updates leave an existing update PR untouched to preserve review and fixes.
+
+            ## Workflow
+
+            1. Keep this PR as a draft while validating the updated fixtures against the supported SDKs.
+            2. Follow the repository's [review workflow](https://github.com/DataDog/system-tests/blob/main/.github/pull_request_template.md).

manifests/cpp.yml

@@ -188,6 +188,7 @@ manifest:
   tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: missing_feature (Automatic baggage propagation is not implemented)
   tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (cpp is not in scope for v1 agentless server SDK configuration-source coverage)
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client)
   tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js)
   tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature
   tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key)

manifests/dotnet.yml

@@ -1207,6 +1207,7 @@ manifest:
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0'  # Modified by easy win activation script
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694)
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature  # Created by easy win activation script
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client)
   tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js)
   tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature
   tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key)

manifests/golang.yml

@@ -2176,6 +2176,7 @@ manifest:
   tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev  # Easy win for all weblogs and version 2.10.1
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694)
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client)
   tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js)
   tests/parametric/test_ffe/test_serial_id_metadata.py: v2.10.1
   tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata:

manifests/java.yml

@@ -4251,6 +4251,7 @@ manifest:
   tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694)
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: v1.69.0-SNAPSHOT
   tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js)
   tests/parametric/test_ffe/test_serial_id_metadata.py: v1.66.0
   tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key)

manifests/nodejs.yml

@@ -2675,6 +2675,7 @@ manifest:
   tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694)
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client)
   tests/parametric/test_ffe/test_rules_configuration.py: 7.0.0-pre  # Exact development version until the first containing release is confirmed.
   tests/parametric/test_ffe/test_serial_id_metadata.py: v6.18.0
   tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key)

manifests/php.yml

@@ -1629,6 +1629,7 @@ manifest:
   tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694)
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client)
   tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js)
   tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature
   tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key)

manifests/python.yml

@@ -3138,6 +3138,7 @@ manifest:
   tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694)
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client)
   tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js)
   tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature
   tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key)

manifests/ruby.yml

@@ -2699,6 +2699,7 @@ manifest:
   tests/parametric/test_ffe/test_configuration_sources.py: v2.46.0-dev
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694)
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client)
   tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js)
   tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature
   tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key)

manifests/rust.yml

@@ -297,6 +297,7 @@ manifest:
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=0.5.3-dev+fd1307f'  # TODO: a lower version might be supported
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694)
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client)
   tests/parametric/test_ffe/test_rules_configuration.py: missing_feature (Standalone rules configuration helpers are currently tested only in Node.js)
   tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature
   tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key)

tests/ffe/README.md

@@ -13,6 +13,52 @@ This directory contains system tests for the Feature Flags & Experimentation (FF
 | `test_flag_eval_evp.py` | Server-side EVP flagevaluation payloads, aggregation, and bounds |
 | `test_evaluation_telemetry_opt_out.py` | Per-subject evaluation telemetry opt-out, default-off behavior, and unchanged exposures |
 
+## Canonical parametric fixtures
+
+The parametric evaluation suite reads the checked-in snapshot under
+`tests/parametric/test_ffe/ffe-system-test-data`. Its upstream source and exact
+commit are recorded in `SOURCE.md` inside that directory.
+
+Change shared evaluator cases in
+[`DataDog/ffe-system-test-data`](https://github.com/DataDog/ffe-system-test-data)
+first, then refresh the snapshot:
+
+```bash
+python3 utils/scripts/update_ffe_fixtures.py --ref <branch-tag-or-commit>
+```
+
+The updater imports `ufc-config.json` and `evaluation-cases/*.json` only. Ordinary
+test runs use those checked-in files without submodules or network access. A
+weekly/manual workflow opens a signed draft PR when their contents change and
+leaves any existing fixture update PR untouched. Review and resolve newly exposed
+SDK failures before merging an update.
+
+Dynamic evaluation checks each case's value, reason, and explicit error code.
+`Test_Feature_Flag_Split_Features` additionally checks every canonical case with
+`result.features` against the public exposure-hook callback and actual exposure
+and evaluation payloads received by the parametric test agent. Exact maps detect
+missing values, extra keys, destination leaks, and boolean/number coercion while
+preserving JSON numeric equivalence. This covers false, zero, empty strings,
+first-valid-key selection, and selected-allocation isolation. For `doLog=false`,
+selected metadata is checked separately and exposure delivery must stay absent.
+
+The destination test currently runs with Java's `1.69.0-SNAPSHOT` SDK and later;
+other clients are marked `incomplete_test_app` until they capture their public
+hook output. Their baseline evaluation cases continue to run. The Java harness
+loads the optional hook API only when this test enables capture, so older SDKs
+can still compile and run baseline tests. Unsupported clients skip this slower
+delivery test before container setup. An opt-in test-agent app factory registers
+the FFE intake routes missing from the pinned agent image and records their
+original payloads. This proves local delivery, not Datadog intake or production.
+
+Cases with optional `capabilities` still run their baseline `result`; enabling
+capability-specific provider options requires separate harness support.
+Remote Configuration tests and the mocked agentless backend read the same UFC.
+Serial-ID metadata and span-enrichment tests select their flag definitions from
+the shared corpus using their dedicated evaluation-case files. Their integration
+harnesses retain trace construction and metadata assertions; no separate local
+configuration copies are needed.
+
 ## Running FFE Tests
 
 Standalone rules-configuration helpers are covered in `tests/parametric/test_ffe/test_rules_configuration.py` (Node.js only for now). These tests use `dd-trace/openfeature/rules-based` in a fresh process without initializing a tracer or provider, fetch protobuf over HTTP, and validate the portable wire output, channel credentials, typed errors, deadlines and cancellation during headers/body delivery, transport failures, invalid options, and redirect protection. They do not initialize a browser SDK or contact Datadog.
@@ -330,7 +376,7 @@ The tests use `interfaces.agent.get_metrics()` to retrieve metrics from the agen
 
 ### 1. Test returns `parse_error` instead of expected reason
 - Check UFC fixture format (especially `totalShards` placement)
-- Verify the fixture matches `flags-v1.json` format in dd-trace-py
+- Verify the fixture matches the canonical `ffe-system-test-data/ufc-config.json` format
 
 ### 2. Test returns `static` instead of `split`
 - Need multiple variations with different shard ranges

tests/ffe/test_flag_eval_evp.py

@@ -244,7 +244,7 @@ def assert_no_duplicate_visible_events(events: list[tuple[JSON, JSON]]) -> None:
 class FlagevaluationEgressContract:
     """One flag-evaluation contract inherited by each supported topology adapter."""
 
-    # Agentless scenarios preload flags-v1.json before the weblog starts. Use the
+    # Agentless scenarios preload the canonical UFC before the weblog starts. Use the
     # same known fixture key in every topology; the Agent scenario installs an
     # equivalent fixture through Remote Config below.
     flag_key = "empty-targeting-key-flag"

tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md

@@ -0,0 +1,12 @@
+# FFE Fixture Snapshot
+
+These files are copied from the canonical FFE fixture repository.
+
+Canonical source: https://github.com/DataDog/ffe-system-test-data
+Source commit: caa6d1ac3e5fbda0ce57ef3ffff3772901f3f466
+
+Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in
+ffe-system-test-data first, then refresh this snapshot.
+
+The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed
+draft test PR only when the server evaluator fixture contents change.

tests/parametric/test_ffe/test-case-boolean-false-assignment.json → tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json

@@ -1,38 +1,41 @@
 [
   {
-    "flag": "boolean-false-assignment",
-    "variationType": "BOOLEAN",
-    "defaultValue": true,
-    "targetingKey": "alice",
     "attributes": {
       "should_disable_feature": true
     },
+    "defaultValue": true,
+    "flag": "boolean-false-assignment",
     "result": {
+      "reason": "TARGETING_MATCH",
       "value": false
-    }
+    },
+    "targetingKey": "alice",
+    "variationType": "BOOLEAN"
   },
   {
-    "flag": "boolean-false-assignment",
-    "variationType": "BOOLEAN",
-    "defaultValue": true,
-    "targetingKey": "bob",
     "attributes": {
       "should_disable_feature": false
     },
+    "defaultValue": true,
+    "flag": "boolean-false-assignment",
     "result": {
+      "reason": "TARGETING_MATCH",
       "value": true
-    }
+    },
+    "targetingKey": "bob",
+    "variationType": "BOOLEAN"
   },
   {
-    "flag": "boolean-false-assignment",
-    "variationType": "BOOLEAN",
-    "defaultValue": true,
-    "targetingKey": "charlie",
     "attributes": {
       "unknown_attribute": "value"
     },
+    "defaultValue": true,
+    "flag": "boolean-false-assignment",
     "result": {
+      "reason": "DEFAULT",
       "value": true
-    }
+    },
+    "targetingKey": "charlie",
+    "variationType": "BOOLEAN"
   }
-]
+]